From e1f47827a05e56f2f6297b32f2df15cc77d545e4 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 02:13:50 +0200 Subject: [PATCH 01/56] feat: run firstmate as a Kubernetes agent OS --- .agents/skills/kubernetes-fleet/SKILL.md | 26 ++ .dockerignore | 16 ++ AGENTS.md | 1 + Dockerfile | 54 ++++ README.md | 6 + THIRD_PARTY_NOTICES.md | 9 + bin/agent-os-container-entrypoint.sh | 15 + bin/agent-os-crewmate.sh | 98 +++++++ bin/agent-os-local.sh | 50 ++++ deploy/orbstack/kustomization.yaml | 7 + deploy/orbstack/namespace.yaml | 8 + deploy/orbstack/primary.yaml | 74 +++++ deploy/orbstack/rbac.yaml | 23 ++ docs/kubernetes.md | 83 ++++++ .../2026-07-12-orbstack-agent-os-demo.md | 258 ++++++++++++++++++ tests/agent-os-container.test.sh | 22 ++ tests/agent-os-kubernetes.test.sh | 75 +++++ tests/agent-os-local.test.sh | 93 +++++++ 18 files changed, 918 insertions(+) create mode 100644 .agents/skills/kubernetes-fleet/SKILL.md create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 THIRD_PARTY_NOTICES.md create mode 100755 bin/agent-os-container-entrypoint.sh create mode 100755 bin/agent-os-crewmate.sh create mode 100755 bin/agent-os-local.sh create mode 100644 deploy/orbstack/kustomization.yaml create mode 100644 deploy/orbstack/namespace.yaml create mode 100644 deploy/orbstack/primary.yaml create mode 100644 deploy/orbstack/rbac.yaml create mode 100644 docs/kubernetes.md create mode 100644 docs/superpowers/plans/2026-07-12-orbstack-agent-os-demo.md create mode 100755 tests/agent-os-container.test.sh create mode 100755 tests/agent-os-kubernetes.test.sh create mode 100755 tests/agent-os-local.test.sh diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md new file mode 100644 index 000000000..a43913e54 --- /dev/null +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -0,0 +1,26 @@ +--- +name: kubernetes-fleet +description: "Operate Agent OS crewmates as Kubernetes Pods with persistent homes and explicit authority boundaries." +user-invocable: false +metadata: + internal: true +--- + +# Kubernetes fleet + +Load this skill only when the current firstmate is running in Kubernetes or is explicitly managing Kubernetes-backed crewmates. + +## Operating contract + +- Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. +- A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. +- Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. +- Use `status` to inspect it and `delete` only after unique work is checkpointed or delivered. +- Never mount the primary home into a child Pod. +- The demo child receives no Kubernetes ServiceAccount token by default. +- The OrbStack primary's cluster-admin binding is a local-demo trust decision, not a production-safe default. +- Pin an explicit host context with `AGENT_OS_CONTEXT`; the launcher refuses ambient host contexts. + +Use ordinary `kubectl exec`, Herdr, files, and Git to supervise the child. +Do not add a custom inter-agent chat protocol or Task/Run service. + diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..e1767260e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.github +.pi +.codex +.claude +.grok +.opencode +.no-mistakes +.env +config +data +projects +state +worktrees +.worktrees + diff --git a/AGENTS.md b/AGENTS.md index 27a13a5be..c231466ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -764,6 +764,7 @@ These skills are not captain-invocable; they are conditional operating reference - `fmx-respond` - load on an `x-mention ` `check:` wake to handle the mention, on an `x-mode-error ...` `check:` wake to report the X-mode configuration blocker, and on any milestone or terminal wake for an X-mode-linked task before posting its completion follow-up; relevant only when X mode is on. - `firstmate-codexapp` - load before coordinating a visible Codex Desktop thread, evaluating a Codex App backend request, or reconciling Codex Desktop host-tool smoke evidence for Firstmate work. - `firstmate-coding-guidelines` - load before changing firstmate's shared, tracked material, as defined by section 1's list, whether editing directly or briefing a crewmate for a firstmate-repo task. +- `kubernetes-fleet` - load before creating, supervising, recovering, or deleting Kubernetes-backed crewmates. ## 14. X mode diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..82cbec6f2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,54 @@ +FROM node:24-bookworm-slim + +ARG TARGETARCH +ARG HERDR_VERSION=0.7.3 +ARG KUBECTL_VERSION=1.34.8 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + curl \ + git \ + jq \ + openssh-client \ + procps \ + tmux \ + && rm -rf /var/lib/apt/lists/* + +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) asset=herdr-linux-x86_64; sha=043ef43ecbabda28465dcff1eec3184518150d567b8b8f20cda9c6c88770641d ;; \ + arm64) asset=herdr-linux-aarch64; sha=ea490094f2c7c39099870857d00c64c628ef7b5eba1967df4258033455ee2cb1 ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + curl -fsSL "https://github.com/ogulcancelik/herdr/releases/download/v${HERDR_VERSION}/${asset}" -o /usr/local/bin/herdr; \ + echo "$sha /usr/local/bin/herdr" | sha256sum -c -; \ + chmod 0755 /usr/local/bin/herdr; \ + mkdir -p /usr/share/licenses/herdr; \ + curl -fsSL "https://raw.githubusercontent.com/ogulcancelik/herdr/v${HERDR_VERSION}/LICENSE" -o /usr/share/licenses/herdr/LICENSE + +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) sha=f6249132865c13abe3c9dd5038f5da65849cb86eee1608c001831504e481aa8c ;; \ + arm64) sha=4c9fe1f717738950c638c38056130a8db5075e6413ae36d8687221a240cdf88b ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + curl -fsSL "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl; \ + echo "$sha /usr/local/bin/kubectl" | sha256sum -c -; \ + chmod 0755 /usr/local/bin/kubectl + +RUN npm install --global @earendil-works/pi-coding-agent@0.80.6 + +ENV FM_HOME=/home/agent \ + HOME=/home/agent \ + HERDR_SESSION=default + +RUN mkdir -p /home/agent /opt/agent-os \ + && chown -R node:node /home/agent /opt/agent-os + +COPY --chown=node:node . /opt/agent-os + +USER node +WORKDIR /opt/agent-os +ENTRYPOINT ["/opt/agent-os/bin/agent-os-container-entrypoint.sh"] diff --git a/README.md b/README.md index b33084744..83881cd91 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,11 @@ An agent distro is a portable directory of instructions, skills, tooling, polici There is no app to install: the cloned repo is the distro - `AGENTS.md`, bundled firstmate skills, and helper scripts that any terminal coding agent can follow. Launching a supported harness inside it instantiates your first mate - and makes you the captain. +This Agent OS fork also packages the distro for Kubernetes: one persistent +first mate can allocate isolated, persistent crewmate containers using ordinary +`kubectl`, while Herdr keeps every agent terminal visible. Start locally with +the [OrbStack Kubernetes demo](docs/kubernetes.md). + ## Features - **One liaison** - you talk only to the first mate; it dispatches, supervises, escalates only real decisions, and reports plain outcomes. @@ -182,6 +187,7 @@ Firstmate's skills live in two separate places with different audiences: ## Documentation +- [docs/kubernetes.md](docs/kubernetes.md) - run the Agent OS controller and isolated crewmates on local OrbStack Kubernetes. - [docs/architecture.md](docs/architecture.md) - how the crew, supervision, worktrees, secondmates, and project modes work. - [docs/configuration.md](docs/configuration.md) - environment variables, `FM_HOME`, runtime backend selection, optional X mode, the files you set, and harness support. - [docs/wedge-alarm.md](docs/wedge-alarm.md) - configure the active alert for a wedged away-mode escalation delivery. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 000000000..86fa4f8dc --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,9 @@ +# Third-party notices + +## Herdr + +The Agent OS demo image includes an unmodified Herdr 0.7.3 executable as a separate program. +Herdr is available under AGPL-3.0-or-later or a commercial license. +The image includes Herdr's license at `/usr/share/licenses/herdr/LICENSE`. +The exact corresponding source is available at . +Agent OS invokes Herdr through its documented CLI and socket interfaces. diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh new file mode 100755 index 000000000..57eaf2507 --- /dev/null +++ b/bin/agent-os-container-entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# agent-os-container-entrypoint.sh - seed a persistent home and run Herdr. +set -eu + +FM_HOME=${FM_HOME:-/home/agent} +export FM_HOME HOME="$FM_HOME" + +mkdir -p "$FM_HOME/config" "$FM_HOME/data" "$FM_HOME/projects" "$FM_HOME/state" +if [ ! -e "$FM_HOME/config/backend" ]; then + printf 'herdr\n' > "$FM_HOME/config/backend" +fi + +cd /opt/agent-os +exec herdr server + diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh new file mode 100755 index 000000000..32218f87c --- /dev/null +++ b/bin/agent-os-crewmate.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# agent-os-crewmate.sh - create, inspect, or delete one isolated crewmate Pod. +# Usage: bin/agent-os-crewmate.sh create|status|delete +set -eu + +COMMAND=${1:-} +ID=${2:-} +NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} +IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} +KUBECTL=${AGENT_OS_KUBECTL:-kubectl} + +case "$ID" in + ''|*[!a-z0-9-]*|-*|*-) echo "error: invalid crewmate id '$ID'" >&2; exit 2 ;; +esac + +KUBECTL_ARGS=() +if [ -n "${AGENT_OS_CONTEXT:-}" ]; then + KUBECTL_ARGS=(--context "$AGENT_OS_CONTEXT") +elif [ "${AGENT_OS_IN_CLUSTER:-0}" != 1 ] && [ ! -f /var/run/secrets/kubernetes.io/serviceaccount/token ]; then + echo "error: set AGENT_OS_CONTEXT outside Kubernetes; ambient contexts are refused" >&2 + exit 2 +fi + +POD="agent-os-crewmate-$ID" +PVC="$POD-home" + +case "$COMMAND" in + create) + "$KUBECTL" "${KUBECTL_ARGS[@]}" -n "$NAMESPACE" apply -f - <" >&2 + exit 2 + ;; +esac diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh new file mode 100755 index 000000000..03c1830dd --- /dev/null +++ b/bin/agent-os-local.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# agent-os-local.sh - build and operate the local OrbStack Agent OS demo. +# Usage: bin/agent-os-local.sh build|deploy|status|shell|attach|destroy [--yes] +set -eu + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +CONTEXT=${AGENT_OS_CONTEXT:-orbstack} +NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} +IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} +COMMAND=${1:-} + +if [ "$CONTEXT" != orbstack ] && [ "${AGENT_OS_ALLOW_NON_ORBSTACK:-0}" != 1 ]; then + echo "error: refusing Kubernetes context '$CONTEXT'; set AGENT_OS_ALLOW_NON_ORBSTACK=1 to opt in" >&2 + exit 2 +fi + +cd "$ROOT" + +case "$COMMAND" in + build) + docker build -t "$IMAGE" . + ;; + deploy) + if [ "$CONTEXT" = orbstack ]; then + orbctl start k8s + kubectl --context "$CONTEXT" wait --for=condition=Ready node/orbstack --timeout=120s + fi + kubectl --context "$CONTEXT" apply -k deploy/orbstack + ;; + status) + kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate + ;; + shell) + kubectl --context "$CONTEXT" -n "$NAMESPACE" exec -it statefulset/agent-os-firstmate -- bash + ;; + attach) + kubectl --context "$CONTEXT" -n "$NAMESPACE" exec -it statefulset/agent-os-firstmate -- herdr + ;; + destroy) + if [ "${2:-}" != --yes ]; then + echo "error: destroy requires --yes and deletes only namespace '$NAMESPACE'" >&2 + exit 2 + fi + kubectl --context "$CONTEXT" delete namespace "$NAMESPACE" + ;; + *) + echo "usage: $0 build|deploy|status|shell|attach|destroy [--yes]" >&2 + exit 2 + ;; +esac diff --git a/deploy/orbstack/kustomization.yaml b/deploy/orbstack/kustomization.yaml new file mode 100644 index 000000000..57f9eaf5f --- /dev/null +++ b/deploy/orbstack/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - rbac.yaml + - primary.yaml + diff --git a/deploy/orbstack/namespace.yaml b/deploy/orbstack/namespace.yaml new file mode 100644 index 000000000..8e919fbef --- /dev/null +++ b/deploy/orbstack/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: agent-os-demo + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/part-of: agent-os-demo + diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml new file mode 100644 index 000000000..7c67cbd6e --- /dev/null +++ b/deploy/orbstack/primary.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: agent-os-firstmate-home + namespace: agent-os-demo + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: firstmate +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: agent-os-firstmate + namespace: agent-os-demo + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: firstmate +spec: + serviceName: agent-os-firstmate + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: firstmate + template: + metadata: + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/component: firstmate + spec: + serviceAccountName: agent-os-firstmate + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: firstmate + image: agent-os:dev + imagePullPolicy: Never + env: + - name: FM_HOME + value: /home/agent + - name: HOME + value: /home/agent + readinessProbe: + exec: + command: + - herdr + - status + - --json + initialDelaySeconds: 2 + periodSeconds: 5 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "4" + memory: 8Gi + volumeMounts: + - name: home + mountPath: /home/agent + volumes: + - name: home + persistentVolumeClaim: + claimName: agent-os-firstmate-home + diff --git a/deploy/orbstack/rbac.yaml b/deploy/orbstack/rbac.yaml new file mode 100644 index 000000000..db3773b90 --- /dev/null +++ b/deploy/orbstack/rbac.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: agent-os-firstmate + namespace: agent-os-demo +--- +# Local demo only. Production installations require a reviewed narrower role. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: agent-os-firstmate-local-demo + labels: + app.kubernetes.io/name: agent-os + app.kubernetes.io/part-of: agent-os-demo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: agent-os-firstmate + namespace: agent-os-demo + diff --git a/docs/kubernetes.md b/docs/kubernetes.md new file mode 100644 index 000000000..45b0f088b --- /dev/null +++ b/docs/kubernetes.md @@ -0,0 +1,83 @@ +# Kubernetes Agent OS demo + +This demo runs Firstmate as the persistent controller of an isolated agent +cluster. Kubernetes distributes and isolates the crew; it does not replace +Firstmate's supervision model or Herdr's terminal/session interface. + +The local topology is deliberately small: + +- `agent-os-firstmate-0` is a StatefulSet with a 20 Gi persistent home. +- Herdr 0.7.3 is PID 1 and remains the visible session backend. +- the primary's local-demo ServiceAccount has `cluster-admin`, so it can shape + its own crew and workspace; +- every crewmate is an adaptable Agent OS container with its own 10 Gi PVC; +- crewmates do not receive Kubernetes service-account credentials by default. + +The broad primary grant is only for the isolated local agent cluster. It is not +a production-cluster access pattern. + +## Requirements + +- OrbStack with Kubernetes enabled +- Docker +- `kubectl` + +The helper refuses ambient Kubernetes contexts. It uses `orbstack` unless a +different context is deliberately enabled with `AGENT_OS_ALLOW_NON_ORBSTACK=1`. + +## Run the demo + +```sh +bin/agent-os-local.sh build +bin/agent-os-local.sh deploy +bin/agent-os-local.sh status +bin/agent-os-local.sh shell +``` + +Inside the primary container, authenticate the Pi harness using the provider +flow you choose. Host model credentials are intentionally excluded from the +image and are never copied automatically. The authenticated home persists on +the primary PVC. + +Start Pi from the tracked distro and attach to Herdr from another terminal: + +```sh +# inside the primary shell +cd /opt/agent-os +pi + +# on the host +bin/agent-os-local.sh attach +``` + +The primary can create and manage isolated crewmates directly: + +```sh +bin/agent-os-crewmate.sh create scout-1 +bin/agent-os-crewmate.sh status scout-1 +bin/agent-os-crewmate.sh delete scout-1 +``` + +When run on a host, the crewmate helper requires an explicit context: + +```sh +AGENT_OS_CONTEXT=orbstack bin/agent-os-crewmate.sh status scout-1 +``` + +Destroying the demo requires confirmation and deletes only the demo namespace: + +```sh +bin/agent-os-local.sh destroy --yes +``` + +## What the demo proves + +The primary and child homes survive Pod replacement. A child cannot use the +Kubernetes API through an automatically mounted token. The primary can create, +inspect, and delete child Pods and PVCs using ordinary `kubectl`; no custom +agent communication protocol or GitOps controller is required. + +This is the local substrate, not the final access model. Remote Agent OS +clusters should keep the intelligence cluster isolated and grant product or +production access deliberately per task, with Akua providing stable +infrastructure primitives and guardrails. diff --git a/docs/superpowers/plans/2026-07-12-orbstack-agent-os-demo.md b/docs/superpowers/plans/2026-07-12-orbstack-agent-os-demo.md new file mode 100644 index 000000000..75da12bad --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-orbstack-agent-os-demo.md @@ -0,0 +1,258 @@ +# OrbStack Agent OS Demo Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Package this Firstmate fork as a persistent Kubernetes-native Agent OS demo that runs on OrbStack and can launch one isolated crewmate Pod. + +**Architecture:** Keep Firstmate's supervision and Herdr backend unchanged. Add one OCI image, one namespace-scoped manifest set, and thin Bash CLIs that always target an explicit Kubernetes context. The trusted primary Pod gets cluster-admin only for the local demo; crewmate Pods get no service-account token and keep work on their own PVC. + +**Tech Stack:** Bash, Docker, Kubernetes YAML, Kustomize, OrbStack Kubernetes, Herdr 0.7.3, Node.js 24, Pi 0.80.6. + +## Global Constraints + +- Every host-side `kubectl` command must pass `--context`, defaulting to `orbstack`. +- The demo namespace is exactly `agent-os-demo`; scripts must not enumerate, mutate, or delete unrelated namespaces. +- No host model credentials, kubeconfigs, Git credentials, or home directories may enter the image build context. +- The primary home is a PVC mounted at `/home/agent`; each crewmate gets a different PVC. +- Herdr remains an unmodified separate AGPL executable and must ship with its license and exact source link. +- Firstmate remains usable outside Kubernetes; no existing backend contract is replaced. +- Tests use fake `docker`, `kubectl`, and `orbctl` commands and run before any live mutation. + +--- + +### Task 1: Guarded local bootstrap CLI + +**Files:** +- Create: `tests/agent-os-local.test.sh` +- Create: `bin/agent-os-local.sh` + +**Interfaces:** +- Consumes: `docker`, `kubectl`, and optional `orbctl` executables from `PATH`. +- Produces: `bin/agent-os-local.sh build|deploy|status|shell|attach|destroy` and environment variables `AGENT_OS_CONTEXT`, `AGENT_OS_NAMESPACE`, and `AGENT_OS_IMAGE`. + +- [ ] **Step 1: Write the failing CLI test** + +Create a fake-tool directory that appends every invocation to a log, run `status`, `deploy`, and `destroy`, and assert the exact safety contract: + +```bash +PATH="$fakebin:$PATH" AGENT_OS_TEST_LOG="$log" bin/agent-os-local.sh status +grep -F 'kubectl --context orbstack -n agent-os-demo get statefulset agent-os-firstmate' "$log" + +PATH="$fakebin:$PATH" AGENT_OS_TEST_LOG="$log" bin/agent-os-local.sh deploy +grep -F 'kubectl --context orbstack apply -k deploy/orbstack' "$log" + +if PATH="$fakebin:$PATH" bin/agent-os-local.sh destroy 2>/dev/null; then + fail "destroy must require --yes" +fi +``` + +Also assert that `AGENT_OS_CONTEXT=minekube-prod` is rejected unless `AGENT_OS_ALLOW_NON_ORBSTACK=1`, and that no command omits `--context`. + +- [ ] **Step 2: Run the test to verify RED** + +Run: `bash tests/agent-os-local.test.sh` + +Expected: FAIL because `bin/agent-os-local.sh` does not exist. + +- [ ] **Step 3: Implement the minimal guarded CLI** + +Implement strict Bash with these command mappings: + +```bash +build) docker build -t "$IMAGE" . ;; +deploy) kubectl --context "$CONTEXT" apply -k deploy/orbstack ;; +status) kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate ;; +shell) kubectl --context "$CONTEXT" -n "$NAMESPACE" exec -it statefulset/agent-os-firstmate -- bash ;; +attach) kubectl --context "$CONTEXT" -n "$NAMESPACE" exec -it statefulset/agent-os-firstmate -- herdr ;; +destroy) [ "${2:-}" = --yes ] || exit 2; kubectl --context "$CONTEXT" delete namespace "$NAMESPACE" ;; +``` + +`deploy` may call `orbctl start k8s` only when the context is `orbstack`; it must then wait with `kubectl --context orbstack wait --for=condition=Ready node/orbstack --timeout=120s`. + +- [ ] **Step 4: Run the test to verify GREEN** + +Run: `bash tests/agent-os-local.test.sh` + +Expected: all assertions print `ok` and exit 0. + +### Task 2: Reproducible container and persistent primary + +**Files:** +- Create: `tests/agent-os-container.test.sh` +- Create: `.dockerignore` +- Create: `Dockerfile` +- Create: `bin/agent-os-container-entrypoint.sh` +- Create: `THIRD_PARTY_NOTICES.md` + +**Interfaces:** +- Consumes: repository source as Docker build context. +- Produces: image `agent-os:dev`, `/opt/agent-os`, persistent `FM_HOME=/home/agent`, Herdr server as PID 1, Pi on `PATH`. + +- [ ] **Step 1: Write the failing static container test** + +Assert: + +```bash +grep -F 'FROM node:24-bookworm-slim' Dockerfile +grep -F 'HERDR_VERSION=0.7.3' Dockerfile +grep -F '@earendil-works/pi-coding-agent@0.80.6' Dockerfile +grep -F 'FM_HOME=/home/agent' Dockerfile +grep -F 'exec herdr server' bin/agent-os-container-entrypoint.sh +grep -F '.git' .dockerignore +grep -F '.pi' .dockerignore +grep -F 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' THIRD_PARTY_NOTICES.md +``` + +Also assert `bash -n bin/agent-os-container-entrypoint.sh` succeeds. + +- [ ] **Step 2: Run the test to verify RED** + +Run: `bash tests/agent-os-container.test.sh` + +Expected: FAIL on the first missing file. + +- [ ] **Step 3: Implement the image and entrypoint** + +Use `node:24-bookworm-slim`, install only `bash`, `ca-certificates`, `curl`, `git`, `jq`, `openssh-client`, `procps`, and `tmux`, and download the architecture-specific Herdr 0.7.3 release binary. +Install Pi at the exact package version. +Copy the repository to `/opt/agent-os`. +The entrypoint must create `$FM_HOME/config`, write `herdr` to `config/backend` only when absent, set `HOME=$FM_HOME`, and execute `herdr server` without copying any credentials. + +- [ ] **Step 4: Run static tests and build the image** + +Run: + +```bash +bash tests/agent-os-container.test.sh +docker build -t agent-os:dev . +docker run --rm --entrypoint bash agent-os:dev -lc 'herdr version && pi --version && test -x /opt/agent-os/bin/fm-spawn.sh' +``` + +Expected: static test exits 0, image builds, Herdr reports 0.7.3, Pi reports 0.80.6, and the Firstmate toolbelt exists. + +### Task 3: Kubernetes manifests and isolated crewmate launcher + +**Files:** +- Create: `tests/agent-os-kubernetes.test.sh` +- Create: `deploy/orbstack/kustomization.yaml` +- Create: `deploy/orbstack/namespace.yaml` +- Create: `deploy/orbstack/rbac.yaml` +- Create: `deploy/orbstack/primary.yaml` +- Create: `bin/agent-os-crewmate.sh` +- Create: `.agents/skills/kubernetes-fleet/SKILL.md` +- Modify: `AGENTS.md` + +**Interfaces:** +- Consumes: image `agent-os:dev`, explicit `kubectl` context on a host or in-cluster service-account configuration in the primary Pod. +- Produces: namespace `agent-os-demo`, StatefulSet `agent-os-firstmate`, PVC `agent-os-firstmate-home`, and `bin/agent-os-crewmate.sh create|status|delete `. + +- [ ] **Step 1: Write failing manifest and launcher tests** + +Assert rendered Kustomize output contains: + +```text +Namespace/agent-os-demo +ServiceAccount/agent-os-firstmate +ClusterRoleBinding/agent-os-firstmate-local-demo +PersistentVolumeClaim/agent-os-firstmate-home +StatefulSet/agent-os-firstmate +imagePullPolicy: Never +``` + +Use a fake `kubectl` to assert `agent-os-crewmate.sh create scout-1` applies exactly one PVC and one Pod in `agent-os-demo`, labels both with `agent-os.akua.dev/crewmate=scout-1`, sets `automountServiceAccountToken: false`, and refuses IDs outside `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`. + +- [ ] **Step 2: Run tests to verify RED** + +Run: `bash tests/agent-os-kubernetes.test.sh` + +Expected: FAIL because the manifests and launcher do not exist. + +- [ ] **Step 3: Implement manifests and launcher** + +The primary StatefulSet must mount the PVC at `/home/agent`, run as UID/GID 1000, request `500m` CPU and `1Gi` memory, limit at `4` CPU and `8Gi`, and use the dedicated ServiceAccount. +The local-demo ClusterRoleBinding may grant `cluster-admin` only to `system:serviceaccount:agent-os-demo:agent-os-firstmate` and must include a comment that production installations need a reviewed narrower role. +The crewmate launcher creates a separate PVC mounted at `/home/agent`, reuses `agent-os:dev`, and disables service-account token automount. + +- [ ] **Step 4: Add the conditional operating skill** + +Add a focused internal skill that tells Firstmate to use the launcher only when it is running in Kubernetes, keep children general-purpose, communicate parent-to-child through terminals/files, preserve unique work on the child PVC, and never claim the local-demo RBAC is production-safe. +Add exactly one trigger line to `AGENTS.md` section 13: load the skill before creating, supervising, recovering, or deleting Kubernetes crewmates. + +- [ ] **Step 5: Run tests to verify GREEN** + +Run: + +```bash +bash tests/agent-os-kubernetes.test.sh +for script in bin/agent-os-*.sh; do bash -n "$script"; done +kubectl kustomize deploy/orbstack >/tmp/agent-os-rendered.yaml +``` + +Expected: all tests exit 0 and rendering succeeds. + +### Task 4: Live OrbStack proof and concise documentation + +**Files:** +- Create: `docs/kubernetes.md` +- Modify: `README.md` + +**Interfaces:** +- Consumes: Tasks 1-3 and an active OrbStack Kubernetes context. +- Produces: reproducible local-demo commands and empirical proof of primary/child persistence. + +- [ ] **Step 1: Document only the verified local path** + +Document `build`, `deploy`, `status`, `shell`, `attach`, crewmate create/status/delete, and `destroy --yes`. +State that no credentials are copied automatically and that the user launches Pi from `/opt/agent-os` after authenticating inside the persistent home. +Add one concise README link; keep mechanism detail in `docs/kubernetes.md`. + +- [ ] **Step 2: Deploy to OrbStack** + +Run: + +```bash +bin/agent-os-local.sh build +bin/agent-os-local.sh deploy +kubectl --context orbstack -n agent-os-demo rollout status statefulset/agent-os-firstmate --timeout=180s +``` + +Expected: the StatefulSet has one Ready replica. + +- [ ] **Step 3: Prove Herdr, RBAC, and persistence** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- herdr status --json +kubectl --context orbstack -n agent-os-demo auth can-i create pods --as system:serviceaccount:agent-os-demo:agent-os-firstmate +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- sh -lc 'printf persisted > /home/agent/persistence-proof' +kubectl --context orbstack -n agent-os-demo delete pod agent-os-firstmate-0 +kubectl --context orbstack -n agent-os-demo wait --for=condition=Ready pod/agent-os-firstmate-0 --timeout=180s +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- grep -F persisted /home/agent/persistence-proof +``` + +Expected: Herdr server is compatible, RBAC says `yes`, and the proof survives Pod replacement. + +- [ ] **Step 4: Prove isolated crewmate lifecycle** + +Run the launcher inside the primary Pod to create `scout-1`, wait for readiness, confirm it has a different PVC and no automounted token, write a proof file, restart it, and confirm the file remains. +Delete only `scout-1` through the launcher and confirm the primary remains Ready. + +- [ ] **Step 5: Run final repository verification** + +Run: + +```bash +for script in bin/*.sh bin/backends/*.sh; do bash -n "$script"; done +bin/fm-lint.sh +bash tests/agent-os-local.test.sh +bash tests/agent-os-container.test.sh +bash tests/agent-os-kubernetes.test.sh +git diff --check +``` + +Expected: every command exits 0 with no warnings attributable to the new files. + +- [ ] **Step 6: Commit and ship through the repository gate** + +Commit the reviewed files on `feat/orbstack-demo`, initialize no-mistakes for the `akua-dev/agent-os` push target, run the configured pipeline without `--yes`, and stop at a CI-green PR for the captain's merge decision. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh new file mode 100755 index 000000000..6aaf81499 --- /dev/null +++ b/tests/agent-os-container.test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Static reproducibility and credential-boundary tests for the Agent OS image. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +assert_grep 'FROM node:24-bookworm-slim' "$ROOT/Dockerfile" "image must pin Node 24 Bookworm" +assert_grep 'ARG HERDR_VERSION=0.7.3' "$ROOT/Dockerfile" "image must pin Herdr 0.7.3" +assert_grep 'ARG KUBECTL_VERSION=1.34.8' "$ROOT/Dockerfile" "image must pin kubectl 1.34.8" +assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" +assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" +assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" +assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" +assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" +assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the build context" +assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" +assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "Herdr's exact corresponding source must be named" + +bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" +pass "container files pin dependencies and exclude host credentials" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh new file mode 100755 index 000000000..dfaac9040 --- /dev/null +++ b/tests/agent-os-kubernetes.test.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Kubernetes manifest and isolated crewmate launcher tests. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +LAUNCHER="$ROOT/bin/agent-os-crewmate.sh" +TMP=$(fm_test_tmproot agent-os-kubernetes) +FAKEBIN=$(fm_fakebin "$TMP") +CALLS="$TMP/calls.log" +STDIN_LOG="$TMP/stdin.yaml" + +rendered=$(kubectl kustomize "$ROOT/deploy/orbstack") || fail "OrbStack kustomization did not render" +assert_contains "$rendered" 'kind: Namespace' "render must include the namespace" +assert_contains "$rendered" 'name: agent-os-demo' "render must use the isolated namespace" +assert_contains "$rendered" 'kind: ServiceAccount' "render must include the primary ServiceAccount" +assert_contains "$rendered" 'kind: ClusterRoleBinding' "render must include the explicit local-demo grant" +assert_contains "$rendered" 'name: agent-os-firstmate-home' "render must include the persistent primary home" +assert_contains "$rendered" 'kind: StatefulSet' "render must include the primary StatefulSet" +assert_contains "$rendered" 'imagePullPolicy: Never' "OrbStack must use the locally built image" +assert_contains "$rendered" 'mountPath: /home/agent' "the primary home must mount at FM_HOME" +pass "OrbStack manifests render the isolated persistent primary" + +cat > "$FAKEBIN/kubectl" <<'SH' +#!/usr/bin/env bash +printf 'kubectl' >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +if [ "${*: -2}" = "-f -" ]; then + cat > "$AGENT_OS_STDIN_LOG" +fi +SH +chmod +x "$FAKEBIN/kubectl" + +run_launcher() { + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ + AGENT_OS_IN_CLUSTER=1 "$LAUNCHER" "$@" +} + +: > "$CALLS" +run_launcher create scout-1 +grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must apply only to agent-os-demo" +[ "$(grep -Fc 'kind: PersistentVolumeClaim' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one PVC" +[ "$(grep -Fc 'kind: Pod' "$STDIN_LOG")" -eq 1 ] || fail "create must emit one Pod" +assert_grep 'agent-os.akua.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" +assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" +assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" +pass "crewmate create emits one isolated Pod and PVC" + +: > "$CALLS" +run_launcher delete scout-1 +grep -Fqx 'kubectl -n agent-os-demo delete pod agent-os-crewmate-scout-1 --ignore-not-found' "$CALLS" || \ + fail "delete must target the crewmate Pod explicitly" +grep -Fqx 'kubectl -n agent-os-demo delete pvc agent-os-crewmate-scout-1-home --ignore-not-found' "$CALLS" || \ + fail "delete must target the crewmate PVC explicitly" +pass "crewmate delete removes the Pod and PVC explicitly" + +if run_launcher create 'Bad_ID' >/dev/null 2>&1; then + fail "invalid Kubernetes crewmate IDs must be rejected" +fi +pass "crewmate IDs are validated before kubectl" + +if PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ + "$LAUNCHER" status scout-1 >/dev/null 2>&1; then + fail "host execution without an explicit context must be rejected" +fi +pass "launcher refuses an ambient host Kubernetes context" + +: > "$CALLS" +PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$CALLS" AGENT_OS_STDIN_LOG="$STDIN_LOG" \ + AGENT_OS_CONTEXT=orbstack "$LAUNCHER" status scout-1 +grep -Fqx 'kubectl --context orbstack -n agent-os-demo get pod agent-os-crewmate-scout-1' "$CALLS" || \ + fail "host status must pin the selected context" +pass "host launcher calls require and pin an explicit context" diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh new file mode 100755 index 000000000..8c9ce018d --- /dev/null +++ b/tests/agent-os-local.test.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Safety and command-contract tests for the local OrbStack Agent OS CLI. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +CLI="$ROOT/bin/agent-os-local.sh" +TMP=$(fm_test_tmproot agent-os-local) +FAKEBIN=$(fm_fakebin "$TMP") +LOG="$TMP/calls.log" + +make_fake() { + local name=$1 + cat > "$FAKEBIN/$name" <<'SH' +#!/usr/bin/env bash +printf '%s' "$(basename "$0")" >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +SH + chmod +x "$FAKEBIN/$name" +} + +make_fake docker +make_fake kubectl +make_fake orbctl + +run_cli() { + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" "$CLI" "$@" +} + +assert_call() { + grep -Fqx -- "$1" "$LOG" || fail "$2 (missing exact call: $1)" +} + +test_status_pins_context_and_namespace() { + : > "$LOG" + run_cli status + assert_call 'kubectl --context orbstack -n agent-os-demo get statefulset agent-os-firstmate' \ + "status must target only the OrbStack Agent OS StatefulSet" + pass "status pins the OrbStack context and Agent OS namespace" +} + +test_deploy_starts_local_kubernetes_and_applies_kustomize() { + : > "$LOG" + run_cli deploy + assert_call 'orbctl start k8s' "deploy must start OrbStack Kubernetes" + assert_call 'kubectl --context orbstack wait --for=condition=Ready node/orbstack --timeout=120s' \ + "deploy must wait for the explicit OrbStack node" + assert_call 'kubectl --context orbstack apply -k deploy/orbstack' \ + "deploy must apply only the checked-in OrbStack kustomization" + pass "deploy starts and targets only OrbStack" +} + +test_build_uses_local_image_name() { + : > "$LOG" + run_cli build + assert_call 'docker build -t agent-os:dev .' "build must use the local demo image tag" + pass "build uses the deterministic local image tag" +} + +test_destroy_requires_exact_confirmation() { + local out rc=0 + : > "$LOG" + out=$(run_cli destroy 2>&1) || rc=$? + [ "$rc" -eq 2 ] || fail "destroy without --yes must exit 2, got $rc: $out" + [ ! -s "$LOG" ] || fail "destroy without --yes invoked an external command" + + run_cli destroy --yes + assert_call 'kubectl --context orbstack delete namespace agent-os-demo' \ + "confirmed destroy must delete only agent-os-demo" + pass "destroy requires confirmation and remains namespace-scoped" +} + +test_non_orbstack_context_is_fail_closed() { + local out rc=0 + : > "$LOG" + out=$(PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" AGENT_OS_CONTEXT=minekube-prod "$CLI" status 2>&1) || rc=$? + [ "$rc" -eq 2 ] || fail "non-OrbStack context must exit 2, got $rc: $out" + [ ! -s "$LOG" ] || fail "rejected non-OrbStack context invoked an external command" + + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" AGENT_OS_CONTEXT=kind-agent-os \ + AGENT_OS_ALLOW_NON_ORBSTACK=1 "$CLI" status + assert_call 'kubectl --context kind-agent-os -n agent-os-demo get statefulset agent-os-firstmate' \ + "explicit non-OrbStack opt-in must still pin the chosen context" + pass "non-OrbStack contexts require an explicit opt-in" +} + +test_status_pins_context_and_namespace +test_deploy_starts_local_kubernetes_and_applies_kustomize +test_build_uses_local_image_name +test_destroy_requires_exact_confirmation +test_non_orbstack_context_is_fail_closed From b2dba247f326222371fadfedb5f7c3e5e0b9cad7 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:13:57 +0200 Subject: [PATCH 02/56] docs: design persistent root agent toolchains --- ...root-userns-persistent-toolchain-design.md | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md diff --git a/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md b/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md new file mode 100644 index 000000000..f0a682e77 --- /dev/null +++ b/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md @@ -0,0 +1,109 @@ +# Root user-namespace and persistent toolchain design + +**Date:** 2026-07-12 + +## Purpose + +Agent OS Pods must be able to administer their own container environment without becoming root on the Kubernetes node. +The image must already contain the complete Firstmate toolchain so a fresh controller can begin useful work without an installation round. +Tools and authentication added by an agent at runtime must survive Pod replacement. + +## Decisions + +Every Agent OS Pod runs as UID 0 inside a Kubernetes Pod user namespace. +Every Pod sets `hostUsers: false`, so Kubernetes maps container UID 0 to a distinct unprivileged host UID range. +The deployment fails closed when the node, filesystem, container runtime, or cluster configuration cannot provide Pod user namespaces. +No Agent OS Pod uses privileged mode, a host user namespace, a host PID namespace, a host IPC namespace, a host network namespace, a raw block device, or a host-path mount. + +The image remains the reproducible baseline. +Each agent receives its own PVC-backed home and persistent `/usr/local` tree for runtime adaptation. +No agent mounts another agent's persistent state. + +## Image baseline + +The image includes the universal Firstmate toolchain and the Kubernetes demo runtime: + +- Bash, CA certificates, curl, Git, OpenSSH client, jq, tmux, procps, and rsync. +- GitHub CLI, ripgrep, and fd. +- Node.js, npm, and Pi. +- Herdr and kubectl. +- treehouse and no-mistakes. +- gh-axi, chrome-devtools-axi, lavish-axi, tasks-axi, and quota-axi. + +The Herdr backend does not require Orca. +Agent harnesses other than Pi remain optional runtime additions rather than baseline requirements. +Downloaded release binaries use exact versions and checksums where upstream publishes them. +Global npm packages use exact versions. +The build fails for an unsupported architecture or failed integrity check. + +## Persistent filesystem model + +Every agent PVC contains two independently used trees: + +- `/home/agent` stores Agent OS state, GitHub and model authentication, configuration, projects, package-manager state, and user-installed binaries. +- a PVC subdirectory mounted at `/usr/local` stores global npm packages and tools that installers place in `/usr/local`. + +An init container runs from the same image before the agent starts. +It copies the image's `/usr/local` baseline into the PVC-backed `/usr/local` without deleting agent-installed files. +Image-owned files overwrite older image-owned copies so a new image can update the baseline. +Agent-added paths that do not collide with the baseline remain intact. + +The runtime environment puts `/home/agent/.local/bin`, `/home/agent/.bun/bin`, `/home/agent/.cargo/bin`, and `/usr/local/bin` before system paths. +It sets persistent XDG directories below `/home/agent` and configures npm's global prefix to `/usr/local`. +Installers that honor HOME, XDG paths, npm prefix, Bun home, Cargo home, or `/usr/local` therefore survive Pod replacement. + +`apt` remains available because the process is root inside the container. +Packages installed into `/usr`, `/etc`, or `/var` with `apt` are intentionally ephemeral and disappear on Pod replacement. +Firstmate's required tools do not depend on runtime `apt` because they are part of the image baseline. +An agent that needs an additional durable CLI should prefer `/usr/local` or a persistent home prefix. + +## Pod security and authority + +The primary and crewmate containers set `runAsUser: 0` and `runAsGroup: 0` inside their Pod user namespace. +The init container uses the same user namespace and container-root identity to seed the persistent tool tree. +The Pods use the runtime's normal namespaced capabilities and do not request privileged mode. + +The Firstmate ServiceAccount retains cluster-admin only for the isolated local Agent OS cluster. +Crewmates continue to receive no Kubernetes ServiceAccount token by default. +Container root and Kubernetes API authority remain separate controls. + +Kubernetes v1.34 marks Pod user namespaces beta. +The target node must use a compatible Linux kernel, idmapped-mount filesystem, containerd 2.0 or later or another supported CRI runtime, and a compatible OCI runtime. +The local demo must verify the actual UID map rather than infer support from the Kubernetes version. + +## Startup flow + +1. Kubernetes creates or reattaches the agent's PVC. +2. The init container ensures the persistent home and `/usr/local` directories exist. +3. The init container refreshes image-owned `/usr/local` files while preserving runtime additions. +4. The main container starts as container root with the persistent environment configured. +5. Herdr starts as PID 1. +6. Firstmate bootstrap finds its required tools and reports only missing authentication or genuinely optional additions. + +## Failure behavior + +The demo must refuse to claim readiness when Pod user namespaces are unavailable. +The verification path checks `/proc/self/uid_map` and proves that container UID 0 maps to a nonzero host UID range. +The image build stops on missing releases, unsupported architectures, checksum failures, or missing baseline commands. +The init container stops startup if it cannot seed the persistent tool tree. +No fallback silently removes `hostUsers: false` or grants privileged mode. + +## Verification + +Static tests assert that both primary and generated crewmate Pod specs use `hostUsers: false` and container UID 0. +Container tests assert every baseline command is present and Firstmate bootstrap emits no `MISSING:` diagnostics. +Runtime tests prove the Pod UID map is remapped, root can write to normal container paths, and durable installs can write to both persistent prefixes. +Persistence tests write one tool below `/home/agent/.local/bin` and one below `/usr/local/bin`, replace the Pod, and execute both tools afterward. +Isolation tests prove crewmates have distinct PVCs and no ServiceAccount token. +Authentication verification checks that GitHub CLI configuration survives Pod replacement without embedding credentials in the image. + +## Out of scope + +This change does not persist the entire mutable container root filesystem. +It does not add privileged Pods, host mounts, a custom package service, GitOps, or a new inter-agent protocol. +It does not define production-cluster RBAC beyond retaining the existing local-demo boundary. + +## Sources + +- Kubernetes v1.34 user-namespace documentation: . +- Firstmate's universal toolchain contract: `docs/configuration.md` under `Toolchain` and `bin/fm-bootstrap.sh`. From 5519cb7fe4284da421fae923adec625eabea1ae3 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:18:00 +0200 Subject: [PATCH 03/56] docs: plan persistent root agent toolchains --- ...-07-12-root-userns-persistent-toolchain.md | 476 ++++++++++++++++++ 1 file changed, 476 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md diff --git a/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md b/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md new file mode 100644 index 000000000..b665ae958 --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md @@ -0,0 +1,476 @@ +# Root User Namespace and Persistent Toolchain Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Run every Agent OS Pod as container root mapped to an unprivileged host UID while providing Firstmate's complete baseline toolchain and preserving runtime-installed tools across Pod replacement. + +**Architecture:** Extend the reproducible image with exact binary and npm package versions, then seed the image's `/usr/local` tree into each agent PVC from an init container. The main container mounts that tree at `/usr/local`, uses the same PVC for `/home/agent`, and runs as UID 0 inside a Pod user namespace selected by `hostUsers: false`. + +**Tech Stack:** Docker, Bash, Kubernetes v1.34, Kustomize, StatefulSet, Pods, PVCs, Linux user namespaces, Herdr, Pi, Node.js, npm. + +## Global Constraints + +- Every primary and crewmate Pod sets `hostUsers: false`, `runAsUser: 0`, and `runAsGroup: 0`. +- The primary retains the existing local-demo `cluster-admin` ServiceAccount binding, while crewmates receive no Kubernetes credentials by default. +- No Pod uses privileged mode, host PID, host IPC, host networking, raw block devices, or host-path mounts. +- The deployment fails closed when the Pod UID map does not map container UID 0 to a nonzero host UID. +- Each agent has its own PVC-backed `/home/agent` and `/usr/local`; no persistent state is shared between agents. +- Runtime additions in `/usr/local` and persistent home prefixes survive Pod replacement. +- Runtime `apt` changes outside the persistent prefixes remain intentionally ephemeral. +- Host model and GitHub credentials never enter the image or Docker build context. +- Host Kubernetes commands always pin `--context orbstack`. +- Use TDD for every behavior change and run `bin/fm-lint.sh` before shipping. +- Do not run no-mistakes unless the captain asks for it again. + +--- + +### Task 1: Complete the reproducible image toolchain + +**Files:** +- Modify: `Dockerfile` +- Modify: `tests/agent-os-container.test.sh` + +**Interfaces:** +- Consumes: Docker BuildKit `TARGETARCH` values `amd64` and `arm64`. +- Produces: image commands `gh`, `rg`, `fd`, `treehouse`, `no-mistakes`, `gh-axi`, `chrome-devtools-axi`, `lavish-axi`, `tasks-axi`, and `quota-axi`. + +- [ ] **Step 1: Add failing static pin assertions** + +Add these assertions to `tests/agent-os-container.test.sh`: + +```bash +assert_grep 'ARG GH_VERSION=2.96.0' "$ROOT/Dockerfile" "image must pin GitHub CLI 2.96.0" +assert_grep 'ARG TREEHOUSE_VERSION=2.0.0' "$ROOT/Dockerfile" "image must pin treehouse 2.0.0" +assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin no-mistakes 1.34.0" +assert_grep 'gh-axi@0.1.27' "$ROOT/Dockerfile" "image must pin gh-axi 0.1.27" +assert_grep 'chrome-devtools-axi@0.1.26' "$ROOT/Dockerfile" "image must pin chrome-devtools-axi 0.1.26" +assert_grep 'lavish-axi@0.1.40' "$ROOT/Dockerfile" "image must pin lavish-axi 0.1.40" +assert_grep 'tasks-axi@0.2.2' "$ROOT/Dockerfile" "image must pin tasks-axi 0.2.2" +assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5" +assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" +assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" +``` + +- [ ] **Step 2: Run the test and observe the missing pins** + +Run: `bash tests/agent-os-container.test.sh` + +Expected: FAIL at `image must pin GitHub CLI 2.96.0`. + +- [ ] **Step 3: Install the Debian and exact npm baseline** + +Add `fd-find`, `ripgrep`, and `rsync` to the existing apt package list. +Create `/usr/local/bin/fd` as a symlink to `/usr/bin/fdfind`. +Replace the existing Pi-only npm install with this exact install: + +```dockerfile +RUN npm install --global \ + @earendil-works/pi-coding-agent@0.80.6 \ + gh-axi@0.1.27 \ + chrome-devtools-axi@0.1.26 \ + lavish-axi@0.1.40 \ + tasks-axi@0.2.2 \ + quota-axi@0.1.5 +``` + +- [ ] **Step 4: Install checksum-verified GitHub CLI** + +Add `ARG GH_VERSION=2.96.0` and a `RUN` block that downloads `gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz`, verifies the architecture checksum, and copies `bin/gh` into `/usr/local/bin`. +Use these checksums: + +```text +amd64 83d5c2ccad5498f58bf6368acb1ab32588cf43ab3a4b1c301bf36328b1c8bd60 +arm64 06f86ec7103d41993b76cd78072f43595c34aaa56506d971d9860e67140bf909 +``` + +- [ ] **Step 5: Install checksum-verified treehouse and no-mistakes** + +Add `ARG TREEHOUSE_VERSION=2.0.0` and `ARG NO_MISTAKES_VERSION=1.34.0`. +Download the matching Linux tarball for `TARGETARCH`, verify it, and install its binary into `/usr/local/bin`. +Use these checksums: + +```text +treehouse amd64 b7926c19633ee94582b7f1b58369f22b304ae7228a47253c2148e3a8176f03b0 +treehouse arm64 91bca451bab84df685ee17975c8a9d8cf671b3e95c96b7fc6ff0121ea0aae991 +no-mistakes amd64 449d0276e1b35369ea332dae0eddb5be326c2d4fc9643270af98858cf3906536 +no-mistakes arm64 f157df3e18350edea8abdaa065681bd115a9d321fca86f51e9a0184b3a9d8756 +``` + +- [ ] **Step 6: Run static tests** + +Run: `bash tests/agent-os-container.test.sh` + +Expected: PASS with `container files pin dependencies and exclude host credentials`. + +- [ ] **Step 7: Build and verify every baseline command** + +Run: + +```bash +bin/agent-os-local.sh build +docker run --rm --entrypoint bash agent-os:dev -lc ' + for tool in bash curl git ssh jq tmux ps rsync gh rg fd node npm pi herdr kubectl treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi; do + command -v "$tool" || exit 1 + done + FM_HOME=/tmp/fm HOME=/tmp/fm mkdir -p /tmp/fm/config /tmp/fm/data /tmp/fm/state + FM_HOME=/tmp/fm HOME=/tmp/fm FM_BOOTSTRAP_DETECT_ONLY=1 /opt/agent-os/bin/fm-bootstrap.sh | grep "^MISSING:" && exit 1 || true +' +``` + +Expected: every command resolves and bootstrap emits no `MISSING:` line. + +- [ ] **Step 8: Commit the image baseline** + +```bash +git add Dockerfile tests/agent-os-container.test.sh +git commit -m "feat: bundle the Firstmate toolchain" +``` + +--- + +### Task 2: Add the user-namespace and persistent-prefix initializer + +**Files:** +- Create: `bin/agent-os-init.sh` +- Create: `tests/agent-os-init.test.sh` +- Modify: `Dockerfile` +- Modify: `bin/agent-os-container-entrypoint.sh` + +**Interfaces:** +- Consumes: `AGENT_OS_UID_MAP_PATH`, defaulting to `/proc/self/uid_map`; `AGENT_OS_PERSISTENT_ROOT`, defaulting to `/persistent-agent`. +- Produces: a PVC root used as `/home/agent` and a seeded `$AGENT_OS_PERSISTENT_ROOT/usr-local`; exit zero only for a remapped UID 0 range. + +- [ ] **Step 1: Write failing initializer tests** + +Create `tests/agent-os-init.test.sh` with fixtures for a mapped and host UID map: + +```bash +#!/usr/bin/env bash +set -u +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP=$(fm_test_tmproot agent-os-init) +mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" +printf '0 100000 65536\n' > "$TMP/mapped" +printf '0 0 4294967295\n' > "$TMP/host" +printf 'baseline\n' > "$TMP/source/bin/baseline" +printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" + +AGENT_OS_UID_MAP_PATH="$TMP/mapped" \ +AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ +AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ + "$ROOT/bin/agent-os-init.sh" + +assert_grep baseline "$TMP/persistent/usr-local/bin/baseline" "initializer must seed image tools" +assert_grep runtime "$TMP/persistent/usr-local/bin/runtime-added" "initializer must preserve runtime tools" + +if AGENT_OS_UID_MAP_PATH="$TMP/host" \ + AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ + AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ + "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then + fail "initializer must reject the host user namespace" +fi +pass "initializer verifies remapping and preserves persistent tools" +``` + +- [ ] **Step 2: Run the test and observe the missing initializer** + +Run: `bash tests/agent-os-init.test.sh` + +Expected: FAIL because `bin/agent-os-init.sh` does not exist. + +- [ ] **Step 3: Implement the initializer** + +Create `bin/agent-os-init.sh` as a strict Bash script. +Read three integers from the first UID-map row. +Require inside UID `0`, outside UID greater than `0`, and range at least `65536`. +Create persistent `.config`, `.cache`, `.local/bin`, `.local/share`, `.bun`, `.cargo`, and `usr-local` directories below the PVC root. +Copy the image baseline with `rsync -a "$IMAGE_USR_LOCAL/" "$PERSISTENT_ROOT/usr-local/"` and do not pass `--delete`. + +- [ ] **Step 4: Configure persistent package-manager paths** + +In `Dockerfile`, set: + +```dockerfile +ENV FM_HOME=/home/agent \ + HOME=/home/agent \ + XDG_CONFIG_HOME=/home/agent/.config \ + XDG_DATA_HOME=/home/agent/.local/share \ + XDG_CACHE_HOME=/home/agent/.cache \ + NPM_CONFIG_PREFIX=/usr/local \ + BUN_INSTALL=/home/agent/.bun \ + CARGO_HOME=/home/agent/.cargo \ + PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin \ + HERDR_SESSION=default +``` + +Remove `USER node` and create `/opt/image-usr-local` as a snapshot of the built `/usr/local` tree before runtime mounts hide it. + +- [ ] **Step 5: Make runtime setup persistent and idempotent** + +Update `bin/agent-os-container-entrypoint.sh` to create persistent XDG, local-bin, Bun, and Cargo directories. +After the Herdr backend configuration, install the three AXI SessionStart hooks into persistent HOME only when their marker is absent: + +```bash +for tool in gh-axi chrome-devtools-axi lavish-axi; do + marker="$HOME/.config/agent-os/setup-$tool" + if [ ! -e "$marker" ]; then + "$tool" setup hooks + mkdir -p "$(dirname "$marker")" + : > "$marker" + fi +done +``` + +- [ ] **Step 6: Run initializer and container tests** + +Run: + +```bash +bash tests/agent-os-init.test.sh +bash tests/agent-os-container.test.sh +bash -n bin/agent-os-init.sh bin/agent-os-container-entrypoint.sh +``` + +Expected: all tests pass and Bash reports no syntax errors. + +- [ ] **Step 7: Commit the persistent initializer** + +```bash +git add Dockerfile bin/agent-os-init.sh bin/agent-os-container-entrypoint.sh tests/agent-os-init.test.sh +git commit -m "feat: persist agent-installed tools" +``` + +--- + +### Task 3: Run primary and crewmates as remapped container root + +**Files:** +- Modify: `deploy/orbstack/primary.yaml` +- Modify: `bin/agent-os-crewmate.sh` +- Modify: `tests/agent-os-kubernetes.test.sh` + +**Interfaces:** +- Consumes: image path `/opt/image-usr-local` and per-agent PVC. +- Produces: Pod specs with `hostUsers: false`, root security context, init container `agent-os-init`, whole-PVC home mount, and `usr-local` subPath mount. + +- [ ] **Step 1: Add failing manifest assertions** + +Extend `tests/agent-os-kubernetes.test.sh` for both rendered primary YAML and captured child YAML: + +```bash +assert_contains "$rendered" 'hostUsers: false' "primary must use a Pod user namespace" +assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" +assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" +assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" +assert_grep 'hostUsers: false' "$STDIN_LOG" "children must use Pod user namespaces" +assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" +assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" +assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" +``` + +Retain the existing assertion for `automountServiceAccountToken: false`. + +- [ ] **Step 2: Run the Kubernetes test and observe failure** + +Run: `bash tests/agent-os-kubernetes.test.sh` + +Expected: FAIL at `primary must use a Pod user namespace`. + +- [ ] **Step 3: Update the primary StatefulSet** + +Set `spec.template.spec.hostUsers: false`. +Replace the non-root security context with `runAsUser: 0` and `runAsGroup: 0`. +Add an `agent-os-init` init container using `agent-os:dev`, command `/opt/agent-os/bin/agent-os-init.sh`, and a whole-PVC mount at `/persistent-agent`. +Mount the PVC root at `/home/agent` and PVC subPath `usr-local` at `/usr/local` in the main container. +Do not set `privileged: true` or any host namespace option. + +- [ ] **Step 4: Update generated crewmate Pods** + +Make `bin/agent-os-crewmate.sh create` emit the same user-namespace, root, init-container, home, and `/usr/local` structure. +Keep the distinct child PVC and `automountServiceAccountToken: false` contract unchanged. + +- [ ] **Step 5: Run static Kubernetes tests** + +Run: + +```bash +bash tests/agent-os-kubernetes.test.sh +kubectl kustomize deploy/orbstack >/dev/null +bash -n bin/agent-os-crewmate.sh +``` + +Expected: all checks pass. + +- [ ] **Step 6: Commit the Pod model** + +```bash +git add deploy/orbstack/primary.yaml bin/agent-os-crewmate.sh tests/agent-os-kubernetes.test.sh +git commit -m "feat: isolate root agents with user namespaces" +``` + +--- + +### Task 4: Prove the live user namespace and persistent tool lifecycle + +**Files:** +- Modify if evidence reveals a defect: files owned by Tasks 1 through 3 and their colocated tests. + +**Interfaces:** +- Consumes: OrbStack context, `agent-os:dev`, namespace `agent-os-demo`. +- Produces: empirical proof for remapped root, baseline completeness, and durable runtime tools. + +- [ ] **Step 1: Rebuild and deploy** + +Run: + +```bash +bin/agent-os-local.sh build +bin/agent-os-local.sh deploy +kubectl --context orbstack -n agent-os-demo rollout status statefulset/agent-os-firstmate --timeout=180s +``` + +Expected: the StatefulSet reaches `1/1` Ready. + +- [ ] **Step 2: Verify remapped container root** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- id +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- cat /proc/self/uid_map +``` + +Expected: `id` reports UID 0 and the UID map begins with container UID `0`, a nonzero host UID, and a range of at least `65536`. + +- [ ] **Step 3: Verify the complete baseline** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- bash -lc ' + for tool in gh rg fd treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi; do command -v "$tool" || exit 1; done + FM_BOOTSTRAP_DETECT_ONLY=1 /opt/agent-os/bin/fm-bootstrap.sh | tee /tmp/bootstrap.out + ! grep -q "^MISSING:" /tmp/bootstrap.out +' +``` + +Expected: all tools resolve and bootstrap reports `NEEDS_GH_AUTH` but no missing tool. + +- [ ] **Step 4: Add persistent runtime tools** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- bash -lc ' + printf "#!/bin/sh\necho home-persisted\n" > /home/agent/.local/bin/home-proof + printf "#!/bin/sh\necho usr-local-persisted\n" > /usr/local/bin/usr-local-proof + chmod +x /home/agent/.local/bin/home-proof /usr/local/bin/usr-local-proof +' +``` + +Expected: both writes succeed as container root. + +- [ ] **Step 5: Replace the primary Pod and prove persistence** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo delete pod agent-os-firstmate-0 --wait=true +kubectl --context orbstack -n agent-os-demo rollout status statefulset/agent-os-firstmate --timeout=180s +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- home-proof +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- usr-local-proof +``` + +Expected: output is `home-persisted` and `usr-local-persisted`. + +- [ ] **Step 6: Prove a root crewmate remains Kubernetes-unprivileged** + +Run: + +```bash +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- /opt/agent-os/bin/agent-os-crewmate.sh create root-proof +kubectl --context orbstack -n agent-os-demo wait --for=condition=Ready pod/agent-os-crewmate-root-proof --timeout=180s +kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- id +kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- cat /proc/self/uid_map +kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token +kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- /opt/agent-os/bin/agent-os-crewmate.sh delete root-proof +``` + +Expected: child UID 0 is remapped, no ServiceAccount token exists, and cleanup removes its Pod and PVC. + +- [ ] **Step 7: Convert any live defect into a failing test before fixing it** + +For each failure, add the smallest regression assertion to the owning test, run it to observe failure, implement the fix, and rerun Tasks 1 through 3 tests. + +- [ ] **Step 8: Commit live-proof fixes if needed** + +```bash +git add Dockerfile bin deploy tests +git diff --cached --quiet || git commit -m "fix: complete the persistent root agent runtime" +``` + +--- + +### Task 5: Document, validate, and update the review branch + +**Files:** +- Modify: `docs/kubernetes.md` +- Modify: `README.md` only if the current Kubernetes summary becomes inaccurate. + +**Interfaces:** +- Consumes: verified runtime behavior from Task 4. +- Produces: operator guidance for root isolation, persistent installs, authentication, and known ephemeral paths. + +- [ ] **Step 1: Update the Kubernetes guide** + +Document these verified facts in `docs/kubernetes.md`, one sentence per physical line: + +```text +Agents run as UID 0 inside a Pod user namespace and are mapped to unprivileged node UIDs. +The complete Firstmate baseline is part of the image. +Tools installed below /home/agent persistent prefixes or /usr/local survive Pod replacement. +apt remains available, but files it writes outside persistent prefixes are ephemeral. +GitHub and model authentication live on the individual agent PVC. +The deployment fails instead of falling back when user namespaces are unsupported. +``` + +- [ ] **Step 2: Run the complete focused verification set** + +Run: + +```bash +bash tests/agent-os-local.test.sh +bash tests/agent-os-container.test.sh +bash tests/agent-os-init.test.sh +bash tests/agent-os-kubernetes.test.sh +for script in bin/agent-os-*.sh; do bash -n "$script"; done +git diff --check +bin/fm-lint.sh +``` + +Expected: all tests pass, `git diff --check` is silent, and `fm-lint.sh` exits zero. + +- [ ] **Step 3: Commit documentation** + +```bash +git add docs/kubernetes.md README.md +git diff --cached --quiet || git commit -m "docs: explain persistent root agents" +``` + +- [ ] **Step 4: Verify clean committed state** + +Run: + +```bash +git status --short --branch +git log --oneline origin/feat/orbstack-demo..HEAD +``` + +Expected: no uncommitted files and only the approved spec, plan, and implementation commits are ahead of the remote branch. + +- [ ] **Step 5: Push the existing review branch** + +Run: `git push origin feat/orbstack-demo` + +Expected: GitHub updates pull request . From 868903a546a44e7b834c0cb567690eb037d7d94d Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:20:32 +0200 Subject: [PATCH 04/56] feat: bundle the Firstmate toolchain --- Dockerfile | 45 +++++++++++++++++++++++++++++++- tests/agent-os-container.test.sh | 10 +++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 82cbec6f2..264ebd9f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,17 +3,24 @@ FROM node:24-bookworm-slim ARG TARGETARCH ARG HERDR_VERSION=0.7.3 ARG KUBECTL_VERSION=1.34.8 +ARG GH_VERSION=2.96.0 +ARG TREEHOUSE_VERSION=2.0.0 +ARG NO_MISTAKES_VERSION=1.34.0 RUN apt-get update \ && apt-get install -y --no-install-recommends \ bash \ ca-certificates \ curl \ + fd-find \ git \ jq \ openssh-client \ procps \ + ripgrep \ + rsync \ tmux \ + && ln -s /usr/bin/fdfind /usr/local/bin/fd \ && rm -rf /var/lib/apt/lists/* RUN set -eu; \ @@ -38,7 +45,43 @@ RUN set -eu; \ echo "$sha /usr/local/bin/kubectl" | sha256sum -c -; \ chmod 0755 /usr/local/bin/kubectl -RUN npm install --global @earendil-works/pi-coding-agent@0.80.6 +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) sha=83d5c2ccad5498f58bf6368acb1ab32588cf43ab3a4b1c301bf36328b1c8bd60 ;; \ + arm64) sha=06f86ec7103d41993b76cd78072f43595c34aaa56506d971d9860e67140bf909 ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + asset="gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz"; \ + curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/${asset}" -o "/tmp/${asset}"; \ + echo "$sha /tmp/${asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${asset}" -C /tmp; \ + install -m 0755 "/tmp/gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh" /usr/local/bin/gh; \ + rm -rf "/tmp/${asset}" "/tmp/gh_${GH_VERSION}_linux_${TARGETARCH}" + +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) treehouse_sha=b7926c19633ee94582b7f1b58369f22b304ae7228a47253c2148e3a8176f03b0; no_mistakes_sha=449d0276e1b35369ea332dae0eddb5be326c2d4fc9643270af98858cf3906536 ;; \ + arm64) treehouse_sha=91bca451bab84df685ee17975c8a9d8cf671b3e95c96b7fc6ff0121ea0aae991; no_mistakes_sha=f157df3e18350edea8abdaa065681bd115a9d321fca86f51e9a0184b3a9d8756 ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + treehouse_asset="treehouse-v${TREEHOUSE_VERSION}-linux-${TARGETARCH}.tar.gz"; \ + curl -fsSL "https://github.com/kunchenguid/treehouse/releases/download/v${TREEHOUSE_VERSION}/${treehouse_asset}" -o "/tmp/${treehouse_asset}"; \ + echo "$treehouse_sha /tmp/${treehouse_asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${treehouse_asset}" -C /usr/local/bin; \ + no_mistakes_asset="no-mistakes-v${NO_MISTAKES_VERSION}-linux-${TARGETARCH}.tar.gz"; \ + curl -fsSL "https://github.com/kunchenguid/no-mistakes/releases/download/v${NO_MISTAKES_VERSION}/${no_mistakes_asset}" -o "/tmp/${no_mistakes_asset}"; \ + echo "$no_mistakes_sha /tmp/${no_mistakes_asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${no_mistakes_asset}" -C /usr/local/bin; \ + chmod 0755 /usr/local/bin/treehouse /usr/local/bin/no-mistakes; \ + rm -f "/tmp/${treehouse_asset}" "/tmp/${no_mistakes_asset}" + +RUN npm install --global \ + @earendil-works/pi-coding-agent@0.80.6 \ + gh-axi@0.1.27 \ + chrome-devtools-axi@0.1.26 \ + lavish-axi@0.1.40 \ + tasks-axi@0.2.2 \ + quota-axi@0.1.5 ENV FM_HOME=/home/agent \ HOME=/home/agent \ diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 6aaf81499..01a3bfdb1 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -8,8 +8,18 @@ set -u assert_grep 'FROM node:24-bookworm-slim' "$ROOT/Dockerfile" "image must pin Node 24 Bookworm" assert_grep 'ARG HERDR_VERSION=0.7.3' "$ROOT/Dockerfile" "image must pin Herdr 0.7.3" assert_grep 'ARG KUBECTL_VERSION=1.34.8' "$ROOT/Dockerfile" "image must pin kubectl 1.34.8" +assert_grep 'ARG GH_VERSION=2.96.0' "$ROOT/Dockerfile" "image must pin GitHub CLI 2.96.0" +assert_grep 'ARG TREEHOUSE_VERSION=2.0.0' "$ROOT/Dockerfile" "image must pin treehouse 2.0.0" +assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin no-mistakes 1.34.0" assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" +assert_grep 'gh-axi@0.1.27' "$ROOT/Dockerfile" "image must pin gh-axi 0.1.27" +assert_grep 'chrome-devtools-axi@0.1.26' "$ROOT/Dockerfile" "image must pin chrome-devtools-axi 0.1.26" +assert_grep 'lavish-axi@0.1.40' "$ROOT/Dockerfile" "image must pin lavish-axi 0.1.40" +assert_grep 'tasks-axi@0.2.2' "$ROOT/Dockerfile" "image must pin tasks-axi 0.2.2" +assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5" +assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" +assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" From ea04e9321d03f439f6380814175353e4d7ed6a1b Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:22:30 +0200 Subject: [PATCH 05/56] feat: persist agent-installed tools --- Dockerfile | 14 +++++++--- bin/agent-os-container-entrypoint.sh | 21 +++++++++++++-- bin/agent-os-init.sh | 38 ++++++++++++++++++++++++++++ tests/agent-os-container.test.sh | 7 +++++ tests/agent-os-init.test.sh | 34 +++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 6 deletions(-) create mode 100755 bin/agent-os-init.sh create mode 100755 tests/agent-os-init.test.sh diff --git a/Dockerfile b/Dockerfile index 264ebd9f5..0ae8012fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,13 +85,19 @@ RUN npm install --global \ ENV FM_HOME=/home/agent \ HOME=/home/agent \ + XDG_CONFIG_HOME=/home/agent/.config \ + XDG_DATA_HOME=/home/agent/.local/share \ + XDG_CACHE_HOME=/home/agent/.cache \ + NPM_CONFIG_PREFIX=/usr/local \ + BUN_INSTALL=/home/agent/.bun \ + CARGO_HOME=/home/agent/.cargo \ + PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin \ HERDR_SESSION=default -RUN mkdir -p /home/agent /opt/agent-os \ - && chown -R node:node /home/agent /opt/agent-os +RUN mkdir -p /home/agent /opt/agent-os /opt/image-usr-local \ + && cp -a /usr/local/. /opt/image-usr-local/ -COPY --chown=node:node . /opt/agent-os +COPY . /opt/agent-os -USER node WORKDIR /opt/agent-os ENTRYPOINT ["/opt/agent-os/bin/agent-os-container-entrypoint.sh"] diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 57eaf2507..0cf4d203e 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -5,11 +5,28 @@ set -eu FM_HOME=${FM_HOME:-/home/agent} export FM_HOME HOME="$FM_HOME" -mkdir -p "$FM_HOME/config" "$FM_HOME/data" "$FM_HOME/projects" "$FM_HOME/state" +mkdir -p \ + "$FM_HOME/config" \ + "$FM_HOME/data" \ + "$FM_HOME/projects" \ + "$FM_HOME/state" \ + "$HOME/.config/agent-os" \ + "$HOME/.cache" \ + "$HOME/.local/bin" \ + "$HOME/.local/share" \ + "$HOME/.bun" \ + "$HOME/.cargo" if [ ! -e "$FM_HOME/config/backend" ]; then printf 'herdr\n' > "$FM_HOME/config/backend" fi +for tool in gh-axi chrome-devtools-axi lavish-axi; do + marker="$HOME/.config/agent-os/setup-$tool" + if [ ! -e "$marker" ]; then + "$tool" setup hooks + : > "$marker" + fi +done + cd /opt/agent-os exec herdr server - diff --git a/bin/agent-os-init.sh b/bin/agent-os-init.sh new file mode 100755 index 000000000..e270d8300 --- /dev/null +++ b/bin/agent-os-init.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Verify Pod user-namespace isolation and seed PVC-backed runtime tool paths. +set -eu + +UID_MAP=${AGENT_OS_UID_MAP_PATH:-/proc/self/uid_map} +IMAGE_USR_LOCAL=${AGENT_OS_IMAGE_USR_LOCAL:-/opt/image-usr-local} +PERSISTENT_ROOT=${AGENT_OS_PERSISTENT_ROOT:-/persistent-agent} + +inside= +outside= +length= +read -r inside outside length _ < "$UID_MAP" || { + echo "error: cannot read user namespace map from $UID_MAP" >&2 + exit 1 +} + +case "$inside:$outside:$length" in + *[!0-9:]*|'') + echo "error: invalid user namespace map in $UID_MAP" >&2 + exit 1 + ;; +esac + +if [ "$inside" -ne 0 ] || [ "$outside" -le 0 ] || [ "$length" -lt 65536 ]; then + echo "error: Agent OS requires remapped container root; got '$inside $outside $length'" >&2 + exit 1 +fi + +mkdir -p \ + "$PERSISTENT_ROOT/.config" \ + "$PERSISTENT_ROOT/.cache" \ + "$PERSISTENT_ROOT/.local/bin" \ + "$PERSISTENT_ROOT/.local/share" \ + "$PERSISTENT_ROOT/.bun" \ + "$PERSISTENT_ROOT/.cargo" \ + "$PERSISTENT_ROOT/usr-local" + +rsync -a "$IMAGE_USR_LOCAL/" "$PERSISTENT_ROOT/usr-local/" diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 01a3bfdb1..4663959dd 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -21,7 +21,14 @@ assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5 assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" +assert_grep 'XDG_CONFIG_HOME=/home/agent/.config' "$ROOT/Dockerfile" "image must persist XDG configuration" +assert_grep 'NPM_CONFIG_PREFIX=/usr/local' "$ROOT/Dockerfile" "global npm installs must use persistent /usr/local" +assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin' "$ROOT/Dockerfile" \ + "persistent tool prefixes must lead PATH" +assert_grep '/opt/image-usr-local' "$ROOT/Dockerfile" "image must retain a seed copy of /usr/local" +assert_no_grep 'USER node' "$ROOT/Dockerfile" "Agent OS containers must start as container root" assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" +assert_grep 'setup hooks' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must install persistent AXI hooks" assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the build context" assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" diff --git a/tests/agent-os-init.test.sh b/tests/agent-os-init.test.sh new file mode 100755 index 000000000..a8f6416e8 --- /dev/null +++ b/tests/agent-os-init.test.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# User-namespace verification and persistent tool seeding tests. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP=$(fm_test_tmproot agent-os-init) +mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" +printf '0 100000 65536\n' > "$TMP/mapped" +printf '0 0 4294967295\n' > "$TMP/host" +printf 'baseline\n' > "$TMP/source/bin/baseline" +printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" + +AGENT_OS_UID_MAP_PATH="$TMP/mapped" \ +AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ +AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ + "$ROOT/bin/agent-os-init.sh" + +assert_grep baseline "$TMP/persistent/usr-local/bin/baseline" "initializer must seed image tools" +assert_grep runtime "$TMP/persistent/usr-local/bin/runtime-added" "initializer must preserve runtime tools" + +for directory in .config .cache .local/bin .local/share .bun .cargo usr-local; do + [ -d "$TMP/persistent/$directory" ] || fail "initializer must create persistent $directory" +done + +if AGENT_OS_UID_MAP_PATH="$TMP/host" \ + AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ + AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ + "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then + fail "initializer must reject the host user namespace" +fi + +pass "initializer verifies remapping and preserves persistent tools" From c167dc7f035f6af21f335ca9f1599e754f203a9c Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:23:25 +0200 Subject: [PATCH 06/56] feat: isolate root agents with user namespaces --- bin/agent-os-crewmate.sh | 20 ++++++++++++++++---- deploy/orbstack/primary.yaml | 21 ++++++++++++++++----- tests/agent-os-kubernetes.test.sh | 8 ++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 32218f87c..183bc3aae 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -53,12 +53,21 @@ metadata: app.kubernetes.io/component: crewmate agent-os.akua.dev/crewmate: $ID spec: + hostUsers: false automountServiceAccountToken: false securityContext: - fsGroup: 1000 - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 + fsGroup: 0 + runAsGroup: 0 + runAsUser: 0 + initContainers: + - name: agent-os-init + image: $IMAGE + imagePullPolicy: Never + command: + - /opt/agent-os/bin/agent-os-init.sh + volumeMounts: + - name: home + mountPath: /persistent-agent containers: - name: crewmate image: $IMAGE @@ -78,6 +87,9 @@ spec: volumeMounts: - name: home mountPath: /home/agent + - name: home + mountPath: /usr/local + subPath: usr-local volumes: - name: home persistentVolumeClaim: diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml index 7c67cbd6e..e4e9a80e3 100644 --- a/deploy/orbstack/primary.yaml +++ b/deploy/orbstack/primary.yaml @@ -34,12 +34,21 @@ spec: app.kubernetes.io/name: agent-os app.kubernetes.io/component: firstmate spec: + hostUsers: false serviceAccountName: agent-os-firstmate securityContext: - fsGroup: 1000 - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 + fsGroup: 0 + runAsGroup: 0 + runAsUser: 0 + initContainers: + - name: agent-os-init + image: agent-os:dev + imagePullPolicy: Never + command: + - /opt/agent-os/bin/agent-os-init.sh + volumeMounts: + - name: home + mountPath: /persistent-agent containers: - name: firstmate image: agent-os:dev @@ -67,8 +76,10 @@ spec: volumeMounts: - name: home mountPath: /home/agent + - name: home + mountPath: /usr/local + subPath: usr-local volumes: - name: home persistentVolumeClaim: claimName: agent-os-firstmate-home - diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index dfaac9040..5e5ebbffa 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -20,6 +20,10 @@ assert_contains "$rendered" 'name: agent-os-firstmate-home' "render must include assert_contains "$rendered" 'kind: StatefulSet' "render must include the primary StatefulSet" assert_contains "$rendered" 'imagePullPolicy: Never' "OrbStack must use the locally built image" assert_contains "$rendered" 'mountPath: /home/agent' "the primary home must mount at FM_HOME" +assert_contains "$rendered" 'hostUsers: false' "primary must use a Pod user namespace" +assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" +assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" +assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" pass "OrbStack manifests render the isolated persistent primary" cat > "$FAKEBIN/kubectl" <<'SH' @@ -46,6 +50,10 @@ grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must ap assert_grep 'agent-os.akua.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" +assert_grep 'hostUsers: false' "$STDIN_LOG" "children must use Pod user namespaces" +assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" +assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" +assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" pass "crewmate create emits one isolated Pod and PVC" : > "$CALLS" From b53e32c413e7dcd6d2a2bd3ba329f2571c489d10 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:38:16 +0200 Subject: [PATCH 07/56] fix: run root agents on local OrbStack --- bin/agent-os-crewmate.sh | 1 - bin/agent-os-init.sh | 23 +------ deploy/orbstack/primary.yaml | 1 - ...-07-12-root-userns-persistent-toolchain.md | 62 ++++++++----------- ...root-userns-persistent-toolchain-design.md | 33 +++++----- tests/agent-os-init.test.sh | 12 +--- tests/agent-os-kubernetes.test.sh | 4 +- 7 files changed, 44 insertions(+), 92 deletions(-) diff --git a/bin/agent-os-crewmate.sh b/bin/agent-os-crewmate.sh index 183bc3aae..2767f0b9c 100755 --- a/bin/agent-os-crewmate.sh +++ b/bin/agent-os-crewmate.sh @@ -53,7 +53,6 @@ metadata: app.kubernetes.io/component: crewmate agent-os.akua.dev/crewmate: $ID spec: - hostUsers: false automountServiceAccountToken: false securityContext: fsGroup: 0 diff --git a/bin/agent-os-init.sh b/bin/agent-os-init.sh index e270d8300..4b88ae911 100755 --- a/bin/agent-os-init.sh +++ b/bin/agent-os-init.sh @@ -1,31 +1,10 @@ #!/usr/bin/env bash -# Verify Pod user-namespace isolation and seed PVC-backed runtime tool paths. +# Seed PVC-backed runtime tool paths before the agent container starts. set -eu -UID_MAP=${AGENT_OS_UID_MAP_PATH:-/proc/self/uid_map} IMAGE_USR_LOCAL=${AGENT_OS_IMAGE_USR_LOCAL:-/opt/image-usr-local} PERSISTENT_ROOT=${AGENT_OS_PERSISTENT_ROOT:-/persistent-agent} -inside= -outside= -length= -read -r inside outside length _ < "$UID_MAP" || { - echo "error: cannot read user namespace map from $UID_MAP" >&2 - exit 1 -} - -case "$inside:$outside:$length" in - *[!0-9:]*|'') - echo "error: invalid user namespace map in $UID_MAP" >&2 - exit 1 - ;; -esac - -if [ "$inside" -ne 0 ] || [ "$outside" -le 0 ] || [ "$length" -lt 65536 ]; then - echo "error: Agent OS requires remapped container root; got '$inside $outside $length'" >&2 - exit 1 -fi - mkdir -p \ "$PERSISTENT_ROOT/.config" \ "$PERSISTENT_ROOT/.cache" \ diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml index e4e9a80e3..1a1717d70 100644 --- a/deploy/orbstack/primary.yaml +++ b/deploy/orbstack/primary.yaml @@ -34,7 +34,6 @@ spec: app.kubernetes.io/name: agent-os app.kubernetes.io/component: firstmate spec: - hostUsers: false serviceAccountName: agent-os-firstmate securityContext: fsGroup: 0 diff --git a/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md b/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md index b665ae958..3779d2312 100644 --- a/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md +++ b/docs/superpowers/plans/2026-07-12-root-userns-persistent-toolchain.md @@ -1,19 +1,19 @@ -# Root User Namespace and Persistent Toolchain Implementation Plan +# Local Root Agents and Persistent Toolchain Implementation Plan > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -**Goal:** Run every Agent OS Pod as container root mapped to an unprivileged host UID while providing Firstmate's complete baseline toolchain and preserving runtime-installed tools across Pod replacement. +**Goal:** Run every local Agent OS Pod as container root while providing Firstmate's complete baseline toolchain and preserving runtime-installed tools across Pod replacement. -**Architecture:** Extend the reproducible image with exact binary and npm package versions, then seed the image's `/usr/local` tree into each agent PVC from an init container. The main container mounts that tree at `/usr/local`, uses the same PVC for `/home/agent`, and runs as UID 0 inside a Pod user namespace selected by `hostUsers: false`. +**Architecture:** Extend the reproducible image with exact binary and npm package versions, then seed the image's `/usr/local` tree into each agent PVC from an init container. The main container mounts that tree at `/usr/local`, uses the same PVC for `/home/agent`, and runs as UID 0 on the isolated local OrbStack VM node. -**Tech Stack:** Docker, Bash, Kubernetes v1.34, Kustomize, StatefulSet, Pods, PVCs, Linux user namespaces, Herdr, Pi, Node.js, npm. +**Tech Stack:** Docker, Bash, Kubernetes v1.34, Kustomize, StatefulSet, Pods, PVCs, Herdr, Pi, Node.js, npm. ## Global Constraints -- Every primary and crewmate Pod sets `hostUsers: false`, `runAsUser: 0`, and `runAsGroup: 0`. +- Every primary and crewmate Pod sets `runAsUser: 0` and `runAsGroup: 0`. +- OrbStack's unsupported `hostUsers: false` field is omitted from this isolated local demo. - The primary retains the existing local-demo `cluster-admin` ServiceAccount binding, while crewmates receive no Kubernetes credentials by default. - No Pod uses privileged mode, host PID, host IPC, host networking, raw block devices, or host-path mounts. -- The deployment fails closed when the Pod UID map does not map container UID 0 to a nonzero host UID. - Each agent has its own PVC-backed `/home/agent` and `/usr/local`; no persistent state is shared between agents. - Runtime additions in `/usr/local` and persistent home prefixes survive Pod replacement. - Runtime `apt` changes outside the persistent prefixes remain intentionally ephemeral. @@ -128,7 +128,7 @@ git commit -m "feat: bundle the Firstmate toolchain" --- -### Task 2: Add the user-namespace and persistent-prefix initializer +### Task 2: Add the persistent-prefix initializer **Files:** - Create: `bin/agent-os-init.sh` @@ -137,12 +137,12 @@ git commit -m "feat: bundle the Firstmate toolchain" - Modify: `bin/agent-os-container-entrypoint.sh` **Interfaces:** -- Consumes: `AGENT_OS_UID_MAP_PATH`, defaulting to `/proc/self/uid_map`; `AGENT_OS_PERSISTENT_ROOT`, defaulting to `/persistent-agent`. -- Produces: a PVC root used as `/home/agent` and a seeded `$AGENT_OS_PERSISTENT_ROOT/usr-local`; exit zero only for a remapped UID 0 range. +- Consumes: `AGENT_OS_PERSISTENT_ROOT`, defaulting to `/persistent-agent`, and `AGENT_OS_IMAGE_USR_LOCAL`, defaulting to `/opt/image-usr-local`. +- Produces: a PVC root used as `/home/agent` and a seeded `$AGENT_OS_PERSISTENT_ROOT/usr-local`. - [ ] **Step 1: Write failing initializer tests** -Create `tests/agent-os-init.test.sh` with fixtures for a mapped and host UID map: +Create `tests/agent-os-init.test.sh` with image-baseline and runtime-added fixtures: ```bash #!/usr/bin/env bash @@ -151,12 +151,9 @@ set -u TMP=$(fm_test_tmproot agent-os-init) mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" -printf '0 100000 65536\n' > "$TMP/mapped" -printf '0 0 4294967295\n' > "$TMP/host" printf 'baseline\n' > "$TMP/source/bin/baseline" printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" -AGENT_OS_UID_MAP_PATH="$TMP/mapped" \ AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ "$ROOT/bin/agent-os-init.sh" @@ -164,13 +161,7 @@ AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ assert_grep baseline "$TMP/persistent/usr-local/bin/baseline" "initializer must seed image tools" assert_grep runtime "$TMP/persistent/usr-local/bin/runtime-added" "initializer must preserve runtime tools" -if AGENT_OS_UID_MAP_PATH="$TMP/host" \ - AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ - AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ - "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then - fail "initializer must reject the host user namespace" -fi -pass "initializer verifies remapping and preserves persistent tools" +pass "initializer preserves image and runtime-installed tools" ``` - [ ] **Step 2: Run the test and observe the missing initializer** @@ -243,7 +234,7 @@ git commit -m "feat: persist agent-installed tools" --- -### Task 3: Run primary and crewmates as remapped container root +### Task 3: Run primary and crewmates as local container root **Files:** - Modify: `deploy/orbstack/primary.yaml` @@ -252,18 +243,18 @@ git commit -m "feat: persist agent-installed tools" **Interfaces:** - Consumes: image path `/opt/image-usr-local` and per-agent PVC. -- Produces: Pod specs with `hostUsers: false`, root security context, init container `agent-os-init`, whole-PVC home mount, and `usr-local` subPath mount. +- Produces: Pod specs with root security context, init container `agent-os-init`, whole-PVC home mount, and `usr-local` subPath mount. - [ ] **Step 1: Add failing manifest assertions** Extend `tests/agent-os-kubernetes.test.sh` for both rendered primary YAML and captured child YAML: ```bash -assert_contains "$rendered" 'hostUsers: false' "primary must use a Pod user namespace" +assert_not_contains "$rendered" 'hostUsers: false' "OrbStack demo must not request unsupported Pod user namespaces" assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" -assert_grep 'hostUsers: false' "$STDIN_LOG" "children must use Pod user namespaces" +assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" @@ -275,11 +266,10 @@ Retain the existing assertion for `automountServiceAccountToken: false`. Run: `bash tests/agent-os-kubernetes.test.sh` -Expected: FAIL at `primary must use a Pod user namespace`. +Expected: FAIL while the manifest still requests unsupported Pod user namespaces. - [ ] **Step 3: Update the primary StatefulSet** -Set `spec.template.spec.hostUsers: false`. Replace the non-root security context with `runAsUser: 0` and `runAsGroup: 0`. Add an `agent-os-init` init container using `agent-os:dev`, command `/opt/agent-os/bin/agent-os-init.sh`, and a whole-PVC mount at `/persistent-agent`. Mount the PVC root at `/home/agent` and PVC subPath `usr-local` at `/usr/local` in the main container. @@ -287,7 +277,7 @@ Do not set `privileged: true` or any host namespace option. - [ ] **Step 4: Update generated crewmate Pods** -Make `bin/agent-os-crewmate.sh create` emit the same user-namespace, root, init-container, home, and `/usr/local` structure. +Make `bin/agent-os-crewmate.sh create` emit the same root, init-container, home, and `/usr/local` structure. Keep the distinct child PVC and `automountServiceAccountToken: false` contract unchanged. - [ ] **Step 5: Run static Kubernetes tests** @@ -306,19 +296,19 @@ Expected: all checks pass. ```bash git add deploy/orbstack/primary.yaml bin/agent-os-crewmate.sh tests/agent-os-kubernetes.test.sh -git commit -m "feat: isolate root agents with user namespaces" +git commit -m "feat: run local agents as container root" ``` --- -### Task 4: Prove the live user namespace and persistent tool lifecycle +### Task 4: Prove the live root and persistent tool lifecycle **Files:** - Modify if evidence reveals a defect: files owned by Tasks 1 through 3 and their colocated tests. **Interfaces:** - Consumes: OrbStack context, `agent-os:dev`, namespace `agent-os-demo`. -- Produces: empirical proof for remapped root, baseline completeness, and durable runtime tools. +- Produces: empirical proof for container root, baseline completeness, and durable runtime tools. - [ ] **Step 1: Rebuild and deploy** @@ -332,16 +322,15 @@ kubectl --context orbstack -n agent-os-demo rollout status statefulset/agent-os- Expected: the StatefulSet reaches `1/1` Ready. -- [ ] **Step 2: Verify remapped container root** +- [ ] **Step 2: Verify container root** Run: ```bash kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- id -kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- cat /proc/self/uid_map ``` -Expected: `id` reports UID 0 and the UID map begins with container UID `0`, a nonzero host UID, and a range of at least `65536`. +Expected: `id` reports UID 0. - [ ] **Step 3: Verify the complete baseline** @@ -392,12 +381,11 @@ Run: kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- /opt/agent-os/bin/agent-os-crewmate.sh create root-proof kubectl --context orbstack -n agent-os-demo wait --for=condition=Ready pod/agent-os-crewmate-root-proof --timeout=180s kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- id -kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- cat /proc/self/uid_map kubectl --context orbstack -n agent-os-demo exec agent-os-crewmate-root-proof -- test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token kubectl --context orbstack -n agent-os-demo exec statefulset/agent-os-firstmate -- /opt/agent-os/bin/agent-os-crewmate.sh delete root-proof ``` -Expected: child UID 0 is remapped, no ServiceAccount token exists, and cleanup removes its Pod and PVC. +Expected: child UID 0 is active, no ServiceAccount token exists, and cleanup removes its Pod and PVC. - [ ] **Step 7: Convert any live defect into a failing test before fixing it** @@ -427,12 +415,12 @@ git diff --cached --quiet || git commit -m "fix: complete the persistent root ag Document these verified facts in `docs/kubernetes.md`, one sentence per physical line: ```text -Agents run as UID 0 inside a Pod user namespace and are mapped to unprivileged node UIDs. +Agents run as UID 0 on the isolated local OrbStack VM node. The complete Firstmate baseline is part of the image. Tools installed below /home/agent persistent prefixes or /usr/local survive Pod replacement. apt remains available, but files it writes outside persistent prefixes are ephemeral. GitHub and model authentication live on the individual agent PVC. -The deployment fails instead of falling back when user namespaces are unsupported. +The Pods do not use privileged mode, host namespaces, or host mounts. ``` - [ ] **Step 2: Run the complete focused verification set** diff --git a/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md b/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md index f0a682e77..55f40abd3 100644 --- a/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md +++ b/docs/superpowers/specs/2026-07-12-root-userns-persistent-toolchain-design.md @@ -1,19 +1,19 @@ -# Root user-namespace and persistent toolchain design +# Local root agents and persistent toolchain design **Date:** 2026-07-12 ## Purpose -Agent OS Pods must be able to administer their own container environment without becoming root on the Kubernetes node. +Agent OS Pods must be able to administer their own container environment in the isolated local OrbStack cluster. The image must already contain the complete Firstmate toolchain so a fresh controller can begin useful work without an installation round. Tools and authentication added by an agent at runtime must survive Pod replacement. ## Decisions -Every Agent OS Pod runs as UID 0 inside a Kubernetes Pod user namespace. -Every Pod sets `hostUsers: false`, so Kubernetes maps container UID 0 to a distinct unprivileged host UID range. -The deployment fails closed when the node, filesystem, container runtime, or cluster configuration cannot provide Pod user namespaces. -No Agent OS Pod uses privileged mode, a host user namespace, a host PID namespace, a host IPC namespace, a host network namespace, a raw block device, or a host-path mount. +Every Agent OS Pod runs as UID 0 inside its container. +OrbStack's built-in Kubernetes uses cri-dockerd and rejects `hostUsers: false`, so the local demo does not request a Pod user namespace. +Container root is therefore root on the dedicated OrbStack VM node for this local demo. +No Agent OS Pod uses privileged mode, a host PID namespace, a host IPC namespace, a host network namespace, a raw block device, or a host-path mount. The image remains the reproducible baseline. Each agent receives its own PVC-backed home and persistent `/usr/local` tree for runtime adaptation. @@ -59,17 +59,16 @@ An agent that needs an additional durable CLI should prefer `/usr/local` or a pe ## Pod security and authority -The primary and crewmate containers set `runAsUser: 0` and `runAsGroup: 0` inside their Pod user namespace. -The init container uses the same user namespace and container-root identity to seed the persistent tool tree. -The Pods use the runtime's normal namespaced capabilities and do not request privileged mode. +The primary and crewmate containers set `runAsUser: 0` and `runAsGroup: 0`. +The init container uses the same container-root identity to seed the persistent tool tree. +The Pods use the runtime's normal container isolation and do not request privileged mode. The Firstmate ServiceAccount retains cluster-admin only for the isolated local Agent OS cluster. Crewmates continue to receive no Kubernetes ServiceAccount token by default. Container root and Kubernetes API authority remain separate controls. -Kubernetes v1.34 marks Pod user namespaces beta. -The target node must use a compatible Linux kernel, idmapped-mount filesystem, containerd 2.0 or later or another supported CRI runtime, and a compatible OCI runtime. -The local demo must verify the actual UID map rather than infer support from the Kubernetes version. +This root policy is explicitly limited to the disposable, agents-only local OrbStack VM cluster. +A remote or shared Agent OS cluster must define and verify its own stronger runtime isolation before reusing this policy. ## Startup flow @@ -82,17 +81,15 @@ The local demo must verify the actual UID map rather than infer support from the ## Failure behavior -The demo must refuse to claim readiness when Pod user namespaces are unavailable. -The verification path checks `/proc/self/uid_map` and proves that container UID 0 maps to a nonzero host UID range. The image build stops on missing releases, unsupported architectures, checksum failures, or missing baseline commands. The init container stops startup if it cannot seed the persistent tool tree. -No fallback silently removes `hostUsers: false` or grants privileged mode. +No fallback grants privileged mode, host namespaces, or host mounts. ## Verification -Static tests assert that both primary and generated crewmate Pod specs use `hostUsers: false` and container UID 0. +Static tests assert that both primary and generated crewmate Pod specs use container UID 0 without requesting the unsupported `hostUsers: false` field. Container tests assert every baseline command is present and Firstmate bootstrap emits no `MISSING:` diagnostics. -Runtime tests prove the Pod UID map is remapped, root can write to normal container paths, and durable installs can write to both persistent prefixes. +Runtime tests prove root can write to normal container paths and durable installs can write to both persistent prefixes. Persistence tests write one tool below `/home/agent/.local/bin` and one below `/usr/local/bin`, replace the Pod, and execute both tools afterward. Isolation tests prove crewmates have distinct PVCs and no ServiceAccount token. Authentication verification checks that GitHub CLI configuration survives Pod replacement without embedding credentials in the image. @@ -105,5 +102,5 @@ It does not define production-cluster RBAC beyond retaining the existing local-d ## Sources -- Kubernetes v1.34 user-namespace documentation: . +- Kubernetes v1.34 user-namespace documentation, which records cri-dockerd as unsupported: . - Firstmate's universal toolchain contract: `docs/configuration.md` under `Toolchain` and `bin/fm-bootstrap.sh`. diff --git a/tests/agent-os-init.test.sh b/tests/agent-os-init.test.sh index a8f6416e8..5e29c2854 100755 --- a/tests/agent-os-init.test.sh +++ b/tests/agent-os-init.test.sh @@ -7,12 +7,9 @@ set -u TMP=$(fm_test_tmproot agent-os-init) mkdir -p "$TMP/source/bin" "$TMP/persistent/.local/bin" "$TMP/persistent/usr-local/bin" -printf '0 100000 65536\n' > "$TMP/mapped" -printf '0 0 4294967295\n' > "$TMP/host" printf 'baseline\n' > "$TMP/source/bin/baseline" printf 'runtime\n' > "$TMP/persistent/usr-local/bin/runtime-added" -AGENT_OS_UID_MAP_PATH="$TMP/mapped" \ AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ "$ROOT/bin/agent-os-init.sh" @@ -24,11 +21,4 @@ for directory in .config .cache .local/bin .local/share .bun .cargo usr-local; d [ -d "$TMP/persistent/$directory" ] || fail "initializer must create persistent $directory" done -if AGENT_OS_UID_MAP_PATH="$TMP/host" \ - AGENT_OS_IMAGE_USR_LOCAL="$TMP/source" \ - AGENT_OS_PERSISTENT_ROOT="$TMP/persistent" \ - "$ROOT/bin/agent-os-init.sh" >/dev/null 2>&1; then - fail "initializer must reject the host user namespace" -fi - -pass "initializer verifies remapping and preserves persistent tools" +pass "initializer preserves image and runtime-installed tools" diff --git a/tests/agent-os-kubernetes.test.sh b/tests/agent-os-kubernetes.test.sh index 5e5ebbffa..6a41c088c 100755 --- a/tests/agent-os-kubernetes.test.sh +++ b/tests/agent-os-kubernetes.test.sh @@ -20,7 +20,7 @@ assert_contains "$rendered" 'name: agent-os-firstmate-home' "render must include assert_contains "$rendered" 'kind: StatefulSet' "render must include the primary StatefulSet" assert_contains "$rendered" 'imagePullPolicy: Never' "OrbStack must use the locally built image" assert_contains "$rendered" 'mountPath: /home/agent' "the primary home must mount at FM_HOME" -assert_contains "$rendered" 'hostUsers: false' "primary must use a Pod user namespace" +assert_not_contains "$rendered" 'hostUsers: false' "OrbStack demo must not request unsupported Pod user namespaces" assert_contains "$rendered" 'runAsUser: 0' "primary must run as container root" assert_contains "$rendered" 'name: agent-os-init' "primary must seed persistent tools" assert_contains "$rendered" 'mountPath: /usr/local' "primary must persist /usr/local" @@ -50,7 +50,7 @@ grep -Fqx 'kubectl -n agent-os-demo apply -f -' "$CALLS" || fail "create must ap assert_grep 'agent-os.akua.dev/crewmate: scout-1' "$STDIN_LOG" "child resources need the stable crewmate label" assert_grep 'automountServiceAccountToken: false' "$STDIN_LOG" "children must not receive Kubernetes credentials" assert_grep 'claimName: agent-os-crewmate-scout-1-home' "$STDIN_LOG" "child work must use its own PVC" -assert_grep 'hostUsers: false' "$STDIN_LOG" "children must use Pod user namespaces" +assert_no_grep 'hostUsers: false' "$STDIN_LOG" "OrbStack children must not request unsupported Pod user namespaces" assert_grep 'runAsUser: 0' "$STDIN_LOG" "children must run as container root" assert_grep 'name: agent-os-init' "$STDIN_LOG" "children must seed persistent tools" assert_grep 'mountPath: /usr/local' "$STDIN_LOG" "children must persist /usr/local" From 25b506a52a7092bf98bd6e1fa6e96c655e301bdd Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 08:45:41 +0200 Subject: [PATCH 08/56] docs: explain persistent root agents --- docs/kubernetes.md | 47 +++++++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 45b0f088b..bfde97b14 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -1,20 +1,21 @@ # Kubernetes Agent OS demo -This demo runs Firstmate as the persistent controller of an isolated agent -cluster. Kubernetes distributes and isolates the crew; it does not replace -Firstmate's supervision model or Herdr's terminal/session interface. +This demo runs Firstmate as the persistent controller of an isolated agent cluster. +Kubernetes distributes and isolates the crew; it does not replace Firstmate's supervision model or Herdr's terminal/session interface. The local topology is deliberately small: - `agent-os-firstmate-0` is a StatefulSet with a 20 Gi persistent home. - Herdr 0.7.3 is PID 1 and remains the visible session backend. -- the primary's local-demo ServiceAccount has `cluster-admin`, so it can shape - its own crew and workspace; +- the primary's local-demo ServiceAccount has `cluster-admin`, so it can shape its own crew and workspace; - every crewmate is an adaptable Agent OS container with its own 10 Gi PVC; - crewmates do not receive Kubernetes service-account credentials by default. -The broad primary grant is only for the isolated local agent cluster. It is not -a production-cluster access pattern. +The primary and crewmates run as UID 0 inside their containers. +OrbStack's built-in Kubernetes does not support Pod user namespaces, so container root is also root on the dedicated OrbStack VM node. +The Pods do not use privileged mode, host namespaces, host mounts, or raw block devices. +This root policy and the broad primary grant are only for the isolated local agent cluster. +They are not a production-cluster access pattern. ## Requirements @@ -22,8 +23,8 @@ a production-cluster access pattern. - Docker - `kubectl` -The helper refuses ambient Kubernetes contexts. It uses `orbstack` unless a -different context is deliberately enabled with `AGENT_OS_ALLOW_NON_ORBSTACK=1`. +The helper refuses ambient Kubernetes contexts. +It uses `orbstack` unless a different context is deliberately enabled with `AGENT_OS_ALLOW_NON_ORBSTACK=1`. ## Run the demo @@ -34,10 +35,15 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` -Inside the primary container, authenticate the Pi harness using the provider -flow you choose. Host model credentials are intentionally excluded from the -image and are never copied automatically. The authenticated home persists on -the primary PVC. +The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, treehouse, no-mistakes, and every required AXI CLI. +Authenticate GitHub inside the primary with `gh auth login`. +Authenticate Pi using `/login` and the provider flow you choose. +Host credentials are intentionally excluded from the image and are never copied automatically. +Authentication stored below `/home/agent` persists on the individual agent PVC. + +Tools installed below `/home/agent/.local`, `/home/agent/.bun`, `/home/agent/.cargo`, or `/usr/local` survive Pod replacement. +Global npm installs use the persistent `/usr/local` prefix. +`apt` is available because the container runs as root, but packages written elsewhere in the container filesystem are ephemeral. Start Pi from the tracked distro and attach to Herdr from another terminal: @@ -72,12 +78,11 @@ bin/agent-os-local.sh destroy --yes ## What the demo proves -The primary and child homes survive Pod replacement. A child cannot use the -Kubernetes API through an automatically mounted token. The primary can create, -inspect, and delete child Pods and PVCs using ordinary `kubectl`; no custom -agent communication protocol or GitOps controller is required. +The primary and child homes survive Pod replacement. +Tools installed into the persistent home prefixes and `/usr/local` also survive Pod replacement. +A child cannot use the Kubernetes API through an automatically mounted token. +The primary can create, inspect, and delete child Pods and PVCs using ordinary `kubectl`. +No custom agent communication protocol or GitOps controller is required. -This is the local substrate, not the final access model. Remote Agent OS -clusters should keep the intelligence cluster isolated and grant product or -production access deliberately per task, with Akua providing stable -infrastructure primitives and guardrails. +This is the local substrate, not the final access model. +Remote Agent OS clusters should keep the intelligence cluster isolated and grant product or production access deliberately per task, with Akua providing stable infrastructure primitives and guardrails. From 69100998c1e04cce60db27d1db37a08c0ab05b5e Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 09:33:50 +0200 Subject: [PATCH 09/56] feat: scaffold Agent OS operating tool --- tools/agent-os/.bun-version | 1 + tools/agent-os/.gitignore | 1 + tools/agent-os/AGENTS.md | 46 +++++++ tools/agent-os/README.md | 28 +++++ tools/agent-os/bun.lock | 79 ++++++++++++ tools/agent-os/package.json | 22 ++++ tools/agent-os/src/address.ts | 183 ++++++++++++++++++++++++++++ tools/agent-os/src/cli.ts | 85 +++++++++++++ tools/agent-os/test/address.test.ts | 63 ++++++++++ tools/agent-os/test/cli.test.ts | 36 ++++++ tools/agent-os/tsconfig.json | 15 +++ 11 files changed, 559 insertions(+) create mode 100644 tools/agent-os/.bun-version create mode 100644 tools/agent-os/.gitignore create mode 100644 tools/agent-os/AGENTS.md create mode 100644 tools/agent-os/README.md create mode 100644 tools/agent-os/bun.lock create mode 100644 tools/agent-os/package.json create mode 100644 tools/agent-os/src/address.ts create mode 100755 tools/agent-os/src/cli.ts create mode 100644 tools/agent-os/test/address.test.ts create mode 100644 tools/agent-os/test/cli.test.ts create mode 100644 tools/agent-os/tsconfig.json diff --git a/tools/agent-os/.bun-version b/tools/agent-os/.bun-version new file mode 100644 index 000000000..085c0f266 --- /dev/null +++ b/tools/agent-os/.bun-version @@ -0,0 +1 @@ +1.3.14 diff --git a/tools/agent-os/.gitignore b/tools/agent-os/.gitignore new file mode 100644 index 000000000..c2658d7d1 --- /dev/null +++ b/tools/agent-os/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/tools/agent-os/AGENTS.md b/tools/agent-os/AGENTS.md new file mode 100644 index 000000000..8ff026edf --- /dev/null +++ b/tools/agent-os/AGENTS.md @@ -0,0 +1,46 @@ +# Agent OS Tool Development Guide + +These instructions apply only inside `tools/agent-os/`. +They govern development of the operating tool and must not be copied into the root Firstmate instructions. + +## Purpose + +This tool incrementally moves complex fleet mechanics from Bash into a small Bun and TypeScript CLI. +It extends Firstmate's existing files and lifecycle rather than introducing a parallel state store, controller, or agent model. +Keep compatibility with the root distro's `data/`, `state/`, briefs, status events, and guarded teardown behavior. +Use the Bun channel declared in `.bun-version`. +Keep `.bun-version`, `packageManager`, and the matching Bun type definitions aligned on the latest stable release. + +## Addresses + +Use Multiaddr strings as the single locator for supervised mates. +Persist the canonical string form only. +Do not persist Multiaddr bytes while Agent OS protocols use application-private numeric codes. +Keep credentials, resource requests, authority grants, Pod names, UIDs, and IP addresses out of addresses. +Use stable mate IDs and let each resolver discover the current runtime resource. +Treat `in-cluster` as the reserved alias for the cluster containing the caller. +The address grammar and validation contract are owned by `src/address.ts` and its tests. + +## Command Execution + +Use Bun Shell's `$` for finite calls to trusted CLIs such as `kubectl`, `herdr`, `treehouse`, and Git. +Interpolate dynamic values directly so Bun treats each value as a literal argument. +Never use `{ raw: value }` for dynamic data. +Avoid `bash -c` and validate values that an external command could interpret as flags. +Keep non-zero exits throwing by default and use `.nothrow()` only for an expected negative probe. +Set environment and working directory per command instead of mutating the global `$` instance. +Use `Bun.spawn()` for interactive agents, servers, watches, live logs, cancellation, and other long-running processes. +Existing Bash scripts may be invoked explicitly during migration, but new orchestration and state transitions belong in TypeScript. + +## Scope Discipline + +Prefer a thin adapter over a new framework. +Do not add a database, daemon, custom inter-agent protocol, Kubernetes controller, or duplicate task specification. +Keep mates general-purpose and keep parent-only communication unchanged across local and remote addresses. +Use Kubernetes APIs and later Agent Sandbox only as runtime resolvers behind the same logical address. + +## Verification + +Run `bun run check` from this directory after changes. +Add focused Bun tests beside every new address, resolver, or command contract. +Test failure, ambiguity, and unsafe input paths as well as successful execution. diff --git a/tools/agent-os/README.md b/tools/agent-os/README.md new file mode 100644 index 000000000..f37699a54 --- /dev/null +++ b/tools/agent-os/README.md @@ -0,0 +1,28 @@ +# Agent OS operating tool + +This directory contains the Bun and TypeScript operating tool that will gradually absorb Firstmate's complex fleet mechanics. +The root agent distro remains instructions, skills, trusted CLIs, and durable local files. + +The first implemented contract is a self-describing Agent OS profile over [Multiaddr](https://github.com/multiformats/multiaddr). + +```text +/local/mate/task-x/herdr +/k8s/in-cluster/namespace/agent-os/mate/task-x/herdr +``` + +The Kubernetes form names the cluster alias, namespace, stable mate ID, and terminal protocol without encoding an ephemeral Pod or IP. +The resolver may later find a raw Pod or an Agent Sandbox without changing the stored address. + +## Development + +The tool pins the latest stable Bun release in `.bun-version` and `package.json`. + +```sh +bun install +bun run check +bun run src/cli.ts address local task-x +bun run src/cli.ts address k8s task-x +bun run src/cli.ts address inspect /k8s/in-cluster/namespace/agent-os/mate/task-x/herdr +``` + +See `AGENTS.md` in this directory for the tool-specific development rules. diff --git a/tools/agent-os/bun.lock b/tools/agent-os/bun.lock new file mode 100644 index 000000000..717d11312 --- /dev/null +++ b/tools/agent-os/bun.lock @@ -0,0 +1,79 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "@akua-dev/agent-os-tool", + "dependencies": { + "@multiformats/multiaddr": "13.0.3", + }, + "devDependencies": { + "@types/bun": "1.3.14", + "typescript": "7.0.2", + }, + }, + }, + "packages": { + "@chainsafe/is-ip": ["@chainsafe/is-ip@2.1.0", "", {}, "sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w=="], + + "@multiformats/multiaddr": ["@multiformats/multiaddr@13.0.3", "", { "dependencies": { "@chainsafe/is-ip": "^2.0.1", "multiformats": "^14.0.0", "uint8-varint": "^3.0.0", "uint8arrays": "^6.1.1" } }, "sha512-mEqqJ4r3a/uuFMTpRkU316wGNIDQNhuVWpm+ebKTQeYsfv9jXbPONWM6VVnj3KGUrwfsX7GZOyp4TFqEA2SPCw=="], + + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + + "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], + + "@typescript/typescript-aix-ppc64": ["@typescript/typescript-aix-ppc64@7.0.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ=="], + + "@typescript/typescript-darwin-arm64": ["@typescript/typescript-darwin-arm64@7.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA=="], + + "@typescript/typescript-darwin-x64": ["@typescript/typescript-darwin-x64@7.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA=="], + + "@typescript/typescript-freebsd-arm64": ["@typescript/typescript-freebsd-arm64@7.0.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ=="], + + "@typescript/typescript-freebsd-x64": ["@typescript/typescript-freebsd-x64@7.0.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw=="], + + "@typescript/typescript-linux-arm": ["@typescript/typescript-linux-arm@7.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ=="], + + "@typescript/typescript-linux-arm64": ["@typescript/typescript-linux-arm64@7.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ=="], + + "@typescript/typescript-linux-loong64": ["@typescript/typescript-linux-loong64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ=="], + + "@typescript/typescript-linux-mips64el": ["@typescript/typescript-linux-mips64el@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA=="], + + "@typescript/typescript-linux-ppc64": ["@typescript/typescript-linux-ppc64@7.0.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA=="], + + "@typescript/typescript-linux-riscv64": ["@typescript/typescript-linux-riscv64@7.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ=="], + + "@typescript/typescript-linux-s390x": ["@typescript/typescript-linux-s390x@7.0.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw=="], + + "@typescript/typescript-linux-x64": ["@typescript/typescript-linux-x64@7.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A=="], + + "@typescript/typescript-netbsd-arm64": ["@typescript/typescript-netbsd-arm64@7.0.2", "", { "os": "none", "cpu": "arm64" }, "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA=="], + + "@typescript/typescript-netbsd-x64": ["@typescript/typescript-netbsd-x64@7.0.2", "", { "os": "none", "cpu": "x64" }, "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA=="], + + "@typescript/typescript-openbsd-arm64": ["@typescript/typescript-openbsd-arm64@7.0.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ=="], + + "@typescript/typescript-openbsd-x64": ["@typescript/typescript-openbsd-x64@7.0.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg=="], + + "@typescript/typescript-sunos-x64": ["@typescript/typescript-sunos-x64@7.0.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g=="], + + "@typescript/typescript-win32-arm64": ["@typescript/typescript-win32-arm64@7.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ=="], + + "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + + "multiformats": ["multiformats@14.0.4", "", {}, "sha512-+SXItmOUWzT0kjlIfkZ4NawJaC9jW/mPlyn902V9Qgpc7YDwdEiwqbiZxTyvC0XWh1jZguXca3A/WQ+UOPRLUA=="], + + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], + + "uint8-varint": ["uint8-varint@3.0.0", "", { "dependencies": { "uint8arraylist": "^3.0.1", "uint8arrays": "^6.1.0" } }, "sha512-S4DdpXBaLwKcFo7f0bWzWfHjbZ/i3QhM842qn+ZvHjxqFCfUcEB9SQNcmI69S+zMlcmIcKxsk9Iyw77S2Kxv6Q=="], + + "uint8arraylist": ["uint8arraylist@3.0.2", "", { "dependencies": { "uint8arrays": "^6.0.0" } }, "sha512-LDVoq9BQaGJzGDUovEnoX6rpKCvnY/Jbtws4ikwnBzjRbq5qBAFpBZevUEbSmMM87aO0Sp+wOZy2ZXf5yODmXQ=="], + + "uint8arrays": ["uint8arrays@6.1.1", "", { "dependencies": { "multiformats": "^14.0.0" } }, "sha512-iz7JN0XCSZYA111lhFG2Ui9EhFvTNekqSRHw3lvMHq+dzwWy1OQftxFQREEh4rffU0oSoXdQHsk2TiHKVm4fsA=="], + + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + } +} diff --git a/tools/agent-os/package.json b/tools/agent-os/package.json new file mode 100644 index 000000000..1dd58ead0 --- /dev/null +++ b/tools/agent-os/package.json @@ -0,0 +1,22 @@ +{ + "name": "@akua-dev/agent-os-tool", + "version": "0.0.0", + "private": true, + "type": "module", + "bin": { + "agent-os": "./src/cli.ts" + }, + "scripts": { + "check": "bun run typecheck && bun test", + "test": "bun test", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@multiformats/multiaddr": "13.0.3" + }, + "devDependencies": { + "@types/bun": "1.3.14", + "typescript": "7.0.2" + }, + "packageManager": "bun@1.3.14" +} diff --git a/tools/agent-os/src/address.ts b/tools/agent-os/src/address.ts new file mode 100644 index 000000000..9aee6f568 --- /dev/null +++ b/tools/agent-os/src/address.ts @@ -0,0 +1,183 @@ +import { + V, + multiaddr, + registry, + type Component, + type ProtocolCodec, +} from "@multiformats/multiaddr"; + +const APPLICATION_PROTOCOL_BASE = 0x300000; +const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/; + +export const AGENT_OS_PROTOCOL_CODES = Object.freeze({ + local: APPLICATION_PROTOCOL_BASE, + k8s: APPLICATION_PROTOCOL_BASE + 1, + namespace: APPLICATION_PROTOCOL_BASE + 2, + mate: APPLICATION_PROTOCOL_BASE + 3, + herdr: APPLICATION_PROTOCOL_BASE + 4, +}); + +export type AgentAddress = + | { + address: string; + location: "local"; + mateId: string; + protocol: "herdr"; + } + | { + address: string; + cluster: string; + location: "kubernetes"; + mateId: string; + namespace: string; + protocol: "herdr"; + }; + +export interface KubernetesAddressOptions { + cluster?: string; + namespace?: string; +} + +export class AgentAddressError extends Error { + override name = "AgentAddressError"; +} + +function validateLabel(value: string, component: string): void { + if (!DNS_LABEL.test(value)) { + throw new AgentAddressError( + `Invalid ${component} value ${JSON.stringify(value)}: expected a lowercase DNS label`, + ); + } +} + +const agentOsProtocols: ProtocolCodec[] = [ + { + code: AGENT_OS_PROTOCOL_CODES.local, + name: "local", + size: 0, + }, + { + code: AGENT_OS_PROTOCOL_CODES.k8s, + name: "k8s", + size: V, + validate: (value) => validateLabel(value, "k8s cluster alias"), + }, + { + code: AGENT_OS_PROTOCOL_CODES.namespace, + name: "namespace", + size: V, + validate: (value) => validateLabel(value, "namespace"), + }, + { + code: AGENT_OS_PROTOCOL_CODES.mate, + name: "mate", + size: V, + validate: (value) => validateLabel(value, "mate ID"), + }, + { + code: AGENT_OS_PROTOCOL_CODES.herdr, + name: "herdr", + size: 0, + }, +]; + +function findProtocol(key: string | number): ProtocolCodec | undefined { + try { + return registry.getProtocol(key); + } catch { + return undefined; + } +} + +export function registerAgentOsProtocols(): void { + for (const protocol of agentOsProtocols) { + const byCode = findProtocol(protocol.code); + const byName = findProtocol(protocol.name); + + if (byCode == null && byName == null) { + registry.addProtocol(protocol); + continue; + } + + if (byCode?.name !== protocol.name || byName?.code !== protocol.code) { + throw new AgentAddressError( + `Multiaddr protocol collision for ${protocol.name}/${protocol.code}`, + ); + } + } +} + +registerAgentOsProtocols(); + +function componentValue(component: Component, name: string): string { + if (component.value == null || component.value === "") { + throw new AgentAddressError(`Agent OS address component ${name} needs a value`); + } + + return component.value; +} + +function hasNames(components: Component[], names: string[]): boolean { + return ( + components.length === names.length && + components.every((component, index) => component.name === names[index]) + ); +} + +export function parseAgentAddress(input: string): AgentAddress { + let parsed; + + try { + parsed = multiaddr(input); + } catch (error) { + throw new AgentAddressError(`Invalid Agent OS address: ${String(error)}`); + } + + const address = parsed.toString(); + const components = parsed.getComponents(); + + if (hasNames(components, ["local", "mate", "herdr"])) { + return { + address, + location: "local", + mateId: componentValue(components[1]!, "mate"), + protocol: "herdr", + }; + } + + if ( + hasNames(components, ["k8s", "namespace", "mate", "herdr"]) + ) { + return { + address, + cluster: componentValue(components[0]!, "k8s"), + location: "kubernetes", + mateId: componentValue(components[2]!, "mate"), + namespace: componentValue(components[1]!, "namespace"), + protocol: "herdr", + }; + } + + throw new AgentAddressError(`Unsupported Agent OS address: ${address}`); +} + +export function formatLocalAgentAddress(mateId: string): string { + validateLabel(mateId, "mate ID"); + return parseAgentAddress(`/local/mate/${mateId}/herdr`).address; +} + +export function formatKubernetesAgentAddress( + mateId: string, + options: KubernetesAddressOptions = {}, +): string { + const cluster = options.cluster ?? "in-cluster"; + const namespace = options.namespace ?? "agent-os"; + + validateLabel(cluster, "k8s cluster alias"); + validateLabel(namespace, "namespace"); + validateLabel(mateId, "mate ID"); + + return parseAgentAddress( + `/k8s/${cluster}/namespace/${namespace}/mate/${mateId}/herdr`, + ).address; +} diff --git a/tools/agent-os/src/cli.ts b/tools/agent-os/src/cli.ts new file mode 100755 index 000000000..5cd221ed4 --- /dev/null +++ b/tools/agent-os/src/cli.ts @@ -0,0 +1,85 @@ +#!/usr/bin/env bun + +import { parseArgs } from "node:util"; + +import { + formatKubernetesAgentAddress, + formatLocalAgentAddress, + parseAgentAddress, +} from "./address.ts"; + +const usage = `Usage: + agent-os address local + agent-os address k8s [--cluster ] [--namespace ] + agent-os address inspect `; + +function requireSingleArgument(args: string[]): string { + if (args.length !== 1 || args[0] == null) { + throw new Error(usage); + } + + return args[0]; +} + +function runAddressCommand(args: string[]): void { + const [command, ...rest] = args; + + switch (command) { + case "local": + console.log(formatLocalAgentAddress(requireSingleArgument(rest))); + return; + + case "k8s": { + const { positionals, values } = parseArgs({ + allowPositionals: true, + args: rest, + options: { + cluster: { + default: "in-cluster", + type: "string", + }, + namespace: { + default: "agent-os", + type: "string", + }, + }, + strict: true, + }); + const mateId = requireSingleArgument(positionals); + + console.log( + formatKubernetesAgentAddress(mateId, { + cluster: values.cluster, + namespace: values.namespace, + }), + ); + return; + } + + case "inspect": + console.log( + JSON.stringify(parseAgentAddress(requireSingleArgument(rest)), null, 2), + ); + return; + + default: + throw new Error(usage); + } +} + +function main(): void { + const [command, ...rest] = Bun.argv.slice(2); + + if (command !== "address") { + throw new Error(usage); + } + + runAddressCommand(rest); +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/tools/agent-os/test/address.test.ts b/tools/agent-os/test/address.test.ts new file mode 100644 index 000000000..4f5a7b4d7 --- /dev/null +++ b/tools/agent-os/test/address.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, test } from "bun:test"; + +import { + formatKubernetesAgentAddress, + formatLocalAgentAddress, + parseAgentAddress, +} from "../src/address.ts"; + +describe("Agent OS Multiaddrs", () => { + test("formats and parses a co-located Herdr mate", () => { + const address = formatLocalAgentAddress("task-x"); + + expect(address).toBe("/local/mate/task-x/herdr"); + expect(parseAgentAddress(address)).toEqual({ + address, + location: "local", + mateId: "task-x", + protocol: "herdr", + }); + }); + + test("defaults Kubernetes addresses to the caller's cluster", () => { + const address = formatKubernetesAgentAddress("task-x"); + + expect(address).toBe( + "/k8s/in-cluster/namespace/agent-os/mate/task-x/herdr", + ); + expect(parseAgentAddress(address)).toEqual({ + address, + cluster: "in-cluster", + location: "kubernetes", + mateId: "task-x", + namespace: "agent-os", + protocol: "herdr", + }); + }); + + test("accepts explicit registered cluster and namespace aliases", () => { + const address = formatKubernetesAgentAddress("review-7", { + cluster: "agents-eu", + namespace: "company-factory", + }); + + expect(address).toBe( + "/k8s/agents-eu/namespace/company-factory/mate/review-7/herdr", + ); + }); + + test.each([ + "/k8s/in-cluster/mate/task-x/herdr", + "/k8s/in-cluster/namespace/agent-os/herdr/mate/task-x", + "/local/namespace/agent-os/mate/task-x/herdr", + ])("rejects an unsupported component sequence: %s", (address) => { + expect(() => parseAgentAddress(address)).toThrow("Unsupported Agent OS address"); + }); + + test.each(["Task-X", "-task", "task/other", "--namespace"])( + "rejects an unsafe mate ID: %s", + (mateId) => { + expect(() => formatLocalAgentAddress(mateId)).toThrow("mate"); + }, + ); +}); diff --git a/tools/agent-os/test/cli.test.ts b/tools/agent-os/test/cli.test.ts new file mode 100644 index 000000000..1d6baf2f3 --- /dev/null +++ b/tools/agent-os/test/cli.test.ts @@ -0,0 +1,36 @@ +import { $ } from "bun"; +import { describe, expect, test } from "bun:test"; +import { fileURLToPath } from "node:url"; + +const cli = fileURLToPath(new URL("../src/cli.ts", import.meta.url)); + +describe("agent-os address CLI", () => { + test("prints a Kubernetes address with in-cluster defaults", async () => { + const output = await $`${process.execPath} ${cli} address k8s task-x`.text(); + + expect(output.trim()).toBe( + "/k8s/in-cluster/namespace/agent-os/mate/task-x/herdr", + ); + }); + + test("inspects an address as structured JSON", async () => { + const address = "/local/mate/task-x/herdr"; + const output = await $`${process.execPath} ${cli} address inspect ${address}`.json(); + + expect(output).toEqual({ + address, + location: "local", + mateId: "task-x", + protocol: "herdr", + }); + }); + + test("fails closed for an invalid address", async () => { + const result = await $`${process.execPath} ${cli} address inspect /mate/task-x` + .quiet() + .nothrow(); + + expect(result.exitCode).toBe(1); + expect(result.stderr.toString()).toContain("Unsupported Agent OS address"); + }); +}); diff --git a/tools/agent-os/tsconfig.json b/tools/agent-os/tsconfig.json new file mode 100644 index 000000000..ae04ceb65 --- /dev/null +++ b/tools/agent-os/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "allowImportingTsExtensions": true, + "forceConsistentCasingInFileNames": true, + "module": "Preserve", + "moduleDetection": "force", + "moduleResolution": "Bundler", + "noEmit": true, + "noUncheckedIndexedAccess": true, + "strict": true, + "target": "ESNext", + "types": ["bun"] + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} From 7afe7a0503782964b1f4394980efc7a186b86a1d Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 10:02:33 +0200 Subject: [PATCH 10/56] feat: add Akua mate packaging --- .agents/skills/kubernetes-fleet/SKILL.md | 4 +- .dockerignore | 3 +- Dockerfile | 41 +- THIRD_PARTY_NOTICES.md | 8 + bin/fm-bootstrap.sh | 10 +- docs/kubernetes.md | 12 +- tests/agent-os-container.test.sh | 14 +- tests/fm-bootstrap.test.sh | 18 +- tests/fm-secondmate-harness.test.sh | 2 +- tests/fm-secondmate-liveness.test.sh | 2 +- tests/fm-secondmate-sync.test.sh | 2 +- tests/fm-session-start.test.sh | 2 +- tests/fm-x-mode.test.sh | 2 +- .../.agents/skills/effect-ts/SKILL.md | 191 ++++ .../skills/effect-ts/references/features.md | 504 ++++++++ .../effect-ts/references/guide-effect.md | 447 ++++++++ .../references/guide-error-handling.md | 540 +++++++++ .../effect-ts/references/guide-layers.md | 1018 +++++++++++++++++ .../references/guide-observability.md | 724 ++++++++++++ .../effect-ts/references/guide-retries.md | 433 +++++++ .../effect-ts/references/guide-schedule.md | 379 ++++++ .../effect-ts/references/guide-schema.md | 624 ++++++++++ .../skills/effect-ts/references/guide-sql.md | 536 +++++++++ .../effect-ts/references/guide-testing.md | 504 ++++++++ .../skills/effect-ts/references/setup.md | 89 ++ tools/agent-os/.gitignore | 1 + tools/agent-os/AGENTS.md | 18 +- tools/agent-os/README.md | 5 + tools/agent-os/bun.lock | 42 + tools/agent-os/package.json | 5 +- tools/agent-os/packages/mate/README.md | 25 + tools/agent-os/packages/mate/akua.toml | 5 + .../packages/mate/inputs.example.yaml | 5 + tools/agent-os/packages/mate/package.k | 101 ++ tools/agent-os/scripts/prepare-effect.sh | 13 + tools/agent-os/skills-lock.json | 11 + tools/agent-os/src/cli.ts | 65 +- tools/agent-os/src/json-modules.d.ts | 4 + tools/agent-os/src/mate-package.ts | 151 +++ tools/agent-os/test/mate-package.test.ts | 81 ++ 40 files changed, 6617 insertions(+), 24 deletions(-) create mode 100644 tools/agent-os/.agents/skills/effect-ts/SKILL.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/features.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-effect.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-error-handling.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-layers.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-observability.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-retries.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-schedule.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-schema.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-sql.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/guide-testing.md create mode 100644 tools/agent-os/.agents/skills/effect-ts/references/setup.md create mode 100644 tools/agent-os/packages/mate/README.md create mode 100644 tools/agent-os/packages/mate/akua.toml create mode 100644 tools/agent-os/packages/mate/inputs.example.yaml create mode 100644 tools/agent-os/packages/mate/package.k create mode 100755 tools/agent-os/scripts/prepare-effect.sh create mode 100644 tools/agent-os/skills-lock.json create mode 100644 tools/agent-os/src/json-modules.d.ts create mode 100644 tools/agent-os/src/mate-package.ts create mode 100644 tools/agent-os/test/mate-package.test.ts diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index a43913e54..d39f7c1b1 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -14,6 +14,9 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. +- The image includes Akua's CLI for direct use and the Agent OS tool uses `@akua-dev/sdk` in process. +- Use `agent-os mate render ` when the prepared package is useful, then inspect or edit its ordinary YAML before applying it. +- The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. - Use `status` to inspect it and `delete` only after unique work is checkpointed or delivered. - Never mount the primary home into a child Pod. @@ -23,4 +26,3 @@ Load this skill only when the current firstmate is running in Kubernetes or is e Use ordinary `kubectl exec`, Herdr, files, and Git to supervise the child. Do not add a custom inter-agent chat protocol or Task/Run service. - diff --git a/.dockerignore b/.dockerignore index e1767260e..90b53692f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,4 +13,5 @@ projects state worktrees .worktrees - +node_modules +.repos diff --git a/Dockerfile b/Dockerfile index 0ae8012fd..2a1979ece 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:24-bookworm-slim +FROM node:24-trixie-slim ARG TARGETARCH ARG HERDR_VERSION=0.7.3 @@ -6,6 +6,8 @@ ARG KUBECTL_VERSION=1.34.8 ARG GH_VERSION=2.96.0 ARG TREEHOUSE_VERSION=2.0.0 ARG NO_MISTAKES_VERSION=1.34.0 +ARG BUN_VERSION=1.3.14 +ARG AKUA_VERSION=0.8.25 RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -20,6 +22,7 @@ RUN apt-get update \ ripgrep \ rsync \ tmux \ + unzip \ && ln -s /usr/bin/fdfind /usr/local/bin/fd \ && rm -rf /var/lib/apt/lists/* @@ -75,6 +78,34 @@ RUN set -eu; \ chmod 0755 /usr/local/bin/treehouse /usr/local/bin/no-mistakes; \ rm -f "/tmp/${treehouse_asset}" "/tmp/${no_mistakes_asset}" +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) bun_arch=x64; sha=951ee2aee855f08595aeec6225226a298d3fea83a3dcd6465c09cbccdf7e848f ;; \ + arm64) bun_arch=aarch64; sha=a27ffb63a8310375836e0d6f668ae17fa8d8d18b88c37c821c65331973a19a3b ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + asset="bun-linux-${bun_arch}.zip"; \ + curl -fsSL "https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/${asset}" -o "/tmp/${asset}"; \ + echo "$sha /tmp/${asset}" | sha256sum -c -; \ + unzip -q "/tmp/${asset}" -d /tmp/bun; \ + install -m 0755 "/tmp/bun/bun-linux-${bun_arch}/bun" /usr/local/bin/bun; \ + rm -rf "/tmp/${asset}" /tmp/bun + +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) triple=x86_64-unknown-linux-gnu; sha=bc57afbffe7e18aacd2146e2cd67151c56e7a3c279fe659312ff7ffb359cd03a ;; \ + arm64) triple=aarch64-unknown-linux-gnu; sha=3a3c6bae72764cbd85a6e4e0877a05e5def8f7aeee8563b7918099214a1a313a ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + asset="akua-v${AKUA_VERSION}-${triple}.tar.gz"; \ + curl -fsSL "https://github.com/akua-dev/akua/releases/download/v${AKUA_VERSION}/${asset}" -o "/tmp/${asset}"; \ + echo "$sha /tmp/${asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${asset}" -C /tmp; \ + install -m 0755 /tmp/akua /usr/local/bin/akua; \ + rm -f "/tmp/${asset}" /tmp/akua; \ + mkdir -p /usr/share/licenses/akua; \ + curl -fsSL "https://raw.githubusercontent.com/akua-dev/akua/v${AKUA_VERSION}/LICENSE" -o /usr/share/licenses/akua/LICENSE + RUN npm install --global \ @earendil-works/pi-coding-agent@0.80.6 \ gh-axi@0.1.27 \ @@ -94,10 +125,14 @@ ENV FM_HOME=/home/agent \ PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/bin \ HERDR_SESSION=default -RUN mkdir -p /home/agent /opt/agent-os /opt/image-usr-local \ - && cp -a /usr/local/. /opt/image-usr-local/ +RUN mkdir -p /home/agent /opt/agent-os /opt/image-usr-local COPY . /opt/agent-os +RUN cd /opt/agent-os/tools/agent-os \ + && bun install --frozen-lockfile --production --ignore-scripts \ + && ln -s /opt/agent-os/tools/agent-os/src/cli.ts /usr/local/bin/agent-os \ + && cp -a /usr/local/. /opt/image-usr-local/ + WORKDIR /opt/agent-os ENTRYPOINT ["/opt/agent-os/bin/agent-os-container-entrypoint.sh"] diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 86fa4f8dc..0049de365 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -7,3 +7,11 @@ Herdr is available under AGPL-3.0-or-later or a commercial license. The image includes Herdr's license at `/usr/share/licenses/herdr/LICENSE`. The exact corresponding source is available at . Agent OS invokes Herdr through its documented CLI and socket interfaces. + +## Akua + +The Agent OS demo image includes an unmodified Akua 0.8.25 executable as a separate program. +Akua is available under the Apache License 2.0. +The image includes Akua's license at `/usr/share/licenses/akua/LICENSE`. +The exact source is available at . +Agent OS invokes Akua through its documented CLI. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 7a2b2a9f9..d46478b6b 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -43,7 +43,10 @@ # "treehouse get --lease" support. # no-mistakes is also MISSING when its installed version is older than # 1.31.2. -# tasks-axi and quota-axi are required bootstrap tools (same class as +# Akua, tasks-axi, and quota-axi are required bootstrap tools. +# Akua renders the optional Agent OS mate package into ordinary YAML; +# firstmate may also author and apply Kubernetes YAML directly. +# tasks-axi and quota-axi are in the same class as # lavish-axi). tasks-axi is also version and feature gated (0.1.1+ # with update --archive-body and mv [...]); an installed but incompatible build # reports MISSING like no-mistakes. When @@ -314,6 +317,7 @@ install_cmd() { tmux|node|git|gh|curl|jq|orca) echo "brew install $1 # or the platform's package manager" ;; treehouse) echo "curl -fsSL https://kunchenguid.github.io/treehouse/install.sh | sh" ;; no-mistakes) echo "curl -fsSL https://raw.githubusercontent.com/kunchenguid/no-mistakes/main/docs/install.sh | sh" ;; + akua) echo "curl -fsSL https://cli.akua.dev/install | sh" ;; gh-axi|chrome-devtools-axi|lavish-axi) echo "npm install -g $1 && $1 setup hooks" ;; tasks-axi|quota-axi) echo "npm install -g $1" ;; *) return 1 ;; @@ -322,8 +326,8 @@ install_cmd() { BACKEND=$(fm_backend_name) case "$BACKEND" in - orca) TOOLS="orca node git gh no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; - *) TOOLS="tmux node git gh treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; + orca) TOOLS="orca node git gh akua no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; + *) TOOLS="tmux node git gh akua treehouse no-mistakes gh-axi chrome-devtools-axi lavish-axi tasks-axi quota-axi" ;; esac NO_MISTAKES_MIN_MAJOR=1 NO_MISTAKES_MIN_MINOR=31 diff --git a/docs/kubernetes.md b/docs/kubernetes.md index bfde97b14..553bba78e 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -35,7 +35,7 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` -The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, treehouse, no-mistakes, and every required AXI CLI. +The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, treehouse, no-mistakes, and every required AXI CLI. Authenticate GitHub inside the primary with `gh auth login`. Authenticate Pi using `/login` and the provider flow you choose. Host credentials are intentionally excluded from the image and are never copied automatically. @@ -64,6 +64,16 @@ bin/agent-os-crewmate.sh status scout-1 bin/agent-os-crewmate.sh delete scout-1 ``` +It can also render the prepared Akua package without spawning a subprocess from the TypeScript tool: + +```sh +agent-os mate render scout-1 --namespace agent-os-demo --image agent-os:dev --out /tmp/scout-1 +kubectl apply -f /tmp/scout-1 +``` + +This command uses `@akua-dev/sdk` and only writes ordinary manifests. +Firstmate can inspect or edit those files, invoke the bundled `akua` CLI directly, modify the package, or author equivalent YAML without the helper. + When run on a host, the crewmate helper requires an explicit context: ```sh diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 4663959dd..1ceefb506 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -5,12 +5,15 @@ set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -assert_grep 'FROM node:24-bookworm-slim' "$ROOT/Dockerfile" "image must pin Node 24 Bookworm" +assert_grep 'FROM node:24-trixie-slim' "$ROOT/Dockerfile" "image must pin Node 24 Trixie" assert_grep 'ARG HERDR_VERSION=0.7.3' "$ROOT/Dockerfile" "image must pin Herdr 0.7.3" assert_grep 'ARG KUBECTL_VERSION=1.34.8' "$ROOT/Dockerfile" "image must pin kubectl 1.34.8" assert_grep 'ARG GH_VERSION=2.96.0' "$ROOT/Dockerfile" "image must pin GitHub CLI 2.96.0" assert_grep 'ARG TREEHOUSE_VERSION=2.0.0' "$ROOT/Dockerfile" "image must pin treehouse 2.0.0" assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin no-mistakes 1.34.0" +assert_grep 'ARG BUN_VERSION=1.3.14' "$ROOT/Dockerfile" "image must pin stable Bun 1.3.14" +assert_grep 'ARG AKUA_VERSION=0.8.25' "$ROOT/Dockerfile" "image must pin Akua 0.8.25" +assert_grep '"@akua-dev/sdk": "0.8.24"' "$ROOT/tools/agent-os/package.json" "tool must pin the Akua SDK" assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" assert_grep 'gh-axi@0.1.27' "$ROOT/Dockerfile" "image must pin gh-axi 0.1.27" @@ -26,14 +29,23 @@ assert_grep 'NPM_CONFIG_PREFIX=/usr/local' "$ROOT/Dockerfile" "global npm instal assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin' "$ROOT/Dockerfile" \ "persistent tool prefixes must lead PATH" assert_grep '/opt/image-usr-local' "$ROOT/Dockerfile" "image must retain a seed copy of /usr/local" +assert_grep 'akua-dev/akua/releases/download/v${AKUA_VERSION}' "$ROOT/Dockerfile" "image must install Akua from its release" +assert_grep 'ln -s /opt/agent-os/tools/agent-os/src/cli.ts /usr/local/bin/agent-os' "$ROOT/Dockerfile" \ + "image must expose the Agent OS tool" +assert_grep 'bun install --frozen-lockfile --production --ignore-scripts' "$ROOT/Dockerfile" \ + "image install must not clone development source checkouts" assert_no_grep 'USER node' "$ROOT/Dockerfile" "Agent OS containers must start as container root" assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" assert_grep 'setup hooks' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must install persistent AXI hooks" assert_grep '.git' "$ROOT/.dockerignore" "git metadata must stay out of the build context" assert_grep '.pi' "$ROOT/.dockerignore" "Pi credentials must stay out of the build context" assert_grep '.codex' "$ROOT/.dockerignore" "Codex credentials must stay out of the build context" +assert_grep 'node_modules' "$ROOT/.dockerignore" "host dependencies must stay out of the build context" +assert_grep '.repos' "$ROOT/.dockerignore" "development source checkouts must stay out of the build context" assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PARTY_NOTICES.md" \ "Herdr's exact corresponding source must be named" +assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "Akua's exact source must be named" bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" pass "container files pin dependencies and exclude host credentials" diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index f16314e95..83204e381 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -27,7 +27,7 @@ TMP_ROOT=$(fm_test_tmproot fm-bootstrap-tests) make_fake_toolchain() { local dir=$1 fakebin fakebin=$(fm_fakebin "$dir") - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash if [ "${1:-}" = auth ] && [ "${2:-}" = status ]; then @@ -327,6 +327,21 @@ SH pass "bootstrap requires git with an install instruction" } +test_akua_is_required_with_supported_install_instruction() { + local case_dir fakebin out expected + case_dir="$TMP_ROOT/akua-required" + mkdir -p "$case_dir/home/config" + printf '%s\n' manual > "$case_dir/home/config/backlog-backend" + fakebin=$(make_fake_toolchain "$case_dir") + rm -f "$fakebin/akua" + + out=$(PATH="$fakebin:$BASE_PATH" FM_HOME="$case_dir/home" FM_ROOT_OVERRIDE="$case_dir/home" \ + FM_FAKE_TREEHOUSE_LEASE_HELP=1 "$ROOT/bin/fm-bootstrap.sh") + expected="MISSING: akua (install: curl -fsSL https://cli.akua.dev/install | sh)" + [ "$out" = "$expected" ] || fail "missing Akua should report the supported install instruction, got: $out" + pass "bootstrap requires Akua with an install instruction" +} + test_orca_backend_gates_orca_tool_only_when_selected() { local case_dir fakebin out missing_orca missing_orca="MISSING: orca (install: brew install orca # or the platform's package manager)" @@ -495,6 +510,7 @@ ROWS test_bootstrap_reporting test_no_mistakes_min_version test_git_is_required_with_supported_install_instruction +test_akua_is_required_with_supported_install_instruction test_orca_backend_gates_orca_tool_only_when_selected test_fleet_sync_timeout_scales_with_origin_backed_project_count test_fleet_sync_timeout_floor_preserves_small_fleets diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index 1a314c050..71b22ae11 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -720,7 +720,7 @@ make_fake_toolchain() { local dir=$1 fakebin fakebin="$dir/fakebin" mkdir -p "$fakebin" - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-secondmate-liveness.test.sh b/tests/fm-secondmate-liveness.test.sh index f639e836e..b7308e28c 100755 --- a/tests/fm-secondmate-liveness.test.sh +++ b/tests/fm-secondmate-liveness.test.sh @@ -169,7 +169,7 @@ test_agent_alive_dispatcher_routes_and_falls_back() { make_toolchain() { local dir=$1 fakebin fakebin=$(fm_fakebin "$dir") - fm_fake_exit0 "$fakebin" node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-secondmate-sync.test.sh b/tests/fm-secondmate-sync.test.sh index bc806523d..80c30c4da 100755 --- a/tests/fm-secondmate-sync.test.sh +++ b/tests/fm-secondmate-sync.test.sh @@ -289,7 +289,7 @@ make_fake_toolchain() { local dir=$1 fakebin fakebin="$dir/fakebin" mkdir -p "$fakebin" - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-session-start.test.sh b/tests/fm-session-start.test.sh index 4e1d68eff..c7b3f953a 100755 --- a/tests/fm-session-start.test.sh +++ b/tests/fm-session-start.test.sh @@ -53,7 +53,7 @@ new_world() { # test deliberately breaks one. Mirrors fm-bootstrap.test.sh's fixture. make_fake_toolchain() { local fakebin=$1 - fm_fake_exit0 "$fakebin" tmux node gh-axi chrome-devtools-axi lavish-axi + fm_fake_exit0 "$fakebin" tmux node akua gh-axi chrome-devtools-axi lavish-axi cat > "$fakebin/gh" <<'SH' #!/usr/bin/env bash exit 0 diff --git a/tests/fm-x-mode.test.sh b/tests/fm-x-mode.test.sh index 89b71187c..9b45d9870 100755 --- a/tests/fm-x-mode.test.sh +++ b/tests/fm-x-mode.test.sh @@ -442,7 +442,7 @@ test_bootstrap_reports_missing_x_dependency() { local home fakebin out tool tool_path home="$TMP_ROOT/boot-missing-x"; mkdir -p "$home" fakebin=$(fm_fakebin "$home") - fm_fake_exit0 "$fakebin" tmux node no-mistakes gh-axi chrome-devtools-axi lavish-axi curl + fm_fake_exit0 "$fakebin" tmux node akua no-mistakes gh-axi chrome-devtools-axi lavish-axi curl for tool in dirname grep tail; do tool_path=$(command -v "$tool") || fail "test host must provide $tool" ln -s "$tool_path" "$fakebin/$tool" diff --git a/tools/agent-os/.agents/skills/effect-ts/SKILL.md b/tools/agent-os/.agents/skills/effect-ts/SKILL.md new file mode 100644 index 000000000..62a95f239 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/SKILL.md @@ -0,0 +1,191 @@ +--- +name: effect-ts +description: Use this skill whenever working in a repository that uses Effect, even if the current task is in a new file or the user does not explicitly ask for Effect help. Apply it to any work that should follow the repository's Effect patterns, conventions, architecture, or supporting tooling. Also use it for questions about Effect patterns, services, layers, schemas, streams, runtimes, or typed error handling. +--- + +# Effect Expert + +Expert guidance for programming with the Effect library, covering error handling, dependency injection, composability, and testing patterns. + +## Prerequisite + +Before doing any other Effect-related work, check that `./.repos/effect` exists at the root of the repository where the skill is being used. + +If it does not exist, stop and prompt the user with the setup task documented in `./references/setup.md`. + +## Research Strategy + +Effect has many ways to accomplish the same task. Proactively research best practices when working with Effect patterns, especially for moderate to high complexity tasks. + +Use the local guides in `./references/` first. They are the preferred source for best practices, conventions, and common implementation patterns. + +Only go directly to the vendored Effect repo when: + +- the guides do not cover the question +- you need exact API details or signatures +- you need deeper implementation details +- you need to verify a behavior against the source + +### Research Sources + +1. Local skill guides first. Start with the relevant files in `./references/` before doing deeper research. +2. Codebase patterns second. Examine similar patterns in the current project before implementing. If Effect patterns already exist, follow them for consistency. If no patterns exist, skip this step. +3. Effect source code last. For gaps in the guides, complex type errors, unclear behavior, or implementation details, examine the vendored Effect source at `./.repos/effect/packages/effect/src/`. + +### When To Research + +- Always research for services, layers, or complex dependency injection. +- Always research for error handling with multiple error types or complex error hierarchies. +- Always research for stream-based operations and reactive patterns. +- Always research for resource management with scoped effects and cleanup. +- Always research for concurrent or performance-critical code. +- Always research for unfamiliar testing patterns. +- Research when needed for complex refactors from promises or try/catch into Effect. +- Research when needed for new service dependencies or layer restructuring. +- Research when needed for custom error types or extensions of existing error hierarchies. +- Research when needed for integrations with external systems such as databases, APIs, or third-party services. + +### Research Approach + +- Focus on canonical, readable, and maintainable solutions rather than clever optimizations. +- Verify suggested approaches against existing codebase patterns when those patterns exist. +- When multiple approaches are possible, prefer the most idiomatic Effect solution supported by the codebase and the vendored source. + +### Codebase Pattern Discovery + +When working in a project that uses Effect, check for existing patterns before implementing new code: + +1. Search for Effect imports and existing module usage to understand current conventions. +2. Identify how services and layers are structured in the project. +3. Note how errors are defined and propagated. +4. Examine how Effect code is tested in the project. + +If no Effect patterns exist in the codebase, proceed using canonical patterns from the vendored Effect source and examples. Do not block on missing codebase patterns. + +### Feature Discovery + +When you need to discover available Effect modules, packages, or capabilities, search `./references/features.md` first. + +- Use it to identify the right package or module for a task. +- Use the listed repo paths to jump directly into the vendored source under `./.repos/effect`. +- Use it before inventing custom abstractions when Effect may already provide the functionality. + +### Guide Discovery + +When the task touches one of these areas, consult the matching guide before implementing: + +- `./references/guide-effect.md` for core `Effect` usage patterns, common constructors, composition, provisioning, and runtime boundaries +- `./references/guide-error-handling.md` for defining errors, schema-based errors, failure handling, defects, and interrupts +- `./references/guide-layers.md` for services, layer construction, composition, and provisioning patterns +- `./references/guide-observability.md` for `Effect.fn`, spans, logging, metrics, and telemetry wiring +- `./references/guide-retries.md` for retry policies, retry conditions, fallback strategies, and `ExecutionPlan` +- `./references/guide-schedule.md` for retries, repeats, backoff, polling, cron, and schedule composition +- `./references/guide-schema.md` for schema design, transformations, unions, recursion, opaque/branded types, and schema best practices +- `./references/guide-sql.md` for Effect SQL usage, transactions, resolvers, schema-aware SQL, and migrations +- `./references/guide-testing.md` for detailed `@effect/vitest` usage, layered test setup, property tests, and test services + +These guides should be treated as the default implementation guidance. Do not skip them and jump straight to `./.repos/effect` unless you need source-level confirmation or the guides do not answer the question. + +## Effect Principles + +Apply these core principles when writing Effect code. + +## Installation + +When installing Effect packages in a user repository: + +- use `effect@beta` +- keep all `@effect/*` packages on aligned versions +- install only the packages needed for the user's runtime and actual task + +### Version Rules + +- `effect` should be installed as `effect@beta` +- if you install any `@effect/*` package, make sure all `@effect/*` packages use matching versions +- do not mix unrelated `@effect/*` versions in the same project + +### Package Selection + +Choose packages based on the runtime and the work being done. + +- core library: `effect@beta` +- Node.js runtime needs: install the matching `@effect/platform-node` +- browser runtime needs: install the matching `@effect/platform-browser` +- Bun runtime needs: install the matching `@effect/platform-bun` +- Vitest integration needs: install the matching `@effect/vitest` +- OpenTelemetry integration needs: install the matching `@effect/opentelemetry` + +Install additional `@effect/*` packages only when the user task actually needs them. + +### Practical Rule + +- start with `effect@beta` +- add `@effect/*` packages as needed by runtime and features +- keep the full installed Effect package set version-aligned + +### Error Handling + +- Use Effect's typed error system instead of throwing exceptions. +- Define descriptive error types with proper error propagation. +- Prefer `Schema.TaggedErrorClass` when the error can be schema-defined. +- Use `Effect.fail`, `Effect.catchTag`, and `Effect.catch` for error control flow. + +### Dependency Injection + +- Implement dependency injection using services and layers. +- Define services with `Context.Tag`. +- Compose layers with `Layer.merge` and `Layer.provide`. +- Use `Effect.provide` to inject dependencies at the edge, avoid providing locally. +- Keep services encapsulated; avoid exporting trivial accessor wrappers that only forward to one service method. + +### Composability + +- Leverage Effect composability for complex operations. +- Use appropriate constructors such as `Effect.succeed`, `Effect.fail`, `Effect.tryPromise`, `Effect.try`, and `Effect.sync`. +- Apply proper resource management with scoped effects. +- Chain operations with `Effect.flatMap`, `Effect.map`, and `Effect.tap`. + +### Business Logic Functions + +- Prefer `Effect.fn` for reusable business-logic functions that return `Effect`. +- Prefer `Effect.fn` over raw `Effect.gen` definitions even when the function takes no arguments. +- If you do not want an explicit named span, use `Effect.fn` without a span name. +- Do not use `Effect.fnUntraced` as the default. +- Use `Effect.fnUntraced` only for edge cases with a concrete low-level reason, such as measured hot-path overhead. + +### TypeScript Preferences + +- Never use `any`. +- Never use `as` casts. +- Never use unsafe type assertions or escape hatches. +- Never use `namespace`. +- Prefer correct typing, schema-driven decoding, narrowing, and proper generic constraints instead of forcing types. +- If a value comes from an external boundary, validate or decode it instead of asserting its type. +- If a type is hard to express, simplify the design or introduce a properly typed helper instead of using unsafe TypeScript. +- For layers, do not hide them inside `namespace` blocks. Prefer either `static` members on the service class or plain exported layer constants. + +### Code Quality + +- Write type-safe code that leverages Effect's type system. +- Use `Effect.gen` for readable sequential code. +- Implement proper testing patterns using Effect testing utilities. +- Prefer existing Effect primitives before introducing custom helpers. +- Prefer `Schema.Class` / `Schema.TaggedClass` variants over plain `Schema.Struct` for named reusable schemas when possible. + +### Explaining Solutions + +When providing solutions, explain the Effect concepts being used and why they fit the specific use case. If you encounter patterns not covered in local references, prefer consistency with the codebase when possible and otherwise rely on the vendored Effect source. + +## References + +- `./references/features.md` +- `./references/guide-effect.md` +- `./references/guide-error-handling.md` +- `./references/guide-layers.md` +- `./references/guide-observability.md` +- `./references/guide-retries.md` +- `./references/guide-schedule.md` +- `./references/guide-schema.md` +- `./references/guide-sql.md` +- `./references/guide-testing.md` +- `./references/setup.md` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/features.md b/tools/agent-os/.agents/skills/effect-ts/references/features.md new file mode 100644 index 000000000..a605d3eab --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/features.md @@ -0,0 +1,504 @@ +# Features + +Public package and module surface area to be aware of when researching or implementing a solution. Each module entry includes its vendored repo path so it can be located quickly. + +## `effect` Package + +Package path: `packages/effect` + +- `Array` - `packages/effect/src/Array.ts` - Immutable array utilities. Use: transform arrays immutably. +- `BigDecimal` - `packages/effect/src/BigDecimal.ts` - Arbitrary-precision decimal arithmetic. Use: avoid floating-point errors. +- `BigInt` - `packages/effect/src/BigInt.ts` - `bigint` utilities and instances. Use: work with large integers. +- `Boolean` - `packages/effect/src/Boolean.ts` - Boolean utilities and instances. Use: compose boolean logic. +- `Brand` - `packages/effect/src/Brand.ts` - Branded type helpers. Use: prevent accidental type mixing. +- `Cache` - `packages/effect/src/Cache.ts` - Effect cache utilities. Use: memoize effectful lookups. +- `Cause` - `packages/effect/src/Cause.ts` - Structured effect failure representation. Use: inspect failures and defects. +- `Channel` - `packages/effect/src/Channel.ts` - Bidirectional streaming I/O abstraction. Use: build stream pipelines. +- `ChannelSchema` - `packages/effect/src/ChannelSchema.ts` - Schema helpers for channels. Use: type channel protocols. +- `Chunk` - `packages/effect/src/Chunk.ts` - Immutable high-performance sequence. Use: efficient functional sequences. +- `Clock` - `packages/effect/src/Clock.ts` - Time service and sleep utilities. Use: schedule and measure time. +- `Combiner` - `packages/effect/src/Combiner.ts` - Combine two values of one type. Use: merge values consistently. +- `Config` - `packages/effect/src/Config.ts` - Schema-driven configuration loading. Use: read typed app config. +- `ConfigProvider` - `packages/effect/src/ConfigProvider.ts` - Configuration data source abstraction. Use: load config from env/files. +- `Console` - `packages/effect/src/Console.ts` - Effectful console operations. Use: log and debug safely. +- `Context` - `packages/effect/src/Context.ts` - Dependency injection context table. Use: provide and access services. +- `Cron` - `packages/effect/src/Cron.ts` - Cron scheduling utilities. Use: express recurring schedules. +- `Data` - `packages/effect/src/Data.ts` - Immutable data and tagged unions. Use: model domain data and errors. +- `DateTime` - `packages/effect/src/DateTime.ts` - Date-time utilities. Use: handle timestamps and calendars. +- `Deferred` - `packages/effect/src/Deferred.ts` - Single-assignment async variable. Use: coordinate concurrent fibers. +- `Differ` - `packages/effect/src/Differ.ts` - Value diffing utilities. Use: compute incremental updates. +- `Duration` - `packages/effect/src/Duration.ts` - Precise time span type. Use: represent delays and intervals. +- `Effect` - `packages/effect/src/Effect.ts` - Core effect type and operators. Use: model async and concurrent work. +- `Effectable` - `packages/effect/src/Effectable.ts` - Protocols for effect-like values. Use: integrate custom yieldables. +- `Encoding` - `packages/effect/src/Encoding.ts` - Base64, Base64Url, and Hex codecs. Use: encode binary/text values. +- `Equal` - `packages/effect/src/Equal.ts` - Structural and custom equality. Use: compare immutable values deeply. +- `Equivalence` - `packages/effect/src/Equivalence.ts` - Equivalence relation helpers. Use: dedupe or compare with custom rules. +- `ErrorReporter` - `packages/effect/src/ErrorReporter.ts` - Pluggable error reporting. Use: forward failures to observability tools. +- `ExecutionPlan` - `packages/effect/src/ExecutionPlan.ts` - Execution plan utilities. Use: inspect planned work. +- `Exit` - `packages/effect/src/Exit.ts` - Synchronous effect outcome value. Use: inspect success or failure. +- `Fiber` - `packages/effect/src/Fiber.ts` - Lightweight concurrency primitive. Use: fork and join work. +- `FiberHandle` - `packages/effect/src/FiberHandle.ts` - Managed fiber handle helpers. Use: control child fibers. +- `FiberMap` - `packages/effect/src/FiberMap.ts` - Fiber map utilities. Use: track fibers by key. +- `FiberSet` - `packages/effect/src/FiberSet.ts` - Fiber set utilities. Use: manage groups of fibers. +- `FileSystem` - `packages/effect/src/FileSystem.ts` - Effectful file system abstraction. Use: read and write files safely. +- `Filter` - `packages/effect/src/Filter.ts` - Filter result utilities. Use: compose filtering operations. +- `Formatter` - `packages/effect/src/Formatter.ts` - Human-readable value formatting. Use: pretty-print data for logs. +- `Function` - `packages/effect/src/Function.ts` - Core function helpers. Use: pipe and compose functions. +- `Graph` - `packages/effect/src/Graph.ts` - Graph utilities. Use: model dependency graphs. +- `Hash` - `packages/effect/src/Hash.ts` - Hashing utilities. Use: hash values for collections. +- `HashMap` - `packages/effect/src/HashMap.ts` - Immutable hash map. Use: keyed immutable storage. +- `HashRing` - `packages/effect/src/HashRing.ts` - Consistent hashing utilities. Use: distribute keys across nodes. +- `HashSet` - `packages/effect/src/HashSet.ts` - Immutable hash set. Use: store unique immutable values. +- `HKT` - `packages/effect/src/HKT.ts` - Higher-kinded type utilities. Use: define generic type classes. +- `Inspectable` - `packages/effect/src/Inspectable.ts` - Custom inspection helpers. Use: improve debugging output. +- `Iterable` - `packages/effect/src/Iterable.ts` - Functional iterable utilities. Use: process lazy iterables. +- `JsonPatch` - `packages/effect/src/JsonPatch.ts` - JSON Patch operations. Use: diff and patch JSON documents. +- `JsonPointer` - `packages/effect/src/JsonPointer.ts` - JSON Pointer token helpers. Use: escape pointer path segments. +- `JsonSchema` - `packages/effect/src/JsonSchema.ts` - JSON Schema dialect conversion. Use: convert schemas between formats. +- `Latch` - `packages/effect/src/Latch.ts` - Latch synchronization primitive. Use: gate concurrent progress. +- `Layer` - `packages/effect/src/Layer.ts` - Service construction recipes. Use: build and wire dependencies. +- `LayerMap` - `packages/effect/src/LayerMap.ts` - Layer registry helpers. Use: share and look up layers. +- `Logger` - `packages/effect/src/Logger.ts` - Structured logging system. Use: emit runtime logs. +- `LogLevel` - `packages/effect/src/LogLevel.ts` - Log level utilities. Use: filter logs by severity. +- `ManagedRuntime` - `packages/effect/src/ManagedRuntime.ts` - Managed runtime helpers. Use: own runtime lifecycle. +- `Match` - `packages/effect/src/Match.ts` - Type-safe pattern matching. Use: replace switch-heavy branching. +- `Metric` - `packages/effect/src/Metric.ts` - Application metrics system. Use: count, gauge, and histogram events. +- `MutableHashMap` - `packages/effect/src/MutableHashMap.ts` - Mutable hash map. Use: high-performance mutable key-value storage. +- `MutableHashSet` - `packages/effect/src/MutableHashSet.ts` - Mutable hash set. Use: high-performance mutable uniqueness checks. +- `MutableList` - `packages/effect/src/MutableList.ts` - Mutable linked-list-like buffer. Use: efficient append/prepend workloads. +- `MutableRef` - `packages/effect/src/MutableRef.ts` - Mutable reference container. Use: hold local mutable state. +- `Newtype` - `packages/effect/src/Newtype.ts` - Zero-cost wrapper types. Use: distinguish same-shaped values. +- `NonEmptyIterable` - `packages/effect/src/NonEmptyIterable.ts` - Non-empty iterable helpers. Use: require at least one element. +- `Number` - `packages/effect/src/Number.ts` - Number utilities and instances. Use: do typed numeric operations. +- `Optic` - `packages/effect/src/Optic.ts` - Immutable data accessors and updaters. Use: read or update nested state. +- `Option` - `packages/effect/src/Option.ts` - Optional value type. Use: model absence safely. +- `Order` - `packages/effect/src/Order.ts` - Total ordering type class. Use: sort and compare values. +- `Ordering` - `packages/effect/src/Ordering.ts` - Comparison result helpers. Use: combine sort outcomes. +- `PartitionedSemaphore` - `packages/effect/src/PartitionedSemaphore.ts` - Partition-aware semaphore. Use: limit concurrency by key. +- `Path` - `packages/effect/src/Path.ts` - Path utilities. Use: work with filesystem-style paths. +- `Pipeable` - `packages/effect/src/Pipeable.ts` - Pipeable protocol helpers. Use: support fluent pipelines. +- `PlatformError` - `packages/effect/src/PlatformError.ts` - Platform error types. Use: normalize platform-specific failures. +- `Pool` - `packages/effect/src/Pool.ts` - Resource pool utilities. Use: reuse scarce resources. +- `Predicate` - `packages/effect/src/Predicate.ts` - Predicate and refinement helpers. Use: filter and narrow values. +- `PrimaryKey` - `packages/effect/src/PrimaryKey.ts` - Primary key helpers. Use: define stable string identifiers. +- `PubSub` - `packages/effect/src/PubSub.ts` - Publish-subscribe hub. Use: broadcast messages to subscribers. +- `Pull` - `packages/effect/src/Pull.ts` - Pull protocol helpers. Use: model pull-based streaming. +- `Queue` - `packages/effect/src/Queue.ts` - Effect queue utilities. Use: buffer producer-consumer work. +- `Random` - `packages/effect/src/Random.ts` - Randomness service. Use: generate testable random values. +- `RcMap` - `packages/effect/src/RcMap.ts` - Reference-counted map. Use: share keyed resources. +- `RcRef` - `packages/effect/src/RcRef.ts` - Reference-counted ref. Use: share scoped resources. +- `Record` - `packages/effect/src/Record.ts` - Record utilities. Use: transform string-keyed objects. +- `Redactable` - `packages/effect/src/Redactable.ts` - Context-aware redaction protocol. Use: mask sensitive values. +- `Redacted` - `packages/effect/src/Redacted.ts` - Sensitive value wrapper. Use: keep secrets out of logs. +- `Reducer` - `packages/effect/src/Reducer.ts` - Fold values into one result. Use: aggregate collections. +- `Ref` - `packages/effect/src/Ref.ts` - Atomic mutable reference. Use: manage concurrent state. +- `References` - `packages/effect/src/References.ts` - Runtime reference services. Use: tune runtime configuration. +- `RegExp` - `packages/effect/src/RegExp.ts` - RegExp utilities. Use: work with regular expressions. +- `Request` - `packages/effect/src/Request.ts` - External request description. Use: batch and cache data fetching. +- `RequestResolver` - `packages/effect/src/RequestResolver.ts` - Request execution abstraction. Use: resolve batched requests. +- `Resource` - `packages/effect/src/Resource.ts` - Resource utilities. Use: acquire and release shared resources. +- `Result` - `packages/effect/src/Result.ts` - Synchronous success/failure type. Use: validate without effects. +- `Runtime` - `packages/effect/src/Runtime.ts` - Effect runtime helpers. Use: run main programs. +- `Schedule` - `packages/effect/src/Schedule.ts` - Retry and repetition schedules. Use: control retry timing. +- `Scheduler` - `packages/effect/src/Scheduler.ts` - Scheduler utilities. Use: control task execution. +- `Schema` - `packages/effect/src/Schema.ts` - Data schema, validation, and codecs. Use: decode and encode typed data. +- `SchemaAST` - `packages/effect/src/SchemaAST.ts` - Runtime schema AST. Use: inspect or transform schemas. +- `SchemaGetter` - `packages/effect/src/SchemaGetter.ts` - One-way schema transformations. Use: decode individual fields. +- `SchemaIssue` - `packages/effect/src/SchemaIssue.ts` - Structured schema validation errors. Use: inspect parse failures. +- `SchemaParser` - `packages/effect/src/SchemaParser.ts` - Schema parsing helpers. Use: build parsing workflows. +- `SchemaRepresentation` - `packages/effect/src/SchemaRepresentation.ts` - Serializable schema IR. Use: round-trip schemas through JSON/codegen. +- `SchemaTransformation` - `packages/effect/src/SchemaTransformation.ts` - Bidirectional schema transformations. Use: map encoded and decoded forms. +- `SchemaUtils` - `packages/effect/src/SchemaUtils.ts` - Schema utility helpers. Use: support schema internals. +- `Scope` - `packages/effect/src/Scope.ts` - Resource lifetime scope. Use: ensure cleanup of acquired resources. +- `ScopedCache` - `packages/effect/src/ScopedCache.ts` - Scoped cache utilities. Use: cache scoped resources. +- `ScopedRef` - `packages/effect/src/ScopedRef.ts` - Scoped mutable reference. Use: swap scoped resources safely. +- `Semaphore` - `packages/effect/src/Semaphore.ts` - Concurrency semaphore. Use: limit parallel access. +- `Sink` - `packages/effect/src/Sink.ts` - Stream consumer abstraction. Use: fold or write stream inputs. +- `Stdio` - `packages/effect/src/Stdio.ts` - Standard I/O utilities. Use: access stdin/stdout/stderr. +- `Stream` - `packages/effect/src/Stream.ts` - Functional stream abstraction. Use: process streaming data. +- `String` - `packages/effect/src/String.ts` - String utilities and instances. Use: manipulate text functionally. +- `Struct` - `packages/effect/src/Struct.ts` - Immutable object utilities. Use: transform plain objects. +- `SubscriptionRef` - `packages/effect/src/SubscriptionRef.ts` - Subscribable reference. Use: observe state changes. +- `Symbol` - `packages/effect/src/Symbol.ts` - Symbol utilities. Use: work with JavaScript symbols. +- `SynchronizedRef` - `packages/effect/src/SynchronizedRef.ts` - Effect-synchronized ref. Use: update state with effects. +- `Take` - `packages/effect/src/Take.ts` - Stream take representation. Use: carry chunk, end, or error. +- `Terminal` - `packages/effect/src/Terminal.ts` - Terminal interaction utilities. Use: build CLI prompts/output. +- `Tracer` - `packages/effect/src/Tracer.ts` - Tracing abstractions. Use: create spans and traces. +- `Trie` - `packages/effect/src/Trie.ts` - Prefix tree for strings. Use: do prefix lookups. +- `Tuple` - `packages/effect/src/Tuple.ts` - Immutable tuple utilities. Use: transform fixed-length arrays. +- `TxChunk` - `packages/effect/src/TxChunk.ts` - Transactional chunk. Use: mutate chunk state in STM. +- `TxDeferred` - `packages/effect/src/TxDeferred.ts` - Transactional deferred cell. Use: coordinate STM completion. +- `TxHashMap` - `packages/effect/src/TxHashMap.ts` - Transactional hash map. Use: keyed STM state. +- `TxHashSet` - `packages/effect/src/TxHashSet.ts` - Transactional hash set. Use: unique STM state. +- `TxPriorityQueue` - `packages/effect/src/TxPriorityQueue.ts` - Transactional priority queue. Use: ordered STM queueing. +- `TxPubSub` - `packages/effect/src/TxPubSub.ts` - Transactional pub-sub hub. Use: broadcast within STM. +- `TxQueue` - `packages/effect/src/TxQueue.ts` - Transactional queue. Use: queue data within STM. +- `TxReentrantLock` - `packages/effect/src/TxReentrantLock.ts` - Transactional reentrant RW lock. Use: coordinate STM access. +- `TxRef` - `packages/effect/src/TxRef.ts` - Transactional reference. Use: read and write STM state. +- `TxSemaphore` - `packages/effect/src/TxSemaphore.ts` - Transactional semaphore. Use: limit STM concurrency. +- `TxSubscriptionRef` - `packages/effect/src/TxSubscriptionRef.ts` - Transactional subscribable ref. Use: observe committed STM changes. +- `Types` - `packages/effect/src/Types.ts` - Type-level utility aliases. Use: shape complex TypeScript types. +- `UndefinedOr` - `packages/effect/src/UndefinedOr.ts` - Helpers for `A | undefined`. Use: model lightweight optional values. +- `Unify` - `packages/effect/src/Unify.ts` - Type unification helpers. Use: simplify inferred types. +- `Utils` - `packages/effect/src/Utils.ts` - Generator and HKT internals. Use: support yieldable APIs. + +### `effect/testing` + +- `FastCheck` - `packages/effect/src/testing/FastCheck.ts` - Re-export of fast-check for property testing. Use: generate random test cases. +- `TestClock` - `packages/effect/src/testing/TestClock.ts` - Controllable clock for tests. Use: advance time deterministically. +- `TestConsole` - `packages/effect/src/testing/TestConsole.ts` - Test console implementation. Use: assert console output. +- `TestSchema` - `packages/effect/src/testing/TestSchema.ts` - Schema assertion helpers. Use: verify decode/encode behavior. + +### `effect/unstable/ai` + +- `AiError` - `packages/effect/src/unstable/ai/AiError.ts` - AI-related error types. Use: handle model failures. +- `AnthropicStructuredOutput` - `packages/effect/src/unstable/ai/AnthropicStructuredOutput.ts` - Anthropic structured-output codec helpers. Use: parse structured model responses. +- `Chat` - `packages/effect/src/unstable/ai/Chat.ts` - Stateful AI conversation API. Use: manage chat sessions. +- `EmbeddingModel` - `packages/effect/src/unstable/ai/EmbeddingModel.ts` - Provider-agnostic embedding API. Use: generate text vectors. +- `IdGenerator` - `packages/effect/src/unstable/ai/IdGenerator.ts` - Pluggable ID generation. Use: issue stable request IDs. +- `LanguageModel` - `packages/effect/src/unstable/ai/LanguageModel.ts` - AI text generation with tools. Use: call LLMs. +- `McpSchema` - `packages/effect/src/unstable/ai/McpSchema.ts` - MCP schema helpers. Use: type MCP messages. +- `McpServer` - `packages/effect/src/unstable/ai/McpServer.ts` - MCP server support. Use: expose model context tools. +- `Model` - `packages/effect/src/unstable/ai/Model.ts` - Unified AI provider interface. Use: swap AI backends. +- `OpenAiStructuredOutput` - `packages/effect/src/unstable/ai/OpenAiStructuredOutput.ts` - OpenAI structured-output codecs. Use: decode typed model output. +- `Prompt` - `packages/effect/src/unstable/ai/Prompt.ts` - Prompt-building data structures. Use: compose prompts. +- `Response` - `packages/effect/src/unstable/ai/Response.ts` - AI response data structures. Use: inspect completions. +- `ResponseIdTracker` - `packages/effect/src/unstable/ai/ResponseIdTracker.ts` - Response ID tracking utilities. Use: correlate model replies. +- `Telemetry` - `packages/effect/src/unstable/ai/Telemetry.ts` - OpenTelemetry integration for AI operations. Use: trace model calls. +- `Tokenizer` - `packages/effect/src/unstable/ai/Tokenizer.ts` - Tokenization and truncation utilities. Use: enforce token budgets. +- `Tool` - `packages/effect/src/unstable/ai/Tool.ts` - Tool definition and management. Use: expose callable tools. +- `Toolkit` - `packages/effect/src/unstable/ai/Toolkit.ts` - Collection of tools. Use: bundle tool implementations. + +### `effect/unstable/cli` + +- `Argument` - `packages/effect/src/unstable/cli/Argument.ts` - CLI argument definitions. Use: positional arguments. +- `CliError` - `packages/effect/src/unstable/cli/CliError.ts` - CLI error types. Use: report parse failures. +- `CliOutput` - `packages/effect/src/unstable/cli/CliOutput.ts` - CLI rendering/output helpers. Use: format command output. +- `Command` - `packages/effect/src/unstable/cli/Command.ts` - CLI command definitions. Use: build subcommands. +- `Completions` - `packages/effect/src/unstable/cli/Completions.ts` - Shell completion generation. Use: generate completion scripts. +- `Flag` - `packages/effect/src/unstable/cli/Flag.ts` - CLI flag definitions. Use: parse options. +- `GlobalFlag` - `packages/effect/src/unstable/cli/GlobalFlag.ts` - Global CLI flags. Use: shared top-level options. +- `HelpDoc` - `packages/effect/src/unstable/cli/HelpDoc.ts` - CLI help document model. Use: render usage text. +- `Param` - `packages/effect/src/unstable/cli/Param.ts` - CLI parameter helpers. Use: define typed params. +- `Primitive` - `packages/effect/src/unstable/cli/Primitive.ts` - Primitive CLI parsers. Use: parse numbers or strings. +- `Prompt` - `packages/effect/src/unstable/cli/Prompt.ts` - Interactive CLI prompts. Use: ask users for input. + +### `effect/unstable/cluster` + +- `ClusterCron` - `packages/effect/src/unstable/cluster/ClusterCron.ts` - Cluster cron scheduling. Use: distributed recurring jobs. +- `ClusterError` - `packages/effect/src/unstable/cluster/ClusterError.ts` - Cluster error types. Use: classify cluster failures. +- `ClusterMetrics` - `packages/effect/src/unstable/cluster/ClusterMetrics.ts` - Cluster metrics helpers. Use: observe cluster health. +- `ClusterSchema` - `packages/effect/src/unstable/cluster/ClusterSchema.ts` - Cluster schema types. Use: encode cluster messages. +- `ClusterWorkflowEngine` - `packages/effect/src/unstable/cluster/ClusterWorkflowEngine.ts` - Workflow engine for clusters. Use: run distributed workflows. +- `DeliverAt` - `packages/effect/src/unstable/cluster/DeliverAt.ts` - Deferred delivery scheduling. Use: schedule message delivery. +- `Entity` - `packages/effect/src/unstable/cluster/Entity.ts` - Cluster entity abstraction. Use: model sharded actors. +- `EntityAddress` - `packages/effect/src/unstable/cluster/EntityAddress.ts` - Entity addressing types. Use: route entity messages. +- `EntityId` - `packages/effect/src/unstable/cluster/EntityId.ts` - Typed entity identifiers. Use: identify actor instances. +- `EntityProxy` - `packages/effect/src/unstable/cluster/EntityProxy.ts` - Client proxy for entities. Use: call remote entities. +- `EntityProxyServer` - `packages/effect/src/unstable/cluster/EntityProxyServer.ts` - Server for entity proxies. Use: serve entity calls. +- `EntityResource` - `packages/effect/src/unstable/cluster/EntityResource.ts` - Entity-backed resource helpers. Use: manage entity state. +- `EntityType` - `packages/effect/src/unstable/cluster/EntityType.ts` - Entity type descriptors. Use: register entity kinds. +- `Envelope` - `packages/effect/src/unstable/cluster/Envelope.ts` - Message envelope types. Use: wrap cluster messages. +- `HttpRunner` - `packages/effect/src/unstable/cluster/HttpRunner.ts` - HTTP-based runner. Use: transport cluster traffic over HTTP. +- `K8sHttpClient` - `packages/effect/src/unstable/cluster/K8sHttpClient.ts` - Kubernetes HTTP client helpers. Use: discover cluster peers. +- `MachineId` - `packages/effect/src/unstable/cluster/MachineId.ts` - Machine identifier utilities. Use: label cluster nodes. +- `Message` - `packages/effect/src/unstable/cluster/Message.ts` - Cluster message model. Use: send typed payloads. +- `MessageStorage` - `packages/effect/src/unstable/cluster/MessageStorage.ts` - Message persistence abstraction. Use: durable message storage. +- `Reply` - `packages/effect/src/unstable/cluster/Reply.ts` - Reply message helpers. Use: respond to cluster requests. +- `Runner` - `packages/effect/src/unstable/cluster/Runner.ts` - Cluster runner abstraction. Use: host cluster workloads. +- `RunnerAddress` - `packages/effect/src/unstable/cluster/RunnerAddress.ts` - Runner addressing types. Use: locate a runner. +- `RunnerHealth` - `packages/effect/src/unstable/cluster/RunnerHealth.ts` - Runner health reporting. Use: track node readiness. +- `Runners` - `packages/effect/src/unstable/cluster/Runners.ts` - Runner collection utilities. Use: manage active runners. +- `RunnerServer` - `packages/effect/src/unstable/cluster/RunnerServer.ts` - Runner server implementation. Use: expose runner endpoints. +- `RunnerStorage` - `packages/effect/src/unstable/cluster/RunnerStorage.ts` - Runner persistence abstraction. Use: store runner metadata. +- `ShardId` - `packages/effect/src/unstable/cluster/ShardId.ts` - Shard identifier type. Use: map keys to shards. +- `Sharding` - `packages/effect/src/unstable/cluster/Sharding.ts` - Sharding utilities. Use: distribute entities. +- `ShardingConfig` - `packages/effect/src/unstable/cluster/ShardingConfig.ts` - Sharding configuration. Use: tune partitioning behavior. +- `ShardingRegistrationEvent` - `packages/effect/src/unstable/cluster/ShardingRegistrationEvent.ts` - Sharding registration events. Use: observe topology changes. +- `SingleRunner` - `packages/effect/src/unstable/cluster/SingleRunner.ts` - Single-process runner. Use: local cluster execution. +- `Singleton` - `packages/effect/src/unstable/cluster/Singleton.ts` - Cluster singleton abstraction. Use: one active service instance. +- `SingletonAddress` - `packages/effect/src/unstable/cluster/SingletonAddress.ts` - Singleton addressing types. Use: route singleton calls. +- `Snowflake` - `packages/effect/src/unstable/cluster/Snowflake.ts` - Snowflake-style ID generation. Use: unique distributed IDs. +- `SocketRunner` - `packages/effect/src/unstable/cluster/SocketRunner.ts` - Socket-based runner. Use: cluster transport over sockets. +- `SqlMessageStorage` - `packages/effect/src/unstable/cluster/SqlMessageStorage.ts` - SQL-backed message storage. Use: persist cluster messages. +- `SqlRunnerStorage` - `packages/effect/src/unstable/cluster/SqlRunnerStorage.ts` - SQL-backed runner storage. Use: persist runner state. +- `TestRunner` - `packages/effect/src/unstable/cluster/TestRunner.ts` - Test runner utilities. Use: simulate cluster execution. + +### `effect/unstable/devtools` + +- `DevTools` - `packages/effect/src/unstable/devtools/DevTools.ts` - Devtools integration. Use: inspect runtime behavior. +- `DevToolsClient` - `packages/effect/src/unstable/devtools/DevToolsClient.ts` - Devtools client API. Use: connect to devtools server. +- `DevToolsSchema` - `packages/effect/src/unstable/devtools/DevToolsSchema.ts` - Devtools message schemas. Use: type devtools payloads. +- `DevToolsServer` - `packages/effect/src/unstable/devtools/DevToolsServer.ts` - Devtools server API. Use: expose inspection endpoints. + +### `effect/unstable/encoding` + +- `Msgpack` - `packages/effect/src/unstable/encoding/Msgpack.ts` - MessagePack encoding helpers. Use: compact binary serialization. +- `Ndjson` - `packages/effect/src/unstable/encoding/Ndjson.ts` - NDJSON encoding helpers. Use: stream JSON lines. +- `Sse` - `packages/effect/src/unstable/encoding/Sse.ts` - Server-sent event encoding helpers. Use: emit SSE streams. + +### `effect/unstable/eventlog` + +- `Event` - `packages/effect/src/unstable/eventlog/Event.ts` - Event model types. Use: represent domain events. +- `EventGroup` - `packages/effect/src/unstable/eventlog/EventGroup.ts` - Event grouping helpers. Use: batch related events. +- `EventJournal` - `packages/effect/src/unstable/eventlog/EventJournal.ts` - Event journal abstraction. Use: append and read events. +- `EventLog` - `packages/effect/src/unstable/eventlog/EventLog.ts` - Event log API. Use: stream recorded events. +- `EventLogEncryption` - `packages/effect/src/unstable/eventlog/EventLogEncryption.ts` - Event log encryption helpers. Use: protect event payloads. +- `EventLogMessage` - `packages/effect/src/unstable/eventlog/EventLogMessage.ts` - Event log message types. Use: transport log entries. +- `EventLogRemote` - `packages/effect/src/unstable/eventlog/EventLogRemote.ts` - Remote event log client/server helpers. Use: access remote logs. +- `EventLogServer` - `packages/effect/src/unstable/eventlog/EventLogServer.ts` - Event log server support. Use: host event streams. +- `EventLogServerEncrypted` - `packages/effect/src/unstable/eventlog/EventLogServerEncrypted.ts` - Encrypted event log server. Use: secure event serving. +- `EventLogServerUnencrypted` - `packages/effect/src/unstable/eventlog/EventLogServerUnencrypted.ts` - Unencrypted event log server. Use: plain local serving. +- `EventLogSessionAuth` - `packages/effect/src/unstable/eventlog/EventLogSessionAuth.ts` - Event log session auth helpers. Use: authorize event sessions. +- `SqlEventJournal` - `packages/effect/src/unstable/eventlog/SqlEventJournal.ts` - SQL-backed event journal. Use: persist events in SQL. +- `SqlEventLogServerEncrypted` - `packages/effect/src/unstable/eventlog/SqlEventLogServerEncrypted.ts` - SQL-backed encrypted event server. Use: secure persisted logs. +- `SqlEventLogServerUnencrypted` - `packages/effect/src/unstable/eventlog/SqlEventLogServerUnencrypted.ts` - SQL-backed plain event server. Use: simple persisted logs. + +### `effect/unstable/http` + +- `Cookies` - `packages/effect/src/unstable/http/Cookies.ts` - HTTP cookie helpers. Use: parse and set cookies. +- `Etag` - `packages/effect/src/unstable/http/Etag.ts` - ETag utilities. Use: cache validation headers. +- `FetchHttpClient` - `packages/effect/src/unstable/http/FetchHttpClient.ts` - Fetch-based HTTP client. Use: run requests in browsers. +- `FindMyWay` - `packages/effect/src/unstable/http/FindMyWay.ts` - `find-my-way` router integration. Use: path routing. +- `Headers` - `packages/effect/src/unstable/http/Headers.ts` - HTTP header utilities. Use: manage request headers. +- `HttpBody` - `packages/effect/src/unstable/http/HttpBody.ts` - HTTP body representations. Use: send JSON or bytes. +- `HttpClient` - `packages/effect/src/unstable/http/HttpClient.ts` - Effect HTTP client API. Use: perform outbound requests. +- `HttpClientError` - `packages/effect/src/unstable/http/HttpClientError.ts` - HTTP client error types. Use: handle request failures. +- `HttpClientRequest` - `packages/effect/src/unstable/http/HttpClientRequest.ts` - HTTP client request builders. Use: construct requests. +- `HttpClientResponse` - `packages/effect/src/unstable/http/HttpClientResponse.ts` - HTTP client response utilities. Use: read status and body. +- `HttpEffect` - `packages/effect/src/unstable/http/HttpEffect.ts` - HTTP-specific effect helpers. Use: compose HTTP workflows. +- `HttpIncomingMessage` - `packages/effect/src/unstable/http/HttpIncomingMessage.ts` - Incoming message abstraction. Use: read request bodies. +- `HttpMethod` - `packages/effect/src/unstable/http/HttpMethod.ts` - HTTP method helpers. Use: branch on verbs. +- `HttpMiddleware` - `packages/effect/src/unstable/http/HttpMiddleware.ts` - HTTP middleware utilities. Use: add auth or logging. +- `HttpPlatform` - `packages/effect/src/unstable/http/HttpPlatform.ts` - Platform HTTP integration. Use: supply runtime adapters. +- `HttpRouter` - `packages/effect/src/unstable/http/HttpRouter.ts` - HTTP routing abstraction. Use: define endpoints. +- `HttpServer` - `packages/effect/src/unstable/http/HttpServer.ts` - HTTP server API. Use: serve requests. +- `HttpServerError` - `packages/effect/src/unstable/http/HttpServerError.ts` - HTTP server error types. Use: render server failures. +- `HttpServerRequest` - `packages/effect/src/unstable/http/HttpServerRequest.ts` - Server request utilities. Use: inspect inbound requests. +- `HttpServerRespondable` - `packages/effect/src/unstable/http/HttpServerRespondable.ts` - Response conversion protocol. Use: return custom response types. +- `HttpServerResponse` - `packages/effect/src/unstable/http/HttpServerResponse.ts` - Server response builders. Use: send JSON or text. +- `HttpStaticServer` - `packages/effect/src/unstable/http/HttpStaticServer.ts` - Static file serving helpers. Use: serve assets. +- `HttpTraceContext` - `packages/effect/src/unstable/http/HttpTraceContext.ts` - HTTP trace-context propagation. Use: carry tracing headers. +- `Multipart` - `packages/effect/src/unstable/http/Multipart.ts` - Multipart form-data helpers. Use: file uploads. +- `Multipasta` - `packages/effect/src/unstable/http/Multipasta.ts` - Multipasta integration. Use: parse multipart streams. +- `Template` - `packages/effect/src/unstable/http/Template.ts` - HTTP templating helpers. Use: generate responses from templates. +- `Url` - `packages/effect/src/unstable/http/Url.ts` - URL utilities. Use: parse and build URLs. +- `UrlParams` - `packages/effect/src/unstable/http/UrlParams.ts` - URL parameter helpers. Use: encode query strings. + +### `effect/unstable/httpapi` + +- `HttpApi` - `packages/effect/src/unstable/httpapi/HttpApi.ts` - Typed HTTP API description. Use: define an API contract. +- `HttpApiBuilder` - `packages/effect/src/unstable/httpapi/HttpApiBuilder.ts` - Build servers from `HttpApi`. Use: implement typed endpoints. +- `HttpApiClient` - `packages/effect/src/unstable/httpapi/HttpApiClient.ts` - Client generation for `HttpApi`. Use: call typed APIs. +- `HttpApiEndpoint` - `packages/effect/src/unstable/httpapi/HttpApiEndpoint.ts` - Endpoint descriptors. Use: define route inputs/outputs. +- `HttpApiError` - `packages/effect/src/unstable/httpapi/HttpApiError.ts` - HTTP API error types. Use: model typed API failures. +- `HttpApiGroup` - `packages/effect/src/unstable/httpapi/HttpApiGroup.ts` - Group API endpoints. Use: organize related routes. +- `HttpApiMiddleware` - `packages/effect/src/unstable/httpapi/HttpApiMiddleware.ts` - API middleware helpers. Use: add auth policies. +- `HttpApiScalar` - `packages/effect/src/unstable/httpapi/HttpApiScalar.ts` - Scalar UI/integration helpers. Use: expose API docs tooling. +- `HttpApiSchema` - `packages/effect/src/unstable/httpapi/HttpApiSchema.ts` - Schema annotations for HTTP metadata. Use: tag schema fields for APIs. +- `HttpApiSecurity` - `packages/effect/src/unstable/httpapi/HttpApiSecurity.ts` - Security scheme helpers. Use: define auth requirements. +- `HttpApiSwagger` - `packages/effect/src/unstable/httpapi/HttpApiSwagger.ts` - Swagger/OpenAPI integration. Use: serve interactive docs. +- `HttpApiTest` - `packages/effect/src/unstable/httpapi/HttpApiTest.ts` - HttpApi test helpers. Use: test typed endpoints. +- `OpenApi` - `packages/effect/src/unstable/httpapi/OpenApi.ts` - OpenAPI generation helpers. Use: export API specs. + +### `effect/unstable/observability` + +- `Otlp` - `packages/effect/src/unstable/observability/Otlp.ts` - OTLP integration entrypoint. Use: configure OTLP telemetry. +- `OtlpExporter` - `packages/effect/src/unstable/observability/OtlpExporter.ts` - OTLP exporter utilities. Use: send telemetry externally. +- `OtlpLogger` - `packages/effect/src/unstable/observability/OtlpLogger.ts` - OTLP log export support. Use: ship structured logs. +- `OtlpMetrics` - `packages/effect/src/unstable/observability/OtlpMetrics.ts` - OTLP metrics export support. Use: export counters and histograms. +- `OtlpResource` - `packages/effect/src/unstable/observability/OtlpResource.ts` - OTLP resource metadata helpers. Use: tag service identity. +- `OtlpSerialization` - `packages/effect/src/unstable/observability/OtlpSerialization.ts` - OTLP serialization helpers. Use: encode telemetry payloads. +- `OtlpTracer` - `packages/effect/src/unstable/observability/OtlpTracer.ts` - OTLP tracing export support. Use: export spans. +- `PrometheusMetrics` - `packages/effect/src/unstable/observability/PrometheusMetrics.ts` - Prometheus metrics exporter. Use: expose scrape endpoint. + +### `effect/unstable/persistence` + +- `KeyValueStore` - `packages/effect/src/unstable/persistence/KeyValueStore.ts` - Key-value storage abstraction. Use: persist simple state. +- `Persistable` - `packages/effect/src/unstable/persistence/Persistable.ts` - Persistable type helpers. Use: encode stored values. +- `PersistedCache` - `packages/effect/src/unstable/persistence/PersistedCache.ts` - Durable cache. Use: cache across restarts. +- `PersistedQueue` - `packages/effect/src/unstable/persistence/PersistedQueue.ts` - Durable queue. Use: persist work items. +- `Persistence` - `packages/effect/src/unstable/persistence/Persistence.ts` - Persistence service APIs. Use: back durable components. +- `RateLimiter` - `packages/effect/src/unstable/persistence/RateLimiter.ts` - Persistent rate limiter. Use: enforce cross-process limits. +- `Redis` - `packages/effect/src/unstable/persistence/Redis.ts` - Redis-backed persistence helpers. Use: store state in Redis. + +### `effect/unstable/process` + +- `ChildProcess` - `packages/effect/src/unstable/process/ChildProcess.ts` - Child process abstractions. Use: manage spawned commands. +- `ChildProcessSpawner` - `packages/effect/src/unstable/process/ChildProcessSpawner.ts` - Generic child-process spawning service. Use: launch subprocesses. + +### `effect/unstable/reactivity` + +- `AsyncResult` - `packages/effect/src/unstable/reactivity/AsyncResult.ts` - Reactive async result type. Use: represent loading/error/data. +- `Atom` - `packages/effect/src/unstable/reactivity/Atom.ts` - Reactive atom state primitive. Use: local reactive state. +- `AtomHttpApi` - `packages/effect/src/unstable/reactivity/AtomHttpApi.ts` - HttpApi integration for atoms. Use: sync state over HTTP. +- `AtomRef` - `packages/effect/src/unstable/reactivity/AtomRef.ts` - Ref-backed atom helpers. Use: bridge refs to reactivity. +- `AtomRegistry` - `packages/effect/src/unstable/reactivity/AtomRegistry.ts` - Atom registry utilities. Use: track application atoms. +- `AtomRpc` - `packages/effect/src/unstable/reactivity/AtomRpc.ts` - RPC integration for atoms. Use: sync state over RPC. +- `Hydration` - `packages/effect/src/unstable/reactivity/Hydration.ts` - Hydration helpers. Use: restore reactive state. +- `Reactivity` - `packages/effect/src/unstable/reactivity/Reactivity.ts` - Core reactivity utilities. Use: compose reactive computations. + +### `effect/unstable/rpc` + +- `Rpc` - `packages/effect/src/unstable/rpc/Rpc.ts` - Typed RPC description. Use: define RPC contracts. +- `RpcClient` - `packages/effect/src/unstable/rpc/RpcClient.ts` - RPC client API. Use: call remote procedures. +- `RpcClientError` - `packages/effect/src/unstable/rpc/RpcClientError.ts` - RPC client error types. Use: handle transport failures. +- `RpcGroup` - `packages/effect/src/unstable/rpc/RpcGroup.ts` - Group RPC endpoints. Use: organize procedures. +- `RpcMessage` - `packages/effect/src/unstable/rpc/RpcMessage.ts` - RPC message model. Use: encode request/response payloads. +- `RpcMiddleware` - `packages/effect/src/unstable/rpc/RpcMiddleware.ts` - RPC middleware helpers. Use: add auth or tracing. +- `RpcSchema` - `packages/effect/src/unstable/rpc/RpcSchema.ts` - Schema helpers for RPC. Use: type RPC payloads. +- `RpcSerialization` - `packages/effect/src/unstable/rpc/RpcSerialization.ts` - RPC serialization helpers. Use: encode wire formats. +- `RpcServer` - `packages/effect/src/unstable/rpc/RpcServer.ts` - RPC server API. Use: expose procedures. +- `RpcTest` - `packages/effect/src/unstable/rpc/RpcTest.ts` - RPC testing helpers. Use: test RPC handlers. +- `RpcWorker` - `packages/effect/src/unstable/rpc/RpcWorker.ts` - Worker integration for RPC. Use: run RPC over workers. +- `Utils` - `packages/effect/src/unstable/rpc/Utils.ts` - Shared RPC utilities. Use: support custom RPC plumbing. + +### `effect/unstable/schema` + +- `Model` - `packages/effect/src/unstable/schema/Model.ts` - Unstable schema model helpers. Use: define schema-backed models. +- `VariantSchema` - `packages/effect/src/unstable/schema/VariantSchema.ts` - Variant/discriminated schema helpers. Use: model tagged unions. + +### `effect/unstable/socket` + +- `Socket` - `packages/effect/src/unstable/socket/Socket.ts` - Socket abstractions. Use: connect stream transports. +- `SocketServer` - `packages/effect/src/unstable/socket/SocketServer.ts` - Socket server helpers. Use: accept socket clients. + +### `effect/unstable/sql` + +- `Migrator` - `packages/effect/src/unstable/sql/Migrator.ts` - SQL migration helpers. Use: run schema migrations. +- `SqlClient` - `packages/effect/src/unstable/sql/SqlClient.ts` - SQL client API. Use: execute queries. +- `SqlConnection` - `packages/effect/src/unstable/sql/SqlConnection.ts` - SQL connection abstraction. Use: manage DB sessions. +- `SqlError` - `packages/effect/src/unstable/sql/SqlError.ts` - SQL error types. Use: classify database failures. +- `SqlModel` - `packages/effect/src/unstable/sql/SqlModel.ts` - SQL-backed model helpers. Use: map tables to models. +- `SqlResolver` - `packages/effect/src/unstable/sql/SqlResolver.ts` - SQL request resolution helpers. Use: batch DB-backed requests. +- `SqlSchema` - `packages/effect/src/unstable/sql/SqlSchema.ts` - Schema helpers for SQL. Use: type query values. +- `SqlStream` - `packages/effect/src/unstable/sql/SqlStream.ts` - Streaming SQL query helpers. Use: consume large result sets. +- `Statement` - `packages/effect/src/unstable/sql/Statement.ts` - SQL statement builders/types. Use: prepare typed statements. + +### `effect/unstable/workers` + +- `Transferable` - `packages/effect/src/unstable/workers/Transferable.ts` - Transferable value helpers. Use: send data between workers. +- `Worker` - `packages/effect/src/unstable/workers/Worker.ts` - Worker abstractions. Use: run isolated tasks. +- `WorkerError` - `packages/effect/src/unstable/workers/WorkerError.ts` - Worker error types. Use: handle worker failures. +- `WorkerRunner` - `packages/effect/src/unstable/workers/WorkerRunner.ts` - Worker runner utilities. Use: host jobs in workers. + +### `effect/unstable/workflow` + +- `Activity` - `packages/effect/src/unstable/workflow/Activity.ts` - Workflow activity helpers. Use: define durable steps. +- `DurableClock` - `packages/effect/src/unstable/workflow/DurableClock.ts` - Durable clock API. Use: schedule workflow time. +- `DurableDeferred` - `packages/effect/src/unstable/workflow/DurableDeferred.ts` - Durable deferred primitive. Use: await external completion. +- `DurableQueue` - `packages/effect/src/unstable/workflow/DurableQueue.ts` - Durable queue primitive. Use: persist workflow worklists. +- `Workflow` - `packages/effect/src/unstable/workflow/Workflow.ts` - Workflow definition API. Use: declare durable workflows. +- `WorkflowEngine` - `packages/effect/src/unstable/workflow/WorkflowEngine.ts` - Workflow engine runtime. Use: execute workflows. +- `WorkflowProxy` - `packages/effect/src/unstable/workflow/WorkflowProxy.ts` - Workflow client proxy. Use: call workflow instances. +- `WorkflowProxyServer` - `packages/effect/src/unstable/workflow/WorkflowProxyServer.ts` - Workflow proxy server. Use: expose workflow endpoints. + +## `@effect/opentelemetry` Package + +Package path: `packages/opentelemetry` + +- `Logger` - `packages/opentelemetry/src/Logger.ts` - OpenTelemetry logging integration. Use: export structured logs. +- `Metrics` - `packages/opentelemetry/src/Metrics.ts` - OpenTelemetry metrics integration. Use: publish application metrics. +- `NodeSdk` - `packages/opentelemetry/src/NodeSdk.ts` - Node OpenTelemetry SDK wiring. Use: bootstrap telemetry in Node. +- `Resource` - `packages/opentelemetry/src/Resource.ts` - OpenTelemetry resource helpers. Use: describe service metadata. +- `Tracer` - `packages/opentelemetry/src/Tracer.ts` - OpenTelemetry tracing integration. Use: create and export spans. +- `WebSdk` - `packages/opentelemetry/src/WebSdk.ts` - Web OpenTelemetry SDK wiring. Use: bootstrap telemetry in browsers. + +## `@effect/platform-browser` Package + +Package path: `packages/platform-browser` + +- `BrowserHttpClient` - `packages/platform-browser/src/BrowserHttpClient.ts` - Browser HTTP client implementation. Use: make fetch-based requests. +- `BrowserKeyValueStore` - `packages/platform-browser/src/BrowserKeyValueStore.ts` - Browser key-value storage adapter. Use: persist small client values. +- `BrowserPersistence` - `packages/platform-browser/src/BrowserPersistence.ts` - Browser persistence services. Use: store app state locally. +- `BrowserRuntime` - `packages/platform-browser/src/BrowserRuntime.ts` - Browser runtime entrypoints. Use: run Effect apps in browsers. +- `BrowserSocket` - `packages/platform-browser/src/BrowserSocket.ts` - Browser socket implementation. Use: open WebSocket-style connections. +- `BrowserStream` - `packages/platform-browser/src/BrowserStream.ts` - Browser stream adapters. Use: bridge web streams. +- `BrowserWorker` - `packages/platform-browser/src/BrowserWorker.ts` - Browser worker integration. Use: communicate with web workers. +- `BrowserWorkerRunner` - `packages/platform-browser/src/BrowserWorkerRunner.ts` - Worker-side runtime helpers. Use: run Effect code inside workers. +- `Clipboard` - `packages/platform-browser/src/Clipboard.ts` - Clipboard API wrappers. Use: read or write clipboard data. +- `Geolocation` - `packages/platform-browser/src/Geolocation.ts` - Geolocation API wrappers. Use: access device location. +- `IndexedDb` - `packages/platform-browser/src/IndexedDb.ts` - IndexedDB integration. Use: build browser databases. +- `IndexedDbDatabase` - `packages/platform-browser/src/IndexedDbDatabase.ts` - IndexedDB database helpers. Use: define database handles. +- `IndexedDbQueryBuilder` - `packages/platform-browser/src/IndexedDbQueryBuilder.ts` - IndexedDB query builder. Use: compose indexed queries. +- `IndexedDbTable` - `packages/platform-browser/src/IndexedDbTable.ts` - IndexedDB table helpers. Use: work with object stores. +- `IndexedDbVersion` - `packages/platform-browser/src/IndexedDbVersion.ts` - IndexedDB versioning helpers. Use: manage schema upgrades. +- `Permissions` - `packages/platform-browser/src/Permissions.ts` - Permissions API wrappers. Use: query browser permissions. + +## `@effect/platform-bun` Package + +Package path: `packages/platform-bun` + +- `BunChildProcessSpawner` - `packages/platform-bun/src/BunChildProcessSpawner.ts` - Bun child-process spawner. Use: launch subprocesses. +- `BunClusterHttp` - `packages/platform-bun/src/BunClusterHttp.ts` - Bun clustered HTTP helpers. Use: scale HTTP servers. +- `BunClusterSocket` - `packages/platform-bun/src/BunClusterSocket.ts` - Bun clustered socket helpers. Use: scale socket servers. +- `BunFileSystem` - `packages/platform-bun/src/BunFileSystem.ts` - Bun file system implementation. Use: do Bun-based file I/O. +- `BunHttpClient` - `packages/platform-bun/src/BunHttpClient.ts` - Bun HTTP client implementation. Use: make outbound HTTP requests. +- `BunHttpPlatform` - `packages/platform-bun/src/BunHttpPlatform.ts` - Bun HTTP platform services. Use: provide HTTP runtime pieces. +- `BunHttpServer` - `packages/platform-bun/src/BunHttpServer.ts` - Bun HTTP server implementation. Use: serve HTTP endpoints. +- `BunHttpServerRequest` - `packages/platform-bun/src/BunHttpServerRequest.ts` - Bun request adapters. Use: read incoming HTTP requests. +- `BunMultipart` - `packages/platform-bun/src/BunMultipart.ts` - Bun multipart parsing. Use: handle form uploads. +- `BunPath` - `packages/platform-bun/src/BunPath.ts` - Bun path service. Use: resolve filesystem paths. +- `BunRedis` - `packages/platform-bun/src/BunRedis.ts` - Bun Redis integration. Use: talk to Redis. +- `BunRuntime` - `packages/platform-bun/src/BunRuntime.ts` - Bun runtime entrypoints. Use: run Effect apps on Bun. +- `BunServices` - `packages/platform-bun/src/BunServices.ts` - Bun service bundle. Use: provide common Bun services. +- `BunSink` - `packages/platform-bun/src/BunSink.ts` - Bun sink adapters. Use: write streamed output. +- `BunSocket` - `packages/platform-bun/src/BunSocket.ts` - Bun socket implementation. Use: manage socket connections. +- `BunSocketServer` - `packages/platform-bun/src/BunSocketServer.ts` - Bun socket server implementation. Use: accept socket clients. +- `BunStdio` - `packages/platform-bun/src/BunStdio.ts` - Bun stdio integration. Use: access stdin/stdout/stderr. +- `BunStream` - `packages/platform-bun/src/BunStream.ts` - Bun stream adapters. Use: bridge Bun streams. +- `BunTerminal` - `packages/platform-bun/src/BunTerminal.ts` - Bun terminal integration. Use: build CLI terminal interactions. +- `BunWorker` - `packages/platform-bun/src/BunWorker.ts` - Bun worker integration. Use: communicate with workers. +- `BunWorkerRunner` - `packages/platform-bun/src/BunWorkerRunner.ts` - Bun worker runtime helpers. Use: run Effect code in workers. + +## `@effect/platform-node` Package + +Package path: `packages/platform-node` + +- `Mime` - `packages/platform-node/src/Mime.ts` - MIME type helpers. Use: detect or assign content types. +- `NodeChildProcessSpawner` - `packages/platform-node/src/NodeChildProcessSpawner.ts` - Node child-process spawner. Use: launch subprocesses. +- `NodeClusterHttp` - `packages/platform-node/src/NodeClusterHttp.ts` - Node clustered HTTP helpers. Use: scale HTTP servers. +- `NodeClusterSocket` - `packages/platform-node/src/NodeClusterSocket.ts` - Node clustered socket helpers. Use: scale socket servers. +- `NodeFileSystem` - `packages/platform-node/src/NodeFileSystem.ts` - Node file system implementation. Use: do Node-based file I/O. +- `NodeHttpClient` - `packages/platform-node/src/NodeHttpClient.ts` - Node HTTP client implementation. Use: make outbound HTTP requests. +- `NodeHttpIncomingMessage` - `packages/platform-node/src/NodeHttpIncomingMessage.ts` - Node incoming message adapters. Use: read raw Node HTTP messages. +- `NodeHttpPlatform` - `packages/platform-node/src/NodeHttpPlatform.ts` - Node HTTP platform services. Use: provide HTTP runtime pieces. +- `NodeHttpServer` - `packages/platform-node/src/NodeHttpServer.ts` - Node HTTP server implementation. Use: serve HTTP endpoints. +- `NodeHttpServerRequest` - `packages/platform-node/src/NodeHttpServerRequest.ts` - Node request adapters. Use: read incoming HTTP requests. +- `NodeMultipart` - `packages/platform-node/src/NodeMultipart.ts` - Node multipart parsing. Use: handle form uploads. +- `NodePath` - `packages/platform-node/src/NodePath.ts` - Node path service. Use: resolve filesystem paths. +- `NodeRedis` - `packages/platform-node/src/NodeRedis.ts` - Node Redis integration. Use: talk to Redis. +- `NodeRuntime` - `packages/platform-node/src/NodeRuntime.ts` - Node runtime entrypoints. Use: run Effect apps on Node. +- `NodeServices` - `packages/platform-node/src/NodeServices.ts` - Node service bundle. Use: provide common Node services. +- `NodeSink` - `packages/platform-node/src/NodeSink.ts` - Node sink adapters. Use: write streamed output. +- `NodeSocket` - `packages/platform-node/src/NodeSocket.ts` - Node socket implementation. Use: manage socket connections. +- `NodeSocketServer` - `packages/platform-node/src/NodeSocketServer.ts` - Node socket server implementation. Use: accept socket clients. +- `NodeStdio` - `packages/platform-node/src/NodeStdio.ts` - Node stdio integration. Use: access stdin/stdout/stderr. +- `NodeStream` - `packages/platform-node/src/NodeStream.ts` - Node stream adapters. Use: bridge Node streams. +- `NodeTerminal` - `packages/platform-node/src/NodeTerminal.ts` - Node terminal integration. Use: build CLI terminal interactions. +- `NodeWorker` - `packages/platform-node/src/NodeWorker.ts` - Node worker integration. Use: communicate with worker threads. +- `NodeWorkerRunner` - `packages/platform-node/src/NodeWorkerRunner.ts` - Node worker runtime helpers. Use: run Effect code in workers. +- `Undici` - `packages/platform-node/src/Undici.ts` - Undici integration helpers. Use: use Undici-based HTTP features. + +## `@effect/platform-node-shared` Package + +Package path: `packages/platform-node-shared` + +- No public `src/index.ts` barrel is present in this vendored repo, so there are no barrel exports to inventory here. + +## `@effect/vitest` Package + +Package path: `packages/vitest` + +- `vitest` - `packages/vitest/src/index.ts` - Re-export of `vitest` APIs. Use: use standard Vitest APIs. +- `API` - `packages/vitest/src/index.ts` - Test API type alias. Use: type custom test helpers. +- `Vitest` - `packages/vitest/src/index.ts` - Effect-aware Vitest namespace types. Use: type Effect-based tests. +- `addEqualityTesters` - `packages/vitest/src/index.ts` - Installs equality testers. Use: compare Effect values in assertions. +- `effect` - `packages/vitest/src/index.ts` - Effect-aware `it` variant. Use: write scoped Effect tests. +- `live` - `packages/vitest/src/index.ts` - Live-service test variant. Use: run tests with live services. +- `layer` - `packages/vitest/src/index.ts` - Share a `Layer` across tests. Use: provide services to test groups. +- `flakyTest` - `packages/vitest/src/index.ts` - Flaky-test wrapper. Use: stabilize eventually consistent checks. +- `prop` - `packages/vitest/src/index.ts` - Property-test helper. Use: generate property-based cases. +- `it` - `packages/vitest/src/index.ts` - Extended Vitest `it`. Use: mix regular and Effect tests. +- `makeMethods` - `packages/vitest/src/index.ts` - Build extended test methods. Use: wrap a custom test API. +- `describeWrapped` - `packages/vitest/src/index.ts` - `describe` helper with Effect methods. Use: define grouped Effect test suites. \ No newline at end of file diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-effect.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-effect.md new file mode 100644 index 000000000..086b1260a --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-effect.md @@ -0,0 +1,447 @@ +# Effect Guide + +This guide is based on common usage patterns in the vendored repo at `./.repos/effect`. + +Key source areas: + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/tools/` +- `./.repos/effect/packages/platform-*` +- `./.repos/effect/packages/opentelemetry/` +- `./.repos/effect/packages/vitest/` + +## Mental Model + +`Effect` is the default way to represent application work. + +It describes a computation that: + +- succeeds with `A` +- fails with `E` +- requires services `R` + +The repo consistently uses `Effect` as the main abstraction for: + +- business workflows +- service methods +- platform integrations +- resource lifecycles +- tests + +## Most Common Patterns In The Repo + +The dominant usage pattern is: + +1. use `Effect.gen` for workflows and orchestration +2. use `Effect.fn` for reusable effectful functions +3. use precise constructors such as `succeed`, `fail`, `sync`, `try`, and `tryPromise` +4. use `map`, `flatMap`, and `tap` for local transformations +5. access services in implementations and `provide*` only at edges +6. use `acquireRelease` and `scoped` for owned resources +7. use `catchTag` and `match` for typed recovery +8. use `run*` only at runtime boundaries + +## Prefer `Effect.fn` For Reusable Operations + +For reusable effectful operations, prefer `Effect.fn`. + +```ts +import { Effect } from "effect" + +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + return { id: userId, name: "Ada" } +}) +``` + +Use `Effect.fn` when: + +- the operation is reusable +- the operation takes parameters +- the operation is part of business logic or a module API +- you want consistent tracing and stack frames + +Do not treat `Effect.fnUntraced` as the default. If you do not want an explicit named span, use `Effect.fn` without a span name. + +Repo examples: + +- `./.repos/effect/packages/tools/utils/src/Codegen.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/OpenApiPatch.ts` + +## Use `Effect.gen` For Workflows + +Use `Effect.gen` for orchestration and sequential workflows, especially when there are multiple `yield*` steps. + +```ts +const program = Effect.gen(function*() { + const config = yield* Config + const repo = yield* UserRepo + const user = yield* repo.getById("u_123") + return { config, user } +}) +``` + +Use `Effect.gen` when: + +- the body is a workflow +- you are reading multiple services +- you have branching or multiple sequential steps +- you are implementing a layer, handler, or orchestration + +Repo examples: + +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/OpenApiGenerator.ts` + +## `Effect.fn` vs `Effect.gen` + +Use this rule: + +- reusable operation: `Effect.fn` +- inline workflow block: `Effect.gen` + +Good split: + +```ts +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + const repo = yield* UserRepo + return yield* repo.getById(userId) +}) + +const program = Effect.gen(function*() { + const user = yield* loadUser("u_123") + yield* Effect.logInfo("loaded user", user) +}) +``` + +## `Effect.fnUntraced` Is An Escape Hatch + +For application and business code, `Effect.fnUntraced` is not the default. + +Use it only when: + +- the function is an internal low-level helper +- observability is intentionally being traded away +- there is a concrete performance or tracing reason + +If the only goal is to avoid an explicit named span, prefer: + +```ts +const normalizeUser = Effect.fn(function*(input: string) { + return input.trim().toLowerCase() +}) +``` + +Instead of: + +```ts +const normalizeUser = Effect.fnUntraced(function*(input: string) { + return input.trim().toLowerCase() +}) +``` + +## Constructor Functions + +The repo uses constructor functions very deliberately. + +### `Effect.succeed` + +Use for pure successful values. + +```ts +const ok = Effect.succeed(42) +``` + +### `Effect.fail` + +Use for expected typed failures. + +```ts +const notFound = Effect.fail(UserNotFound.make({ userId: "u_123" })) +``` + +### `Effect.sync` + +Use for synchronous side effects or pure synchronous construction that should live inside `Effect`. + +```ts +const buildConfig = Effect.sync(() => ({ retries: 3 })) +``` + +### `Effect.try` + +Use for synchronous code that may throw. + +```ts +import { Effect, Schema } from "effect" + +class ParseError extends Schema.TaggedErrorClass()("ParseError", { + cause: Schema.Defect +}) {} + +const parseJson = (input: string) => + Effect.try({ + try: () => JSON.parse(input), + catch: (cause) => ParseError.make({ cause }) + }) +``` + +### `Effect.tryPromise` + +Use for Promise-returning APIs. + +```ts +import { Effect, Schema } from "effect" + +class FetchError extends Schema.TaggedErrorClass()("FetchError", { + cause: Schema.Defect +}) {} + +const fetchText = (url: string) => + Effect.tryPromise({ + try: () => fetch(url).then((response) => response.text()), + catch: (cause) => FetchError.make({ cause }) + }) +``` + +Preferred rule: + +- pure value: `succeed` +- expected failure: `fail` +- synchronous non-throwing effect: `sync` +- synchronous throwing boundary: `try` +- Promise boundary: `tryPromise` + +## Local Composition + +The repo uses `map`, `flatMap`, and `tap` constantly for small local transformations. + +### `Effect.map` + +Use to transform successful values. + +```ts +const userName = loadUser("u_123").pipe( + Effect.map((user) => user.name) +) +``` + +### `Effect.flatMap` + +Use when the next step returns another `Effect`. + +```ts +const result = loadUser("u_123").pipe( + Effect.flatMap((user) => saveAudit(user.id)) +) +``` + +### `Effect.tap` + +Use for side effects that should preserve the main value. + +```ts +const result = loadUser("u_123").pipe( + Effect.tap((user) => Effect.logDebug("loaded user", { userId: user.id })) +) +``` + +Preferred rule: + +- outer workflow: `Effect.gen` +- local transformation: `map`, `flatMap`, `tap` + +## Services And Provisioning + +Repo style is: + +- access services in implementation code +- provide them at boundaries + +### Access services in implementations + +```ts +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + const repo = yield* UserRepo + return yield* repo.getById(userId) +}) +``` + +or: + +```ts +const loadUser = (userId: string) => + Effect.service(UserRepo).pipe( + Effect.flatMap((repo) => repo.getById(userId)) + ) +``` + +### Provide at the edge + +```ts +const program = loadUser("u_123").pipe( + Effect.provide(AppLayer) +) +``` + +Use `provideService` and `provideServiceEffect` for targeted overrides, especially in tests or framework boundaries. + +Do not default to exporting thin accessor functions that just fetch a service and forward to one service method. Prefer real business operations or direct service usage within the owning workflow. + +Repo examples: + +- `./.repos/effect/packages/tools/utils/src/bin.ts` +- `./.repos/effect/packages/tools/openapi-generator/test/` + +## Error Handling + +Common repo patterns: + +- `catchTag` for expected tagged errors +- `match` for totalizing an effect into a value +- `catchCause` for full-cause infra handling + +### `Effect.catchTag` + +Use for targeted typed recovery. + +```ts +const safe = loadUser("u_123").pipe( + Effect.catchTag("UserNotFound", () => Effect.succeed(null)) +) +``` + +### `Effect.match` + +Use when the caller wants a value either way. + +```ts +const result = loadUser("u_123").pipe( + Effect.match({ + onFailure: () => null, + onSuccess: (user) => user + }) +) +``` + +For deeper guidance, see `./references/guide-error-handling.md`. + +## Resource Management + +One of the strongest repo patterns is explicit resource ownership. + +### `Effect.acquireRelease` + +Use for resources that must be cleaned up. + +```ts +const connection = Effect.acquireRelease( + openConnection, + (conn) => closeConnection(conn) +) +``` + +### `Effect.scoped` + +Use when a workflow consumes scoped resources and should tie cleanup to scope lifetime. + +```ts +const program = Effect.scoped( + Effect.gen(function*() { + const conn = yield* connection + return yield* conn.query("select 1") + }) +) +``` + +Repo examples: + +- `./.repos/effect/packages/platform-node/` +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` + +## SQL And Runtime Integrations + +When Effect already provides a domain module for a capability, prefer that module over direct raw runtime client usage in business code. + +Important example: + +- prefer Effect SQL modules from `effect/unstable/sql/*` over embedding a native SQL driver directly in domain services + +Why: + +- transactions, spans, and typed errors stay inside the Effect model +- layering stays cleaner +- migrations and query conventions stay consistent + +For SQL-specific guidance, see `./references/guide-sql.md`. + +## Observability + +The repo uses observability around meaningful boundaries, not every tiny helper. + +Common patterns: + +- `Effect.fn` for named operations +- `Effect.withSpan` for nested span boundaries +- `Effect.log*` for operational events +- `Effect.track` for metrics + +For detailed guidance, see `./references/guide-observability.md`. + +## Runtime Boundaries + +The repo keeps `run*` APIs at true runtime boundaries. + +### `Effect.runPromise` + +Use when leaving Effect world into Promise-based hosts. + +### `Effect.runFork` + +Use for background fibers or long-running integration hooks. + +### `Effect.runSync` + +Use sparingly, mostly in specialized internals where synchrony is guaranteed. + +Preferred rule: + +- library/business code should return `Effect` +- entrypoints and integration boundaries should run `Effect` + +If you have multiple external entrypoints, prefer `ManagedRuntime`. + +## Commonly Used Effect APIs In This Repo + +These are the most practically important `Effect` functions to know first: + +- `Effect.fn` +- `Effect.gen` +- `Effect.succeed` +- `Effect.fail` +- `Effect.sync` +- `Effect.try` +- `Effect.tryPromise` +- `Effect.map` +- `Effect.flatMap` +- `Effect.tap` +- `Effect.service` +- `Effect.provide` +- `Effect.provideService` +- `Effect.catchTag` +- `Effect.match` +- `Effect.acquireRelease` +- `Effect.scoped` +- `Effect.withSpan` +- `Effect.logInfo` +- `Effect.logDebug` +- `Effect.runPromise` + +## Good Repo Examples To Study + +- `./.repos/effect/packages/tools/utils/src/Codegen.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/OpenApiPatch.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/OpenApiGenerator.ts` +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` +- `./.repos/effect/packages/platform-node/` +- `./.repos/effect/packages/vitest/src/index.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-error-handling.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-error-handling.md new file mode 100644 index 000000000..b842bca1d --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-error-handling.md @@ -0,0 +1,540 @@ +# Error Handling Guide + +This guide is based on the vendored Effect source in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Data.ts` +- `./.repos/effect/packages/effect/src/Schema.ts` +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` + +## Mental Model + +Effect distinguishes three failure modes: + +- failure: expected, typed errors in the `E` channel of `Effect` +- defect: unexpected unchecked failures, represented as `Cause.Die` +- interrupt: cooperative cancellation, represented as `Cause.Interrupt` + +This distinction is explicit in `Cause`. + +Repo references: + +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` + +## Preferred Error Definition Styles + +Preference order: + +1. use schema-based errors when possible +2. fall back to `Data.TaggedError` only when the error payload is not meaningfully serializable or schema-shaped + +Schema-based errors are strictly more powerful because they give you: + +- typed yieldable errors +- schema-defined fields +- encode/decode support +- better protocol and boundary interoperability +- stronger documentation and tooling hooks + +### 1. `Schema.TaggedErrorClass` for schema-backed tagged errors + +Use `Schema.TaggedErrorClass` by default when the error can be described with schemas. + +Why: + +- it creates a yieldable tagged error +- fields are defined with `Schema` +- the error shape can participate in schema-based tooling and encode/decode flows + +Repo references: + +- `./.repos/effect/packages/effect/src/Schema.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` + +Example: + +```ts +import { Effect, Schema } from "effect" + +class InvalidPayload extends Schema.TaggedErrorClass()( + "InvalidPayload", + { + field: Schema.String, + reason: Schema.String + } +) {} + +const validate = Effect.fail( + InvalidPayload.make({ + field: "email", + reason: "missing" + }) +) +``` + +Use this when: + +- the error is part of a protocol or transport boundary +- the error needs a precise schema representation +- the error should be serializable or documented structurally + +This is also a good default for domain errors when their payload is schema-friendly. + +### 2. `Schema.ErrorClass` for schema-backed errors without `_tag` routing + +Use `Schema.ErrorClass` when you want schema-defined error objects but do not specifically need tag-based pattern matching. + +Repo references: + +- `./.repos/effect/packages/effect/src/Schema.ts` +- examples across `./.repos/effect/packages/effect/src/unstable/*` + +Example shape from the vendored repo: + +- `./.repos/effect/packages/effect/src/unstable/httpapi/HttpApiError.ts` +- `./.repos/effect/packages/effect/src/unstable/workers/WorkerError.ts` + +### 3. `Data.TaggedError` for non-serializable or lightweight domain errors + +Use `Data.TaggedError` when schema-based errors are not a good fit. + +This is mainly the fallback for: + +- non-serializable payloads +- ad hoc in-memory-only errors +- cases where schema shape would be artificial or misleading + +Repo reference: + +- `./.repos/effect/packages/effect/src/Data.ts` + +Example: + +```ts +import { Data, Effect } from "effect" + +class UserNotFound extends Data.TaggedError("UserNotFound")<{ + readonly userId: string +}> {} + +const loadUser = (userId: string) => + Effect.fail(new UserNotFound({ userId })) + +const program = Effect.gen(function*() { + yield* loadUser("u_123") +}) +``` + +## When To Prefer `Data.TaggedError` vs `Schema.TaggedErrorClass` + +Prefer `Schema.TaggedErrorClass` when: + +- the error can be expressed as a schema +- the error payload must be described by schemas +- the error crosses process, protocol, persistence, or serialization boundaries +- you want the error type to participate in schema tooling + +Prefer `Data.TaggedError` when: + +- the payload is not meaningfully serializable +- the payload cannot reasonably be modeled as a schema +- the error is intentionally local and in-memory only + +## Schema-Based Error Workflows + +### Boundary validation should fail with `SchemaError` + +When validating external input, Effect's schema APIs return `SchemaError` in the error channel. + +Repo references: + +- `./.repos/effect/packages/effect/src/Schema.ts` +- `Schema.decodeUnknownEffect` +- `Schema.decodeUnknownExit` +- `Schema.encodeUnknownEffect` + +Example: + +```ts +import { Effect, Schema } from "effect" + +const UserPayload = Schema.Struct({ + id: Schema.String, + email: Schema.String +}) + +const decodeUser = Schema.decodeUnknownEffect(UserPayload) +``` + +This gives you: + +- success: validated typed data +- failure: `Schema.SchemaError` + +### Normalize `SchemaError` at the boundary + +For application code, it is often better to convert `SchemaError` into a domain error near the boundary. + +Example: + +```ts +import { Data, Effect, Schema } from "effect" + +class InvalidRequestBody extends Data.TaggedError("InvalidRequestBody")<{ + readonly message: string +}> {} + +const UserPayload = Schema.Struct({ + id: Schema.String, + email: Schema.String +}) + +const decodeUser = (input: unknown) => + Schema.decodeUnknownEffect(UserPayload)(input).pipe( + Effect.catchTag("SchemaError", (error) => + Effect.fail(new InvalidRequestBody({ message: error.message })) + ) + ) +``` + +Why: + +- transport validation stays close to the transport layer +- the rest of the application can work with domain-specific errors + +### Use schema-backed errors for protocol errors + +The vendored repo uses schema-backed errors in places like SQL, RPC, sockets, and HTTP APIs. + +Strong examples: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` +- `./.repos/effect/packages/effect/src/unstable/socket/Socket.ts` +- `./.repos/effect/packages/effect/src/unstable/eventlog/EventLogMessage.ts` + +These are good reference points when the error contract matters externally. + +## Wrapping Foreign Or Generic Errors + +When an error comes from a library, runtime API, or generic `Error`, prefer wrapping it in a typed error instead of leaking the foreign error directly through your domain or protocol boundary. + +This is a very common pattern in the vendored repo. + +Good examples: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClientError.ts` +- `./.repos/effect/packages/effect/src/unstable/socket/Socket.ts` +- `./.repos/effect/packages/effect/src/unstable/workers/WorkerError.ts` +- `./.repos/effect/packages/effect/src/unstable/persistence/Redis.ts` + +### Preferred Pattern + +Wrap the foreign error in a schema-backed typed error and preserve the original error in a `cause` field. + +Prefer using: + +- `Schema.Defect` when you want to preserve a generic encoded defect +- `Schema.DefectWithStack` when the stack should be preserved in the schema contract + +Example: + +```ts +import { Effect, Schema } from "effect" + +class TodoStorageError extends Schema.TaggedErrorClass()( + "TodoStorageError", + { + operation: Schema.String, + cause: Schema.Defect + } +) {} + +const makeStorageError = (operation: string) => (cause: unknown) => + TodoStorageError.make({ + operation, + cause + }) + +const loadTodo = (id: number) => + Effect.try({ + try: () => someLibraryCall(id), + catch: makeStorageError("loadTodo") + }) +``` + +When stack preservation matters in the encoded schema, prefer: + +```ts +class WorkerFailure extends Schema.TaggedErrorClass()( + "WorkerFailure", + { + cause: Schema.DefectWithStack + } +) {} +``` + +### Why This Is Preferred + +- the application still exposes a typed error contract +- the original foreign failure is preserved for diagnostics +- schema-aware transports can encode and decode the failure shape +- business code does not become coupled to a raw library error type + +### When To Use This Pattern + +Use it when: + +- a third-party library throws or rejects with `Error` +- a runtime API returns generic failures +- a lower-level subsystem failure should be surfaced through a typed domain or protocol error +- you need to preserve the underlying failure for debugging without leaking the foreign type as the public error contract + +### `Schema.Defect` vs `Schema.DefectWithStack` + +Prefer `Schema.Defect` by default. + +Use `Schema.DefectWithStack` when: + +- stack information is part of the intended encoded error contract +- the error is primarily infrastructural or diagnostic +- the downstream consumer benefits from the preserved stack + +### Avoid This Anti-Pattern + +Avoid exposing raw generic errors directly as the application error contract. + +Bad: + +```ts +const loadTodo = (id: number) => + Effect.try({ + try: () => someLibraryCall(id), + catch: (cause) => cause as Error + }) +``` + +Why this is bad: + +- the error channel loses a stable typed contract +- the code depends on unsafe assertions +- transport and schema integration become weaker +- callers must understand foreign error shapes instead of your own typed error model + +## Handling Failures + +### Handle specific tagged errors with `Effect.catchTag` + +Use `catchTag` when your error type has `_tag` and you want focused recovery. + +Repo reference: + +- `./.repos/effect/packages/effect/src/Effect.ts` + +Example: + +```ts +const recovered = program.pipe( + Effect.catchTag("UserNotFound", (error) => + Effect.succeed({ id: error.userId, guest: true }) + ) +) +``` + +### Handle several tagged errors with `Effect.catchTags` + +Use `catchTags` when multiple domain errors should be handled together. + +```ts +const recovered = program.pipe( + Effect.catchTags({ + UserNotFound: () => Effect.succeed(null), + InvalidPayload: (error) => Effect.succeed({ error: error.reason }) + }) +) +``` + +### Handle predicate-based subsets with `Effect.catchIf` + +Use `catchIf` when matching on a predicate or refinement, not just `_tag`. + +### Turn failure into a value with `Effect.match` + +Use `match` when you want to fully fold the typed error channel into a success value. + +```ts +const outcome = program.pipe( + Effect.match({ + onFailure: (error) => ({ ok: false as const, error }), + onSuccess: (value) => ({ ok: true as const, value }) + }) +) +``` + +## Handling Defects + +Defects are not normal domain failures. + +They come from: + +- `Effect.die` +- unchecked exceptions in effectful code +- invariants that were broken + +Repo references: + +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` + +### Preferred rule + +Do not model expected business failures as defects. + +Use defects for: + +- impossible states +- programmer errors +- unrecoverable infrastructure corruption + +### Inspect defects with `sandbox`, `catchCause`, or `matchCause` + +Use `sandbox` to expose `Cause` in the error channel. + +```ts +import { Cause, Effect } from "effect" + +const diagnosed = program.pipe( + Effect.sandbox, + Effect.catchCause((cause) => { + if (Cause.hasDies(cause)) { + return Effect.succeed("defect") + } + return Effect.failCause(cause) + }) +) +``` + +Use `matchCause` or `matchCauseEffect` when you need to distinguish: + +- typed failures +- defects +- interrupts + +### Boundary-only recovery for defects + +If you recover from defects at all, do it only at clear boundaries. + +Examples: + +- worker or RPC boundary +- CLI top-level runner +- HTTP server adapter + +Typical pattern: + +- log or report defect details +- translate to a safe external error +- avoid continuing as if it were a normal domain failure + +### `Effect.orDie` + +Use `orDie` when an error channel should be treated as unrecoverable from this point onward. + +That is appropriate when: + +- a failure has already been validated elsewhere as impossible +- continuing with typed recovery would only obscure a broken invariant + +Do not use `orDie` just to silence a type you do not want to handle. + +## Handling Interrupts + +Interrupts are cancellation, not business failure. + +Repo references: + +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` + +### Use `Effect.interrupt` to stop work cooperatively + +Interrupts signal that the fiber should stop. They should not usually be translated into a domain error. + +### Use `Effect.onInterrupt` for cleanup + +If interrupted work needs special cleanup, use `onInterrupt`. + +```ts +import { Console, Effect } from "effect" + +const program = longRunningTask.pipe( + Effect.onInterrupt(() => Console.log("cleaning up after interrupt")) +) +``` + +### Use `Cause` inspection when interrupts must be distinguished + +When handling full causes, use `Cause.isInterruptReason`, `Cause.hasInterrupts`, or filtering over `cause.reasons`. + +This is useful for: + +- deciding whether to suppress logs for normal cancellation +- keeping retries for failure but not for cancellation +- distinguishing timeout/cancel flows from real errors + +### Do not treat interrupts as ordinary failures + +Avoid patterns that collapse all causes into a single error value too early. Interrupts often need different operational behavior. + +## Recommended Patterns + +### Pattern: domain errors inside the app, schema errors at the edge + +- decode external input with `Schema.decodeUnknownEffect` +- convert `SchemaError` into a domain or transport error near the boundary +- keep the rest of the application on domain errors + +### Pattern: tagged errors for recovery + +- define domain failures with `Data.TaggedError` +- recover with `catchTag` or `catchTags` +- keep `_tag` names stable and descriptive + +### Pattern: schema-backed errors for protocols + +- use `Schema.TaggedErrorClass` or `Schema.ErrorClass` when the error contract itself matters +- follow examples in SQL, socket, RPC, and HTTP modules + +### Pattern: only inspect `Cause` when you really need the full failure structure + +Use `catchCause`, `matchCause`, or `sandbox` when you must distinguish: + +- expected failures +- defects +- interrupts + +Otherwise prefer the simpler typed error operators. + +## Anti-Patterns + +- using defects for expected validation or business-rule failures +- converting every error immediately to `unknown` or `string` +- using `orDie` to avoid proper handling of expected errors +- treating interrupts as ordinary business failures +- leaking `SchemaError` deep into domain code when it should be normalized at the boundary + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/Data.ts` +- `./.repos/effect/packages/effect/src/Cause.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/Schema.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` +- `./.repos/effect/packages/effect/src/unstable/http/HttpClientError.ts` +- `./.repos/effect/packages/effect/src/unstable/http/HttpServerError.ts` +- `./.repos/effect/packages/effect/src/unstable/socket/Socket.ts` +- `./.repos/effect/packages/effect/src/unstable/httpapi/HttpApiError.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-layers.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-layers.md new file mode 100644 index 000000000..0419b5d45 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-layers.md @@ -0,0 +1,1018 @@ +# Layers Guide + +This guide is based on the vendored Effect source in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Context.ts` +- `./.repos/effect/packages/effect/src/Layer.ts` +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/ManagedRuntime.ts` + +## Mental Model + +A service is a typed dependency. + +A layer is a recipe for building one or more services, possibly using other services as dependencies. + +Effect's model is: + +- define service identifiers with `Context.Service` or `Context.Service(...)` +- require services from effects with `Effect.service` or by yielding the service key directly +- build implementations with `Layer` +- provide layers at the program boundary or subsystem boundary + +`Layer` means: + +- `ROut`: services produced by the layer +- `E`: possible failures while constructing the layer +- `RIn`: dependencies required to build it + +Repo references: + +- `./.repos/effect/packages/effect/src/Context.ts` +- `./.repos/effect/packages/effect/src/Layer.ts` + +## Services + +## What A Service Is + +A service is a typed key plus its implementation shape. + +In Effect, services are values in `Context`, not global singletons. + +This gives you: + +- explicit dependencies +- easy substitution in tests +- layer-based composition +- multiple implementations for the same interface + +## Preferred Service Definition Style + +Prefer the class syntax with `Context.Service`. + +This matches the vendored repo's current style. + +There are two good definition styles: + +- explicit service shape in the `Context.Service<...>` generic +- inferred service shape from the `make` argument + +Example: + +```ts +import { Context, Effect, Schema } from "effect" + +class UserRepoError extends Schema.TaggedErrorClass()( + "UserRepoError", + { + message: Schema.String + } +) {} + +class UserRepo extends Context.Service Effect.Effect<{ id: string; name: string }, UserRepoError> +}>()("UserRepo") {} +``` + +Why this style is preferred: + +- the service identifier and shape live in one place +- it works naturally with `yield* UserRepo` +- it matches the patterns used across `.repos/effect` + +Repo reference: + +- `./.repos/effect/packages/effect/src/Context.ts` + +## Service Shape Inference With `make` + +When the implementation shape is clearer than the interface declaration, prefer using the `make` argument so the service shape is inferred from the implementation. + +```ts +import { Context, Effect, Schema } from "effect" + +class UserRepoError extends Schema.TaggedErrorClass()( + "UserRepoError", + { + message: Schema.String + } +) {} + +class UserRepo extends Context.Service()("UserRepo", { + make: Effect.succeed({ + getById: Effect.fn("UserRepo.getById")(function*(id: string) { + return yield* Effect.fail( + UserRepoError.make({ message: `User ${id} not found` }) + ) + }) + }) +}) {} +``` + +Why this style is useful: + +- the implementation and inferred API stay together +- TypeScript derives the service shape automatically +- it avoids repeating the same method signatures twice + +Prefer this style when: + +- the implementation is small and obvious +- the explicit service shape would only duplicate the implementation + +Prefer the explicit generic shape when: + +- you want the contract stated before the implementation +- the API surface should be emphasized separately from the implementation + +## Service Example + +```ts +import { Context, Effect, Schema } from "effect" + +class UserNotFound extends Schema.TaggedErrorClass()( + "UserNotFound", + { + userId: Schema.String + } +) {} + +class UserRepo extends Context.Service Effect.Effect<{ id: string; name: string }, UserNotFound> +}>()("UserRepo") {} + +const loadUser = (userId: string) => + Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById(userId) + }) +``` + +Key points: + +- `UserRepo` is both the identifier and a value you can yield from `Effect.gen` +- the implementation shape is explicit in the service definition +- the effect that uses it stays abstract over the implementation + +## When To Use `Context.Reference` + +Use `Context.Reference` for contextual values with defaults, not for full service APIs. + +Good use cases: + +- current configuration knobs +- current request metadata +- feature flags or tracing flags with defaults + +Repo reference: + +- `./.repos/effect/packages/effect/src/Context.ts` + +Use a full service instead when: + +- behavior matters more than data +- you need multiple methods +- you want a concrete test double or alternate implementation + +## Accessing Services + +Common patterns: + +```ts +const program = Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById("u_123") +}) +``` + +or: + +```ts +const program = Effect.service(UserRepo).pipe( + Effect.flatMap((repo) => repo.getById("u_123")) +) +``` + +Best practice: + +- use `yield* Service` or `Effect.service(Service)` inside business logic +- do not manually thread service implementations through function arguments when they are real application dependencies + +## Service Encapsulation + +Prefer keeping service access inside the business operation that needs it rather than exporting thin accessor wrappers for every method. + +Avoid this pattern: + +```ts +export const createTodo = Effect.fn(function*(title: string) { + const todos = yield* TodoService + return yield* todos.create(title) +}) +``` + +Why this is usually a bad pattern: + +- it leaks the service dependency into a second public API layer +- it encourages a redundant accessor function per service method +- it spreads dependency access patterns across the codebase +- it weakens service encapsulation instead of improving it + +Prefer one of these patterns instead: + +1. Put the real business logic in a function that uses the service internally because it adds behavior beyond simple forwarding. +2. Expose the service itself and call its methods from the module that owns the workflow. +3. If you need a public operation, make it a real business operation, not a trivial alias of one service method. + +Good: + +```ts +export const completeTodo = Effect.fn("completeTodo")(function*(id: number) { + const todos = yield* TodoService + const todo = yield* todos.getById(id) + if (todo.completed) { + return todo + } + return yield* todos.setCompleted(id, true) +}) +``` + +This is good because: + +- the exported function represents a business operation +- the service remains an internal dependency of that operation +- the function adds behavior rather than just forwarding one method call + +## Layers + +## What A Layer Is + +A layer constructs services from dependencies. + +Use a layer when: + +- service construction is effectful +- the service depends on other services +- the service owns resources that must be acquired and released safely +- you want composition and reuse across modules + +## Preferred Layer Constructors + +### `Layer.succeed` + +Use for pure, already-constructed implementations. + +```ts +const UserRepoTest = Layer.succeed(UserRepo)({ + getById: (id) => Effect.succeed({ id, name: "Test User" }) +}) +``` + +Use this when: + +- construction is pure +- no dependencies are needed +- no scoped resources are involved + +### `Layer.effect` + +Use when constructing a service requires effects, other services, or scoped resource acquisition. + +```ts +class Config extends Context.Service()("Config") {} + +const UserRepoLayer = Layer.effect(UserRepo)( + Effect.gen(function*() { + const config = yield* Config + + return { + getById: (id) => + Effect.succeed({ + id, + name: `Fetched from ${config.apiBaseUrl}` + }) + } + }) +) +``` + +Use this when: + +- construction is effectful +- construction depends on other services +- construction needs `Scope` and finalization +- you want typed construction failure + +This is also the correct constructor for services that own resources with acquisition and release semantics. In this repo, `Layer.effect` replaces the old `Layer.scoped` API. + +Typical examples: + +- database pools +- sockets +- background worker processes +- long-lived subscriptions + +### `Layer.effectDiscard` + +Use `Layer.effectDiscard` for scoped startup effects that do not provide a service. + +Good use cases: + +- starting background fibers in a layer +- one-time scoped initialization side effects +- subsystem startup hooks + +### `Layer.effectContext` + +Use when one effect constructs a full `Context` containing multiple services. + +This is useful for subsystem builders that provide several related services together. + +## Layer Composition + +These operators do different things. Do not treat them as interchangeable. + +### Composition Cheat Sheet + +- `Layer.mergeAll(a, b, ...)`: combine outputs of multiple layers +- `Layer.provide(target, dependencies)`: feed dependency outputs into `target` and keep only `target` outputs +- `Layer.provideMerge(target, dependencies)`: feed dependency outputs into `target` and keep both dependency outputs and target outputs +- `Layer.flatMap(layer, f)`: choose the next layer based on the built service value + +### Example Services + +```ts +import { Context, Effect, Layer } from "effect" + +class Config extends Context.Service()("Config") {} + +class Logger extends Context.Service Effect.Effect +}>()("Logger") {} + +class UserRepo extends Context.Service Effect.Effect<{ id: string; name: string }> +}>()("UserRepo") {} + +const ConfigLayer = Layer.succeed(Config)({ + apiBaseUrl: "https://api.example.com" +}) + +const LoggerLayer = Layer.succeed(Logger)({ + log: (message) => Effect.sync(() => console.log(message)) +}) + +const UserRepoLayer = Layer.effect(UserRepo)( + Effect.gen(function*() { + const config = yield* Config + const logger = yield* Logger + + return { + getById: (id) => + Effect.gen(function*() { + yield* logger.log(`loading ${id} from ${config.apiBaseUrl}`) + return { id, name: "Ada" } + }) + } + }) +) +``` + +### `Layer.mergeAll` + +Use `Layer.mergeAll` to combine outputs of independent layers. + +```ts +const Dependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) +``` + +Use this when: + +- the layers provide different services +- neither needs to transform the other directly + +Semantics: + +- inputs are combined +- outputs are combined +- no dependency feeding happens automatically + +Important: + +- `Layer.mergeAll(ConfigLayer, UserRepoLayer)` is wrong if `UserRepoLayer` requires `Config` and `Logger` +- `mergeAll` does not satisfy `UserRepoLayer`'s requirements +- it only places both layers side by side in the output graph + +Correct pattern: + +```ts +const Dependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) +``` + +### `Layer.provide` + +Use `Layer.provide` to satisfy a target layer's dependencies with another layer, while keeping only the target layer's outputs. + +```ts +const Dependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) + +const UserRepoLayerReady = Layer.provide(UserRepoLayer, Dependencies) +``` + +Interpretation: + +- `UserRepoLayer` requires `Config` and `Logger` +- `Dependencies` provides those dependencies +- the resulting layer provides only `UserRepo` +- `Config` and `Logger` are used for construction but are not kept in the final output + +This is the operator to use when you want to hide construction dependencies behind a narrower public layer. + +Example program: + +```ts +const program = Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById("u_123") +}).pipe( + Effect.provide(UserRepoLayerReady) +) +``` + +### `Layer.provideMerge` + +Use `provideMerge` when you want to satisfy dependencies and retain both the dependency outputs and the target outputs. + +```ts +const Dependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) + +const AppLayer = Layer.provideMerge(UserRepoLayer, Dependencies) +``` + +Interpretation: + +- `UserRepoLayer` still gets `Config` and `Logger` +- the resulting layer provides `UserRepo`, `Config`, and `Logger` + +This is useful for assembling larger application layers incrementally, especially when downstream code still needs access to the dependencies. + +Example program: + +```ts +const program = Effect.gen(function*() { + const repo = yield* UserRepo + const logger = yield* Logger + + const user = yield* repo.getById("u_123") + yield* logger.log(user.name) + return user +}).pipe( + Effect.provide(AppLayer) +) +``` + +Preferred rule: + +- use `provide` when you want to hide dependency details +- use `provideMerge` when you want to keep dependency services available downstream + +### `Layer.mergeAll` vs `Layer.provide` vs `Layer.provideMerge` + +Think of them like this: + +- `mergeAll`: put layers next to each other +- `provide`: plug one layer into another, expose only the target +- `provideMerge`: plug one layer into another, expose both sides + +### Composition Style Best Practice + +Layers should almost always be fully composed locally before they are assembled into the final application layer. + +Preferred style: + +- define each service layer separately +- define local subsystem dependency bundles separately +- fully compose each subsystem locally with `Layer.provide` or `Layer.provideMerge` +- assemble the final application layer with `Layer.mergeAll(...)` +- apply top-level cross-cutting provisioning in a small number of explicit trailing steps + +Good pattern: + +```ts +const UserDependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer +) + +const UserLayer = Layer.provide(UserRepoLayer, UserDependencies) + +const BillingDependencies = Layer.mergeAll( + ConfigLayer, + LoggerLayer, + DatabaseLayer +) + +const BillingLayer = Layer.provide(BillingServiceLayer, BillingDependencies) + +const AppLayer = Layer.mergeAll( + UserLayer, + BillingLayer, + HttpLayer +).pipe( + Layer.provide(Telemetry), + Layer.provide(NodeSdk) +) +``` + +Why this style is preferred: + +- subsystem wiring stays local to the subsystem +- the final application layer reads as a high-level composition map +- cross-cutting concerns such as telemetry stay visible at the top level +- it avoids deeply nested inline layer expressions + +Avoid this style when a clearer local name would help: + +```ts +const AppLayer = Layer.provide( + Layer.mergeAll( + Layer.provide(UserRepoLayer, Layer.mergeAll(ConfigLayer, LoggerLayer)), + Layer.provide(BillingServiceLayer, Layer.mergeAll(ConfigLayer, LoggerLayer, DatabaseLayer)), + HttpLayer + ), + Telemetry +).pipe(Layer.provide(NodeSdk)) +``` + +That style is harder to read because: + +- subsystem composition is hidden inside the final assembly +- shared dependencies are harder to spot +- it is harder to refactor or reuse subsystem layers + +### `Layer.flatMap` + +Use `flatMap` when the next layer depends on the actual constructed service value, not just its type-level requirement. + +Example: + +```ts +const UserRepoLayerFromConfig = Layer.flatMap(ConfigLayer, (config) => + Layer.succeed(UserRepo)({ + getById: (id) => Effect.succeed({ id, name: config.apiBaseUrl }) + }) +) +``` + +This is more specialized than `merge` or `provide`. + +Prefer simpler composition first: + +- `merge` for combining +- `provide` for dependency satisfaction +- `flatMap` only when construction logic truly depends on the built value + +## Providing Layers To Effects + +## Preferred Rule + +Provide layers at boundaries. + +Usually that means: + +- the application entrypoint +- a subsystem entrypoint +- a test boundary + +Avoid repeatedly providing layers deep inside business logic unless you are deliberately isolating a subsystem. + +### Anti-Pattern: Local `Effect.provide` + +`Effect.provide` should be used only once at the entry of your program in normal application code. + +Bad pattern: + +```ts +const loadUser = (userId: string) => + Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById(userId) + }).pipe( + Effect.provide(UserRepoLayer) + ) +``` + +Why this is an anti-pattern: + +- it hides dependency wiring inside business logic +- it makes implementations harder to swap in tests +- it prevents clean top-level composition +- it encourages many small local runtimes instead of one coherent application graph +- it makes shared cross-cutting services harder to reason about + +Preferred pattern: + +```ts +const loadUser = (userId: string) => + Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById(userId) + }) + +const program = loadUser("u_123").pipe( + Effect.provide(AppLayer) +) +``` + +Rule of thumb: + +- business logic should require services +- composition should happen in layers +- `Effect.provide` should happen at the outermost entry boundary + +### Multiple Entry Points + +If your code integrates with a framework and has multiple entry points, prefer `ManagedRuntime` instead of repeatedly calling `Effect.provide` at many call sites. + +Typical examples: + +- HTTP handlers registered separately +- queue consumers +- cron jobs +- framework lifecycle hooks +- RPC handlers or worker callbacks + +Preferred pattern: + +```ts +const runtime = ManagedRuntime.make(AppLayer) + +const handleRequest = (id: string) => + runtime.runPromise(loadUser(id)) +``` + +Why: + +- the layer graph is still composed once +- services remain shared according to layer semantics +- the framework integration gets a stable runtime boundary +- resource lifecycle is explicit through `ManagedRuntime` + +Repo reference: + +- `./.repos/effect/packages/effect/src/ManagedRuntime.ts` + +## `Effect.provide` + +Use `Effect.provide` to satisfy an effect's dependencies with a layer or context. + +```ts +const program = loadUser("u_123").pipe( + Effect.provide(UserRepoLayerReady) +) +``` + +This is the main boundary provisioning operator. + +## `Effect.provideService` + +Use `provideService` for a single ad hoc implementation. + +```ts +const program = loadUser("u_123").pipe( + Effect.provideService(UserRepo, { + getById: (id) => Effect.succeed({ id, name: "Inline User" }) + }) +) +``` + +Good use cases: + +- small tests +- one-off overrides +- local customization + +Do not use this as the default replacement for real application layers. + +## `Effect.provideServiceEffect` + +Use `provideServiceEffect` when one service instance must be built effectfully without creating a reusable layer. + +This is useful for targeted overrides, but if the construction is reusable or part of application wiring, prefer a named `Layer.effect`. + +## Best Practices + +## 1. Keep service interfaces small and focused + +Prefer cohesive services over giant "everything" services. + +Good: + +- `UserRepo` +- `Mailer` +- `Clock`-like configuration or time abstractions + +Avoid: + +- large service shapes that mix unrelated responsibilities + +## 2. Prefer layers over manual wiring + +If construction has dependencies or effects, represent it as a layer. + +Avoid manually grabbing dependencies and assembling concrete objects all over the codebase. + +## 2.5 Prefer Effect-native integrations over raw runtime clients + +When Effect already provides a module for a capability, prefer the Effect-native integration over directly embedding a raw runtime client in service code. + +Examples: + +- prefer `effect/unstable/sql` modules over directly coupling business services to native SQL driver APIs +- prefer Effect HTTP modules over direct ad hoc request clients when the project is already using Effect HTTP abstractions + +Why: + +- resource handling, tracing, and errors stay inside the Effect model +- integrations compose better with layers and services +- observability and transactions are easier to keep consistent + +## 3. Keep business logic abstract over implementations + +Business functions should require services, not construct them. + +Good: + +```ts +const sendWelcomeEmail = (userId: string) => + Effect.gen(function*() { + const repo = yield* UserRepo + const user = yield* repo.getById(userId) + return user + }) +``` + +Avoid constructing `UserRepo` inside `sendWelcomeEmail`. + +## 4. Use `Layer.succeed` only for pure values + +Do not hide effectful initialization inside supposedly pure service objects. + +If initialization can fail, depends on effects, or needs scoped acquisition, use `Layer.effect`. + +## 5. Use `Layer.effect` for owned resources + +If the service opens something that must later close, model that lifecycle explicitly. + +This is one of the main reasons layers exist. + +## 6. Prefer top-level composition + +Compose major application layers once near the boundary. + +Good pattern: + +- define `ConfigLayer` +- define `UserRepoLayer` +- define `Dependencies = Layer.mergeAll(...)` +- define `AppLayer` separately with `Layer.provide(...)` or `Layer.provideMerge(...)` +- provide `AppLayer` to the top-level program + +## 7. Use `Layer.fresh` only when you really need a new instance + +Layers are shared by default. + +That is usually what you want. + +Use `Layer.fresh` only when you intentionally need to bypass sharing and rebuild the layer. + +## 7.5 Understand Layer Memoization + +Layers are memoized by reference. + +That means: + +- reusing the same layer value preserves memoization and sharing +- creating a new layer value creates a new memoization identity + +Because of this, functions that return layers should be avoided unless they are absolutely necessary. + +Prefer plain named layer constants over layer factory functions. + +Only use a function returning a layer when: + +- the layer genuinely depends on runtime parameters +- the caller truly needs distinct configurations or instances +- a constant layer value cannot express the construction cleanly + +Even in those cases, call the function once during construction and reuse the resulting layer value. + +### Function Dependencies Should Stay Unprovided + +If a dependency of a layer is itself represented by a function that returns a layer, do not call that function locally just to satisfy the dependency. + +Instead, leave that dependency unprovided and let it be supplied at the edge. + +Bad pattern: + +```ts +const UserLayer = Layer.provide(UserRepoLayer, makeDatabaseLayer(config)) +``` + +Why this is bad: + +- it creates a fresh layer reference locally +- it breaks or weakens memoization and sharing assumptions +- it hides an important construction dependency inside subsystem wiring +- it makes the final application graph harder to understand + +Preferred pattern: + +```ts +const UserLayer = UserRepoLayer + +const DatabaseLayer = makeDatabaseLayer(config) + +const AppLayer = Layer.provideMerge(UserLayer, DatabaseLayer) +``` + +More generally: + +- if a layer depends on a parameterized layer factory, keep that dependency in the required environment when possible +- construct the concrete parameterized layer once at the outer boundary +- provide it only at the edge where the full application graph is assembled + +Rule: + +- do not call layer-producing dependency functions deep inside subsystem composition +- keep those dependencies unprovided until the edge +- provide the concrete layer once in the final top-level assembly + +Bad pattern: + +```ts +const makeDatabaseLayer = () => Layer.effect(DatabaseService)(/* ... */) + +const AppLayer = Layer.mergeAll( + makeDatabaseLayer(), + makeDatabaseLayer() +) +``` + +Why this is bad: + +- each call creates a distinct layer reference +- memoization does not apply across those distinct references +- the underlying resource or service may be constructed more than once +- sharing assumptions become incorrect + +Preferred pattern: + +```ts +const DatabaseLayer = makeDatabaseLayer() + +const AppLayer = Layer.mergeAll( + DatabaseLayer, + OtherLayer +) +``` + +Rule: + +- avoid layer-producing functions unless they are truly necessary +- if a function returns a layer, call it once during construction and bind the result to a named constant +- reuse that layer value everywhere else + +This is especially important for: + +- database layers +- HTTP client layers +- telemetry layers +- queues, workers, and other resource-owning services + +If you intentionally need a distinct instance, make that explicit with a new layer value or `Layer.fresh`, rather than accidentally creating multiple instances by repeatedly calling a layer factory. + +## 8. Treat `Layer.orDie` carefully + +`Layer.orDie` converts layer construction failures into defects. + +Only use it when failure is truly unrecoverable at that boundary. + +Do not use it to hide legitimate configuration or infrastructure failures. + +## 9. Use `ManagedRuntime.make` at true runtime boundaries + +If you need a reusable runtime built from a layer, `ManagedRuntime.make` is the edge tool for that. + +Good use cases: + +- embedding Effect into external frameworks +- scripts or hosts that repeatedly run Effect programs + +Repo reference: + +- `./.repos/effect/packages/effect/src/ManagedRuntime.ts` + +## 10. Prefer explicit test layers + +For tests, prefer: + +- `Layer.succeed` for simple fakes +- `Layer.mock` for partial mocks when appropriate + +This keeps test wiring explicit and close to production composition style. + +## Recommended Patterns + +## Pattern: service definition plus live layer + +```ts +import { Context, Effect, Layer } from "effect" + +class Config extends Context.Service()("Config") {} + +class UserRepo extends Context.Service Effect.Effect<{ id: string; name: string }> +}>()("UserRepo") {} + +const ConfigLayer = Layer.succeed(Config)({ + apiBaseUrl: "https://api.example.com" +}) + +const UserRepoLayer = Layer.effect(UserRepo)( + Effect.gen(function*() { + const config = yield* Config + + return { + getById: (id) => + Effect.succeed({ + id, + name: `Loaded via ${config.apiBaseUrl}` + }) + } + }) +) + +const Dependencies = Layer.mergeAll(ConfigLayer) + +const AppLayer = Layer.provide(UserRepoLayer, Dependencies) +``` + +## Pattern: provide at the top level + +```ts +const program = Effect.gen(function*() { + const repo = yield* UserRepo + return yield* repo.getById("u_123") +}).pipe( + Effect.provide(AppLayer) +) +``` + +## Pattern: single-service override in tests + +```ts +const TestRepo = Layer.succeed(UserRepo)({ + getById: (id) => Effect.succeed({ id, name: "Test" }) +}) +``` + +## Anti-Patterns + +- constructing live services directly inside business logic +- using `Layer.succeed` for values that actually require effectful initialization +- providing the same large layer repeatedly throughout the call graph +- collapsing unrelated responsibilities into one service +- using `Layer.orDie` to hide normal initialization failures +- bypassing layers entirely for resource-owning services + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/Context.ts` +- `./.repos/effect/packages/effect/src/Layer.ts` +- `./.repos/effect/packages/effect/src/ManagedRuntime.ts` +- `./.repos/effect/packages/effect/src/Stream.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` +- `./.repos/effect/packages/effect/src/unstable/persistence/Persistence.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcSerialization.ts` +- `./.repos/effect/packages/effect/src/unstable/reactivity/Reactivity.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-observability.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-observability.md new file mode 100644 index 000000000..0483b7fd5 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-observability.md @@ -0,0 +1,724 @@ +# Observability Guide + +This guide is based on the vendored Effect source in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/Tracer.ts` +- `./.repos/effect/packages/effect/src/Logger.ts` +- `./.repos/effect/packages/effect/src/Metric.ts` +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` + +## Mental Model + +Observable Effect code should make business operations visible by default. + +That means: + +- business logic should show up clearly in stack traces +- important operations should produce spans +- logs should inherit execution context +- metrics should be attached at meaningful boundaries + +The most important best practice is: + +- prefer `Effect.fn(...)` whenever possible for business logic + +Why: + +- it adds stack frames +- it creates spans automatically +- it gives you better tracing and debugging for free +- it keeps business logic observable without extra boilerplate + +Repo reference: + +- `./.repos/effect/packages/effect/src/Effect.ts` + +## Preferred Rule + +Use `Effect.fn` as the default constructor for business-logic functions that return `Effect`. + +Prefer this: + +```ts +import { Effect } from "effect" + +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + return { id: userId, name: "Ada" } +}) +``` + +Over this: + +```ts +import { Effect } from "effect" + +const loadUser = (userId: string) => + Effect.gen(function*() { + return { id: userId, name: "Ada" } + }) +``` + +The second version works, but it throws away useful observability structure that `Effect.fn` gives you automatically. + +## Prefer `Effect.fn` Over Raw `Effect.gen` + +For business-logic definitions, prefer `Effect.fn` over writing raw `Effect.gen` directly, even when the operation takes no arguments. + +Prefer this: + +```ts +const refreshCache = Effect.fn("refreshCache")(function*() { + yield* Effect.logInfo("refreshing cache") +}) +``` + +Over this: + +```ts +const refreshCache = Effect.gen(function*() { + yield* Effect.logInfo("refreshing cache") +}) +``` + +Why: + +- `Effect.fn` gives the operation a clear observable identity +- stack traces are better +- tracing is more consistent +- the codebase gets a uniform shape for business operations + +Use raw `Effect.gen` when necessary, for example: + +- inline effect blocks inside another `Effect.fn` +- small one-off composition at call sites +- top-level assembly code where you are not defining a reusable business operation + +Rule of thumb: + +- reusable business operation: `Effect.fn` +- inline composition block: `Effect.gen` + +## `Effect.fn` vs `Effect.fnUntraced` + +### `Effect.fn` + +Use `Effect.fn` for almost all business logic. + +It is the preferred default because it adds: + +- stack frames +- tracing spans +- optional post-processing of the produced effect + +Example: + +```ts +import { Effect } from "effect" + +const createUser = Effect.fn("createUser")(function*(name: string) { + return { id: "u_123", name } +}) +``` + +Use this for: + +- domain operations +- application services +- handlers +- workflows +- orchestrations +- repository calls + +### `Effect.fnUntraced` + +Use `Effect.fnUntraced` only for edge cases. + +The vendored Effect repo itself uses `fnUntraced` in a number of low-level internals and integration helpers. That does not make it the default recommendation for downstream application or business code. + +If you do not want an explicit named span, prefer `Effect.fn` without a span name so you still keep stack traces and the normal traced-function behavior. + +Prefer this: + +```ts +const normalizeUser = Effect.fn(function*(input: string) { + return input.trim().toLowerCase() +}) +``` + +Over this: + +```ts +const normalizeUser = Effect.fnUntraced(function*(input: string) { + return input.trim().toLowerCase() +}) +``` + +Typical cases: + +- extremely hot low-level internal helpers +- very small internal combinators +- tight loops where you have measured overhead and need to reduce it + +Preferred rule: + +- `Effect.fn` by default +- `Effect.fn` without a span name when you want to avoid an explicit named span +- `Effect.fnUntraced` only with a concrete measured reason + +## Business Logic Patterns + +### Pattern: one named `Effect.fn` per meaningful operation + +Good: + +```ts +const parseCommand = Effect.fn("parseCommand")(function*(input: string) { + return input.trim() +}) + +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + return { id: userId, name: "Ada" } +}) + +const sendWelcomeEmail = Effect.fn("sendWelcomeEmail")(function*(userId: string) { + const user = yield* loadUser(userId) + return user.email +}) +``` + +Why this is good: + +- each operation has a clear span name +- traces reflect business concepts +- stack traces reflect the actual workflow + +### Pattern: use meaningful names + +Span names created through `Effect.fn` should represent business operations, not generic implementation detail. + +Prefer: + +- `loadUser` +- `chargeInvoice` +- `syncGithubInstallation` + +Avoid: + +- `helper` +- `run` +- `process` +- `step1` + +## Explicit Spans + +### `Effect.withSpan` + +Use `withSpan` when you need an explicit span around an effect that is not already naturally represented by a named `Effect.fn`, or when you want a nested sub-operation span. + +Example: + +```ts +import { Effect } from "effect" + +const syncUser = Effect.fn("syncUser")(function*(userId: string) { + const profile = yield* fetchProfile(userId).pipe( + Effect.withSpan("fetchProfile") + ) + + return yield* persistProfile(profile).pipe( + Effect.withSpan("persistProfile") + ) +}) +``` + +Use this when: + +- you want a nested span inside a larger operation +- you are instrumenting an existing effect pipeline +- you need more detailed trace structure than `Effect.fn` alone provides + +### `Effect.withSpanScoped` + +Use `withSpanScoped` when the span should remain open for the lifetime of a scope. + +This is less common in business logic and more common in long-lived resource or streaming workflows. + +### `Effect.withParentSpan` + +Use `withParentSpan` when integrating with an externally created span or continuing a parent span manually. + +This is useful in framework or interoperability boundaries. + +## Span Enrichment + +### `Effect.annotateCurrentSpan` + +Use `annotateCurrentSpan` to attach important structured fields to the current span. + +Example: + +```ts +import { Effect } from "effect" + +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + yield* Effect.annotateCurrentSpan({ userId }) + return { id: userId, name: "Ada" } +}) +``` + +Good span annotations: + +- stable identifiers +- domain-relevant keys +- request or resource identifiers +- small structured values + +Avoid: + +- giant payloads +- secrets +- noisy transient data with little diagnostic value + +## Logging Patterns + +### Use Effect logging inside effects + +Prefer: + +- `Effect.log` +- `Effect.logInfo` +- `Effect.logDebug` +- `Effect.logWarning` +- `Effect.logError` + +These integrate with the current Effect execution context. + +Example: + +```ts +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + yield* Effect.logDebug("loading user", { userId }) + return { id: userId, name: "Ada" } +}) +``` + +### `Effect.withLogSpan` + +Use `withLogSpan` when you want log messages to carry a local logical span label even when you are not creating a full tracing span. + +Example: + +```ts +const program = Effect.logInfo("starting sync").pipe( + Effect.withLogSpan("user-sync") +) +``` + +This is useful for: + +- log grouping +- quick local context +- correlation in plain log output + +### Logging Best Practices + +- log at business boundaries, not every tiny helper +- prefer structured values over concatenated strings +- keep logs high-signal +- avoid duplicate logs at every layer of the stack +- rely on spans plus a few well-placed logs, not log spam + +## Metrics Patterns + +### Track effects at meaningful boundaries + +Use metric tracking on meaningful operations such as: + +- requests +- jobs +- retries +- external calls +- queue handlers + +Repo reference: + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `Effect.track` + +Example: + +```ts +import { Effect, Metric } from "effect" + +const requests = Metric.counter("user_load_requests").pipe( + Metric.withConstantInput(1) +) + +const loadUser = Effect.fn("loadUser")( + function*(userId: string) { + return { id: userId, name: "Ada" } + }, + Effect.track(requests) +) +``` + +### Prefer boundary metrics over micro-metrics + +Good metrics are usually attached to: + +- endpoint handlers +- queue/job handlers +- repository operations +- external API boundaries + +Avoid putting a metric on every tiny internal helper. + +## OpenTelemetry Integration + +For real application observability, compose telemetry at the layer level. + +The vendored repo provides `@effect/opentelemetry` layers such as: + +- `NodeSdk.layer` +- `Tracer.layer` +- `Logger.layer` +- `Metrics.layer` + +Repo references: + +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` + +Preferred composition style: + +```ts +const AppLayer = Layer.mergeAll( + UserLayer, + BillingLayer, + HttpLayer +).pipe( + Layer.provide(Telemetry), + Layer.provide(NodeSdk) +) +``` + +Why: + +- business code stays observability-agnostic +- observability is configured once at the boundary +- spans, logs, and metrics remain consistent across the app + +## OpenTelemetry JS Framework Integration + +The vendored repo includes a real integration layer for the OpenTelemetry JavaScript ecosystem in `@effect/opentelemetry`. + +This is the preferred integration path when the application needs to participate in the standard OpenTelemetry JS framework, exporters, and SDKs. + +Relevant modules: + +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` +- `./.repos/effect/packages/opentelemetry/src/Metrics.ts` +- `./.repos/effect/packages/opentelemetry/src/Logger.ts` +- `./.repos/effect/packages/opentelemetry/src/Resource.ts` +- `./.repos/effect/packages/opentelemetry/src/WebSdk.ts` + +### Preferred Integration Model + +Use `@effect/opentelemetry` layers to bridge Effect observability into OpenTelemetry JS. + +Do not manually wire OpenTelemetry SDK objects inside business code. + +Prefer: + +- configuring tracer, metrics, logger, and resource layers once +- composing them into the application layer graph +- keeping business code written against Effect tracing, logging, and metrics APIs + +This means: + +- application code should keep using `Effect.fn`, `Effect.withSpan`, `Effect.log*`, and Effect metrics +- OpenTelemetry JS should be introduced at the infrastructure layer, not inside domain operations + +### `NodeSdk.layer` + +`NodeSdk.layer(...)` is the main Node.js integration entrypoint. + +From the vendored source, it accepts a configuration that can include: + +- span processors +- tracer config +- metric readers +- temporality preference +- log record processors +- logger provider config +- resource information such as service name and version +- shutdown timeout + +It then builds and merges: + +- resource layer +- tracer layer +- metrics layer +- logger layer + +This makes it the preferred high-level integration for Node applications. + +### Resource Configuration + +OpenTelemetry JS integration should define resource metadata explicitly. + +From `NodeSdk.layer`, the supported resource configuration includes: + +- `serviceName` +- `serviceVersion` +- additional attributes + +This is important because tracer and logger setup depend on the configured resource. + +Best practice: + +- always provide a meaningful service name +- provide service version when available +- use resource attributes for stable deployment or environment metadata + +### Tracing Integration + +The `Tracer` module bridges Effect spans into OpenTelemetry spans. + +Important integration points from the vendored source: + +- `Tracer.layer` +- `Tracer.layerGlobal` +- `Tracer.layerGlobalProvider` +- `Tracer.currentOtelSpan` +- `Tracer.makeExternalSpan` + +Use these when: + +- you need Effect tracing to export through OpenTelemetry JS +- you need to continue or bridge external trace context +- you need access to the current OpenTelemetry span object + +Best practice: + +- keep creating spans with Effect APIs in application code +- use the OpenTelemetry tracer layer to export and bridge them +- use `makeExternalSpan` or parent-span wiring only at integration boundaries + +### Metrics Integration + +The `Metrics` module connects Effect metrics to OpenTelemetry JS metric readers. + +Important details from the vendored implementation: + +- `Metrics.layer(...)` registers a producer against one or more metric readers +- it supports temporality preferences: + - `cumulative` + - `delta` +- it handles shutdown through scoped layer cleanup + +Best practice: + +- choose temporality based on the backend +- configure metric readers in the telemetry layer +- keep application code focused on recording Effect metrics, not exporter mechanics + +### Logger Integration + +The `Logger` module connects Effect logging to OpenTelemetry JS logs. + +Important details from the vendored implementation: + +- it maps Effect log levels to OpenTelemetry severity numbers +- it includes fiber ID, span context, log annotations, and log span timing in emitted attributes +- `Logger.layer({ mergeWithExisting })` can merge with or replace existing application loggers + +Best practice: + +- prefer merging with existing loggers unless there is a strong reason to replace them +- use Effect log annotations and log spans so the OpenTelemetry logger receives structured context automatically + +### Shutdown And Lifecycle + +The vendored layers use scoped acquisition and release for tracer providers, metric readers, and logger providers. + +This is the correct lifecycle model. + +Do not manually call provider shutdown methods from arbitrary business logic. + +Instead: + +- let the OpenTelemetry layers own provider lifecycle +- compose them into the application layer graph +- let the runtime or outer layer scope manage shutdown + +### External Trace Context + +When integrating with frameworks or inbound protocols that already carry trace context, prefer using the OpenTelemetry integration helpers rather than hand-rolling context propagation. + +The vendored tracer module provides: + +- `makeExternalSpan` +- `currentOtelSpan` + +Use these only at integration boundaries such as: + +- HTTP adapters +- RPC adapters +- worker or queue adapters + +Keep business operations oblivious to propagation mechanics. + +### Recommended Pattern + +Preferred architecture: + +1. business code uses Effect observability APIs +2. infrastructure composes `@effect/opentelemetry` layers +3. the final app layer provides telemetry once at the top level + +Example shape: + +```ts +const TelemetryLayer = NodeSdk.layer(() => ({ + resource: { + serviceName: "todo-service", + serviceVersion: "1.0.0" + }, + spanProcessor: mySpanProcessor, + metricReader: myMetricReader, + logRecordProcessor: myLogProcessor +})) + +const AppLayer = Layer.mergeAll( + DomainLayer, + HttpLayer +).pipe( + Layer.provide(TelemetryLayer) +) +``` + +This keeps: + +- app code portable +- OTel JS setup centralized +- shutdown semantics correct +- exported spans, logs, and metrics aligned + +### Anti-Patterns + +- constructing OpenTelemetry SDK clients directly inside business services +- mixing manual exporter setup into domain code +- bypassing Effect logging and tracing APIs in normal business operations +- scattering provider shutdown logic across the application +- configuring telemetry separately in many subsystems instead of one top-level layer + +## Anti-Patterns + +### Anti-Pattern: business logic built from anonymous `Effect.gen` functions everywhere + +Bad: + +```ts +const loadUser = (userId: string) => + Effect.gen(function*() { + return { id: userId, name: "Ada" } + }) +``` + +Why this is bad: + +- weaker tracing structure +- poorer stack traces +- less consistent naming in debugging output + +Preferred: + +```ts +const loadUser = Effect.fn("loadUser")(function*(userId: string) { + return { id: userId, name: "Ada" } +}) +``` + +### Anti-Pattern: using `Effect.fnUntraced` by default + +This throws away free observability. + +If you just do not want an explicit span name, use `Effect.fn` without a name instead. + +Only use `fnUntraced` when you have a specific low-level reason. + +### Anti-Pattern: logging without structure or context + +Bad: + +- giant interpolated strings +- duplicate logs at every layer +- logs with no business identifiers + +Prefer: + +- named operations via `Effect.fn` +- structured logs with IDs and context +- a few high-value logs at operation boundaries + +### Anti-Pattern: hand-instrumenting every helper with spans + +Do not create explicit spans everywhere just because you can. + +Preferred order: + +1. start with `Effect.fn` +2. add `Effect.withSpan` only where extra detail is actually useful +3. add metrics at meaningful boundaries + +## Recommended Patterns + +### Pattern: observable business operation + +```ts +import { Effect } from "effect" + +const fetchUser = Effect.fn("fetchUser")(function*(userId: string) { + yield* Effect.annotateCurrentSpan({ userId }) + yield* Effect.logDebug("fetching user", { userId }) + return { id: userId, name: "Ada" } +}) +``` + +### Pattern: orchestration with nested spans + +```ts +const syncUser = Effect.fn("syncUser")(function*(userId: string) { + const profile = yield* fetchRemoteProfile(userId).pipe( + Effect.withSpan("fetchRemoteProfile") + ) + + return yield* persistProfile(profile).pipe( + Effect.withSpan("persistProfile") + ) +}) +``` + +### Pattern: framework boundary with runtime + +```ts +const runtime = ManagedRuntime.make(AppLayer) + +const handleRequest = (userId: string) => + runtime.runPromise(fetchUser(userId)) +``` + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/Tracer.ts` +- `./.repos/effect/packages/effect/src/Logger.ts` +- `./.repos/effect/packages/effect/src/Metric.ts` +- `./.repos/effect/packages/opentelemetry/src/NodeSdk.ts` +- `./.repos/effect/packages/opentelemetry/src/Tracer.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-retries.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-retries.md new file mode 100644 index 000000000..3ef116fa6 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-retries.md @@ -0,0 +1,433 @@ +# Retries Guide + +This guide is based on retry patterns and `ExecutionPlan` usage in the vendored Effect repo. + +Key source files: + +- `./.repos/effect/packages/effect/src/Effect.ts` +- `./.repos/effect/packages/effect/src/Schedule.ts` +- `./.repos/effect/packages/effect/src/ExecutionPlan.ts` +- `./.repos/effect/packages/effect/test/Effect.test.ts` +- `./.repos/effect/packages/effect/test/ExecutionPlan.test.ts` + +Representative repo usage: + +- `./.repos/effect/packages/effect/src/unstable/workflow/Activity.ts` +- `./.repos/effect/packages/effect/src/unstable/workflow/WorkflowEngine.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClient.ts` +- `./.repos/effect/packages/effect/src/unstable/observability/OtlpExporter.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` + +## Mental Model + +Retries in Effect are not just loops. + +The repo uses three increasingly powerful levels: + +1. simple `Effect.retry` options for bounded or condition-based retries +2. `Schedule` for timing-aware retry policies +3. `ExecutionPlan` for fallback across different provided resources or layers + +Choose the smallest model that correctly expresses the retry policy. + +## Preferred Rule + +Prefer structured retry policies over ad hoc retry loops. + +Use: + +- simple `Effect.retry({ ... })` for straightforward conditions +- `Effect.retry(schedule)` when timing matters +- `ExecutionPlan` when retries should escalate across different layers or resources + +Avoid: + +- hand-written loops with mutable counters +- inline `catch` plus recursive retry logic +- resource fallback logic encoded as nested `catch` chains when `ExecutionPlan` is a better fit + +## `Effect.retry` + +`Effect.retry` is the main retry operator. + +The vendored tests show several important supported forms. + +## Retry On Success vs Failure + +`Effect.retry` only retries failures. + +If the effect succeeds, nothing is retried. + +This is explicitly covered in the module tests. + +## Simple Retry Options + +### `{ times: n }` + +Use this for the simplest bounded retry case. + +```ts +const retried = effect.pipe( + Effect.retry({ times: 3 }) +) +``` + +Use this when: + +- timing does not matter +- you only need a fixed retry count + +### `{ until: predicate }` + +Use `until` when retries should stop once the failure value satisfies a condition. + +The module tests show: + +- pure `until` +- effectful `until` +- that `until` is still evaluated at least once + +Example: + +```ts +const retried = effect.pipe( + Effect.retry({ until: (error) => error._tag === "Done" }) +) +``` + +### `{ while: predicate }` + +Use `while` when retries should continue only while the failure value satisfies a condition. + +The tests also show pure and effectful `while` variants. + +Example: + +```ts +const retried = effect.pipe( + Effect.retry({ while: (error) => error._tag === "Retryable" }) +) +``` + +## Retry With Schedule + +Use a `Schedule` whenever timing matters. + +```ts +const retried = effect.pipe( + Effect.retry(Schedule.recurs(3)) +) +``` + +Or with the richer object form: + +```ts +const retried = effect.pipe( + Effect.retry({ + schedule: Schedule.recurs(3), + while: (error) => error._tag === "Retryable" + }) +) +``` + +This is a very important repo pattern because it lets you combine: + +- retry timing +- retry limits +- retry predicates + +## Current Schedule Metadata During Retries + +The module tests show that retry execution updates `Schedule.CurrentMetadata`. + +This means retry policies and retry-aware effects can inspect: + +- attempt number +- elapsed time +- previous delay timing +- schedule output + +Use this when: + +- logging retry behavior +- building retry-aware diagnostics +- implementing advanced adaptive retry behavior + +## When To Use Simple Options Vs Schedule + +Prefer simple options when: + +- only the retry count matters +- retry timing does not matter +- the retry rule is just a condition on the error + +Prefer a schedule when: + +- retry timing matters +- backoff matters +- jitter matters +- the policy should evolve over time + +## Common Retry Schedules In The Repo + +The vendored repo repeatedly uses these patterns: + +### Fixed retry count + +```ts +Schedule.recurs(3) +``` + +### Exponential backoff + +```ts +Schedule.exponential(500, 1.5) +``` + +### Exponential plus steady fallback spacing + +```ts +Schedule.exponential(500, 1.5).pipe( + Schedule.either(Schedule.spaced(5000)) +) +``` + +This appears in production modules such as RPC and workflow code. + +### Error-sensitive delay policy + +```ts +Schedule.forever.pipe( + Schedule.addDelay((error) => Effect.succeed("1 second")) +) +``` + +The OTLP exporter uses this shape to derive delays from actual HTTP failure details such as rate limits. + +## Retry Only For Specific Failures + +The workflow `Activity` module shows an important advanced pattern: + +- sandbox the effect +- retry only when the `Cause` matches a specific retryable category +- fail or die differently once retries are exhausted + +Example shape from the repo: + +```ts +effect.pipe( + Effect.sandbox, + Effect.retry(policy), + Effect.catch((cause) => { + if (!Cause.hasInterrupts(cause)) { + return Effect.failCause(cause) + } + return Effect.die("interrupted and retries exhausted") + }) +) +``` + +Use this when: + +- retryability depends on the full cause, not just typed failures +- interrupt-specific retry behavior is required +- infrastructure policy is more nuanced than a simple tagged error rule + +## Retry Observability + +Retry logic should be observable. + +Good patterns: + +- keep retries inside named `Effect.fn` operations +- use `Schedule.CurrentMetadata` for diagnostics when needed +- log or annotate retry attempts at meaningful boundaries +- prefer central retry policies over duplicating timing logic everywhere + +Do not spread retry behavior across many small helpers where it becomes hard to see the operational policy. + +## `ExecutionPlan` + +Use `ExecutionPlan` when retries should escalate across different provided resources or layers. + +This is not just about retry timing. It is about retrying the same operation under different provided environments. + +The core use case from `ExecutionPlan.ts` is: + +- try one layer some number of times +- possibly with a schedule and conditions +- then fall back to another layer +- then possibly fall back again + +### What `ExecutionPlan` Solves + +`ExecutionPlan` is the right tool when: + +- the same effect should be retried against multiple alternative providers +- fallback should move across tiers, regions, models, or implementations +- retry policy includes both attempt counts and provider changes + +Examples: + +- fail over between multiple language model providers +- try one upstream cluster, then another +- fall back from a fast but unreliable service to a slower but more reliable one + +## `ExecutionPlan.make` + +Use `ExecutionPlan.make(...)` to define ordered retry/fallback steps. + +Each step can include: + +- `provide` +- `attempts` +- `while` +- `schedule` + +Example shape: + +```ts +const Plan = ExecutionPlan.make( + { + provide: FastLayer, + attempts: 2, + schedule: Schedule.spaced("3 seconds") + }, + { + provide: SafeLayer, + attempts: 3, + schedule: Schedule.spaced("1 second") + }, + { + provide: FinalFallbackLayer + } +) +``` + +### Step Semantics + +For each step: + +- `provide` is the context or layer to use +- `attempts` bounds how many times that step is tried +- `while` can stop retries for that step based on the input +- `schedule` defines the timing policy for retries within that step + +If `attempts` is omitted, the step attempts once unless a schedule is involved in a way that causes further retries. + +## `Effect.withExecutionPlan` And `Stream.withExecutionPlan` + +Use: + +- `Effect.withExecutionPlan` for effects +- `Stream.withExecutionPlan` for streams + +The vendored tests focus on `Stream.withExecutionPlan` and demonstrate: + +- fallback from one provider to another +- fallback after partial stream failure +- the ability to prevent fallback on partial streams + +This is a strong signal that `ExecutionPlan` is particularly useful for long-running or streaming integrations where failure can happen after partial success. + +## `ExecutionPlan.CurrentMetadata` + +`ExecutionPlan` exposes metadata with: + +- `attempt` +- `stepIndex` + +This is useful for: + +- diagnostics +- logging which fallback tier is being used +- understanding which plan step ultimately succeeded + +## `captureRequirements` + +`ExecutionPlan.captureRequirements` converts a plan with requirements into one whose requirements are satisfied from the current context. + +Use this when the plan should be frozen with the current environment before being applied later. + +## `ExecutionPlan.merge` + +Use `ExecutionPlan.merge(...)` when you need to concatenate multiple plans into one ordered plan. + +This is useful for assembling more complex fallback policies out of smaller ones. + +## When To Use `ExecutionPlan` Instead Of `Schedule` + +Use `Schedule` when: + +- only timing and retry conditions change +- the same environment/provider is used for every retry + +Use `ExecutionPlan` when: + +- the provider or layer should change across retry phases +- retries are tied to alternative resources, not just delays +- fallback is part of dependency provisioning strategy + +## Recommended Patterns + +### Pattern: simple bounded retry + +```ts +const retried = effect.pipe( + Effect.retry({ times: 3 }) +) +``` + +### Pattern: retryable-error backoff + +```ts +const retryPolicy = Schedule.exponential(500, 1.5).pipe( + Schedule.either(Schedule.spaced(5000)) +) + +const retried = effect.pipe( + Effect.retry({ + schedule: retryPolicy, + while: (error) => error._tag === "Retryable" + }) +) +``` + +### Pattern: fallback across providers + +```ts +const Plan = ExecutionPlan.make( + { + provide: PrimaryLayer, + attempts: 2, + schedule: Schedule.spaced("1 second") + }, + { + provide: SecondaryLayer, + attempts: 3, + schedule: Schedule.exponential(500, 1.5) + }, + { + provide: FinalFallbackLayer + } +) +``` + +## Anti-Patterns + +- hand-writing retry recursion instead of using `Effect.retry` +- embedding sleep and counters directly in business logic +- using `ExecutionPlan` when a simple `Schedule` is enough +- encoding provider fallback as a maze of nested `catch` branches +- retrying indiscriminately without checking whether the failure is actually retryable + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/test/Effect.test.ts` +- `./.repos/effect/packages/effect/src/Schedule.ts` +- `./.repos/effect/packages/effect/src/ExecutionPlan.ts` +- `./.repos/effect/packages/effect/test/ExecutionPlan.test.ts` +- `./.repos/effect/packages/effect/src/unstable/workflow/Activity.ts` +- `./.repos/effect/packages/effect/src/unstable/workflow/WorkflowEngine.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClient.ts` +- `./.repos/effect/packages/effect/src/unstable/observability/OtlpExporter.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-schedule.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-schedule.md new file mode 100644 index 000000000..2f44ca0d4 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-schedule.md @@ -0,0 +1,379 @@ +# Schedule Guide + +This guide is based on the vendored `Schedule` module and its usage across `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Schedule.ts` +- `./.repos/effect/packages/effect/test/Schedule.test.ts` + +Representative repo usage: + +- `./.repos/effect/packages/effect/src/unstable/workflow/WorkflowEngine.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClient.ts` +- `./.repos/effect/packages/effect/src/unstable/observability/OtlpExporter.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` + +## Mental Model + +`Schedule` is the standard Effect abstraction for: + +- retries +- repeats +- polling +- backoff +- cadence and timing policies + +A schedule describes when the next step should happen and when execution should stop. + +The repo uses schedules heavily for: + +- retry policies +- recurring work +- time-window alignment +- cron-based triggering +- bounded flaky test retries + +## Preferred Rule + +When the timing behavior of an effect matters, prefer expressing it with `Schedule` rather than ad hoc loops, counters, sleeps, or manual retry recursion. + +Prefer: + +- `Effect.retry(schedule)` +- `Effect.repeat(schedule)` +- `Effect.schedule(schedule)` +- `Stream.fromSchedule(schedule)` + +Over: + +- custom retry loops with mutable counters +- `Effect.forever` plus hand-written `Effect.sleep` +- manual backoff code scattered across business logic + +## Common Repo Patterns + +The most common patterns in the vendored repo are: + +1. `Schedule.recurs(n)` for bounded retries or repeats +2. `Schedule.spaced(...)` for simple fixed spacing +3. `Schedule.fixed(...)` or `Schedule.windowed(...)` for interval-aligned work +4. `Schedule.exponential(...)` for backoff +5. `Schedule.either(...)` or sequencing combinators to combine retry policies +6. `Schedule.while(...)` to stop based on metadata or input +7. `Schedule.addDelay(...)` for custom backoff logic +8. `Schedule.jittered(...)` to avoid retry stampedes + +## Retry Vs Repeat + +Use schedules with the right operator: + +- `Effect.retry(schedule)` for failures +- `Effect.repeat(schedule)` for successes +- `Effect.schedule(schedule)` when you want to delay/reschedule an effect directly + +Good rule of thumb: + +- failing workflow: `retry` +- recurring successful workflow: `repeat` +- one effect that should run on a cadence: `schedule` + +## Core Constructors + +### `Schedule.recurs` + +Use `recurs(n)` for a bounded number of additional runs. + +```ts +const retryPolicy = Schedule.recurs(3) +``` + +This is one of the most common repo retry policies. + +### `Schedule.forever` + +Use `forever` when the schedule should never terminate on its own. + +```ts +const retryForever = Schedule.forever +``` + +This is common in infrastructure code and long-running retry loops. + +### `Schedule.spaced` + +Use `spaced(duration)` for simple constant spacing. + +```ts +const pollEverySecond = Schedule.spaced("1 second") +``` + +This is the most straightforward schedule for polling or retry spacing. + +### `Schedule.fixed` + +Use `fixed(duration)` when work should align to fixed interval boundaries. + +The tests show that this differs from simple spacing when the action itself takes time. + +Use it when interval alignment matters more than naïve spacing. + +### `Schedule.windowed` + +Use `windowed(duration)` when you want delays to align to the nearest window boundary. + +This is useful for periodic flush or batching behavior. + +### `Schedule.duration` + +Use `duration(duration)` for a one-shot delay schedule. + +```ts +const onceAfterOneSecond = Schedule.duration("1 second") +``` + +### `Schedule.cron` + +Use `cron(...)` for calendar-based scheduling. + +The module tests cover: + +- minute-level cron +- second-level cron +- calendar matching for specific weekdays and month days + +Use this for: + +- jobs that should follow wall-clock time +- operational schedules +- calendar-driven execution + +## Backoff Patterns + +### `Schedule.exponential` + +Use `exponential(base, factor)` for retry backoff. + +This is a dominant repo pattern. + +Examples from production code: + +- `WorkflowEngine` +- `RpcClient` +- persistence and eventlog modules + +Typical pattern: + +```ts +const retryPolicy = Schedule.exponential(500, 1.5) +``` + +Use this for: + +- network retries +- external system retries +- contention or lock retries + +### Combine exponential with a cap or fallback spacing + +The repo often combines exponential backoff with a more stable spaced fallback using `Schedule.either(...)`. + +Example pattern from production modules: + +```ts +const retryPolicy = Schedule.exponential(500, 1.5).pipe( + Schedule.either(Schedule.spaced(5000)) +) +``` + +This keeps early retries responsive without letting delays grow without bound. + +### `Schedule.jittered` + +Use `jittered(...)` to randomize timing within safe bounds. + +The module tests verify jittered delays remain within a bounded percentage of the original schedule. + +Use it when: + +- many workers or clients may retry at once +- you want to avoid synchronized retry storms +- the system would suffer from coordinated polling spikes + +## Combinators + +### `Schedule.while` + +Use `while(...)` to continue only while a predicate on schedule metadata holds. + +The repo uses this for: + +- stopping after a number of attempts +- filtering retries based on the input error or cause +- bounding retry windows by elapsed time + +Example pattern: + +```ts +const bounded = Schedule.spaced("1 second").pipe( + Schedule.while(({ attempt }) => Effect.succeed(attempt <= 5)) +) +``` + +### `Schedule.andThenResult` + +Use `andThenResult(left, right)` when one schedule should run to completion and then another should take over. + +The module tests show this clearly. + +Use this when: + +- you want an initial aggressive policy followed by a slower steady-state policy +- you want phase-based retry or repeat timing + +### `Schedule.either` + +Use `either` to combine two schedules so both policies influence the resulting timing. + +This appears frequently in repo retry policies that combine exponential growth with a stable fallback cadence. + +### `Schedule.addDelay` + +Use `addDelay` when the delay should depend on the schedule input or output. + +This is a strong fit for custom retry behavior based on the actual error. + +The OTLP exporter uses this style to honor `retry-after` behavior and otherwise fall back to a default delay. + +Example shape: + +```ts +const policy = Schedule.forever.pipe( + Schedule.addDelay((error) => + Effect.succeed("1 second") + ) +) +``` + +Use this when: + +- the delay should depend on the error +- upstream metadata such as rate-limit headers matters +- you need custom backoff without leaving the Schedule model + +## Metadata + +Schedules expose rich metadata including: + +- input +- attempt count +- start time +- current time +- elapsed time +- elapsed since previous run +- output +- duration + +This is one of the reasons schedules are better than ad hoc retry loops. + +Use metadata when: + +- retry behavior depends on the input error +- stop conditions depend on elapsed time +- you want to log or collect retry state + +## Collection And Inspection Helpers + +The module tests highlight several useful helpers: + +- `Schedule.collectInputs(...)` +- `Schedule.collectOutputs(...)` +- `Schedule.collectWhile(...)` +- `Schedule.delays(...)` +- `Schedule.reduce(...)` + +Use these when: + +- you need to inspect or test a schedule +- you want to accumulate state across schedule steps +- you are building a more specialized scheduling policy + +These are especially useful in tests and low-level policy construction. + +## Typical Policies + +### Simple bounded retry + +```ts +const retryPolicy = Schedule.recurs(3) +``` + +### Spaced polling + +```ts +const pollPolicy = Schedule.spaced("5 seconds") +``` + +### Exponential retry with a stable fallback cadence + +```ts +const retryPolicy = Schedule.exponential(500, 1.5).pipe( + Schedule.either(Schedule.spaced("5 seconds")) +) +``` + +### Retry forever with custom delay logic + +```ts +const retryPolicy = Schedule.forever.pipe( + Schedule.addDelay((error) => Effect.succeed("1 second")) +) +``` + +### Cron-driven recurring job + +```ts +const nightly = Schedule.cron("0 0 * * *") +``` + +## Testing Schedules + +The repo tests schedules with `TestClock` and controlled stepping. + +Preferred pattern: + +- use `TestClock` +- advance time explicitly +- inspect emitted delays or outputs + +This keeps schedule tests deterministic. + +The module tests also use helpers built on `Schedule.toStepWithSleep(...)` to inspect schedule behavior precisely. + +## Best Practices + +1. Prefer `Schedule` over ad hoc retry loops. +2. Prefer `Schedule.recurs(...)` for simple bounded retries. +3. Prefer `Schedule.exponential(...)` for backoff. +4. Prefer `Schedule.either(...)` or sequencing combinators to compose retry phases. +5. Prefer `Schedule.addDelay(...)` when delay depends on the actual error. +6. Prefer `Schedule.jittered(...)` for distributed retry behavior. +7. Prefer metadata-driven stop conditions over mutable counters. +8. Prefer testing schedules with `TestClock`. + +## Anti-Patterns + +- hand-writing retry loops with mutable counters and sleeps +- putting backoff logic directly in business code instead of in a schedule +- using `Effect.forever` with embedded `sleep` as a substitute for a schedule +- scattering retry timing logic across many call sites +- ignoring jitter when many clients or workers retry concurrently + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/Schedule.ts` +- `./.repos/effect/packages/effect/test/Schedule.test.ts` +- `./.repos/effect/packages/effect/src/unstable/workflow/WorkflowEngine.ts` +- `./.repos/effect/packages/effect/src/unstable/rpc/RpcClient.ts` +- `./.repos/effect/packages/effect/src/unstable/observability/OtlpExporter.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-schema.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-schema.md new file mode 100644 index 000000000..3d45cfa84 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-schema.md @@ -0,0 +1,624 @@ +# Schema Guide + +This guide is based on the vendored Schema module and common repo usage in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/Schema.ts` +- `./.repos/effect/packages/effect/src/SchemaTransformation.ts` +- `./.repos/effect/packages/effect/src/SchemaGetter.ts` +- `./.repos/effect/packages/effect/src/SchemaIssue.ts` +- `./.repos/effect/packages/effect/src/JsonSchema.ts` + +Representative repo usage: + +- `./.repos/effect/packages/tools/ai-codegen/src/Config.ts` +- `./.repos/effect/packages/platform-node/test/fixtures/rpc-schemas.ts` +- `./.repos/effect/packages/platform-browser/test/IndexedDbQueryBuilder.test.ts` +- `./.repos/effect/packages/tools/openapi-generator/` + +## Mental Model + +Schema is the standard way to: + +- define data shapes +- validate unknown input +- encode typed values back to serialized form +- transform between encoded and decoded representations +- attach metadata and constraints + +The repo uses Schema pervasively for: + +- protocol payloads +- configuration +- HTTP and RPC contracts +- database row decoding +- error types +- derived tooling such as JSON Schema and arbitrary generation + +## Preferred Rule + +Prefer Schema-based types whenever data crosses a boundary or should be validated, transformed, documented, or encoded. + +Typical boundaries: + +- HTTP requests and responses +- RPC payloads +- database rows +- config files +- worker messages +- persisted data +- domain errors + +## What A Schema Actually Is + +A schema is not just a static shape. + +It is a contract between: + +- the decoded in-memory value you want to work with +- the encoded representation that comes from or goes to some boundary + +This is the most important thing many implementations get wrong. + +Do not think of Schema as “a typed struct definition.” +Think of it as: + +- validation +- decoding +- encoding +- transformation +- metadata +- reuse across boundaries + +Because of that, schemas should not be duplicated unless there is a real semantic difference. + +If two schemas describe the same logical model but differ only because one boundary encodes a field differently, prefer one schema with transformations instead of two parallel schemas. + +## Avoid Duplicating Schemas + +Do not create multiple parallel schemas for the same logical entity unless they truly represent different models. + +Bad pattern: + +```ts +const Todo = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Schema.Boolean +}) + +const TodoSql = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Schema.BooleanFromBit +}) +``` + +This is usually a sign that transformations are not being used properly. + +If the model is still “Todo”, do not define a second schema just because one boundary stores `completed` as a bit. + +Prefer deriving or transforming the representation instead. + +Why duplication is bad: + +- the same model is now maintained in multiple places +- fields drift over time +- boundary logic gets copied instead of centralized +- refactors become error-prone + +Only duplicate schemas when there is a real semantic difference, for example: + +- a creation payload really is a different model from a persisted entity +- a public API contract intentionally differs from an internal domain model +- a projection or partial view is intentionally a different type + +If the difference is only encoding, use a transformation. + +## Prefer `Class` Variants Over `Struct` Variants When Possible + +When a schema represents a named domain model, reusable payload, or long-lived API shape, prefer `Schema.Class`, `Schema.TaggedClass`, or `Schema.TaggedErrorClass` over a bare `Schema.Struct`. + +Prefer: + +```ts +import { Schema } from "effect" + +export class User extends Schema.Class("User")({ + id: Schema.String, + name: Schema.String +}) {} +``` + +Over: + +```ts +import { Schema } from "effect" + +export const User = Schema.Struct({ + id: Schema.String, + name: Schema.String +}) +``` + +Why `Class` variants are usually better: + +- the schema has a stable, named identity +- reusable models are easier to recognize in code and traces +- constructors and validation are packaged together +- extension patterns are clearer +- named schemas read better in contracts and tooling output + +Use `Struct` when: + +- the shape is local and anonymous +- it is a small inline request or response shape +- introducing a class would add unnecessary ceremony +- the schema is primarily a one-off composition fragment + +Good rule of thumb: + +- reusable named model: `Class` +- reusable tagged union member: `TaggedClass` +- reusable error payload: `TaggedErrorClass` +- small inline object shape: `Struct` + +## One Logical Model, Multiple Representations + +The right Schema mindset is: + +- one logical model +- multiple encoded forms when needed +- transformations connecting them + +For example, a `Todo` may be: + +- a boolean in memory +- a bit in SQL +- a string in some external API + +That does not automatically mean you need three separate top-level schemas. + +Prefer: + +- one main schema for the logical model +- transformed field schemas or transformed object schemas for boundary-specific encoding +- derived request/result schemas when the shape is actually different + +## Common Schema Building Blocks + +Common primitives and collections used throughout the repo: + +- `Schema.String` +- `Schema.Number` +- `Schema.Boolean` +- `Schema.BigInt` +- `Schema.Array(...)` +- `Schema.Record(key, value)` +- `Schema.Tuple([...])` +- `Schema.Struct({...})` +- `Schema.Union([...])` + +Example: + +```ts +const Todo = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Schema.Boolean +}) +``` + +## `Class`, `TaggedClass`, and `TaggedErrorClass` + +### `Schema.Class` + +Use for named reusable schema-backed models. + +```ts +class Product extends Schema.Class("Product")({ + id: Schema.String, + price: Schema.Number +}) {} +``` + +### Constructor Rule + +When constructing schema classes, prefer `X.make(...)` over `new X(...)`. + +Prefer: + +```ts +const todo = Todo.make({ + id: 1, + title: "write docs", + completed: false +}) +``` + +Over: + +```ts +const todo = new Todo({ + id: 1, + title: "write docs", + completed: false +}) +``` + +Why: + +- it is the intended schema-class construction style +- it makes schema-backed construction explicit +- it keeps the codebase consistent +- it reads better across `Class`, `TaggedClass`, and `TaggedErrorClass` + +Use this rule consistently for: + +- `Schema.Class` +- `Schema.TaggedClass` +- `Schema.TaggedErrorClass` + +### `Schema.TaggedClass` + +Use for members of tagged unions. + +```ts +class Circle extends Schema.TaggedClass()("Circle", { + radius: Schema.Number +}) {} + +class Rectangle extends Schema.TaggedClass()("Rectangle", { + width: Schema.Number, + height: Schema.Number +}) {} +``` + +### `Schema.TaggedErrorClass` + +Use for schema-backed typed errors. + +```ts +class NotFound extends Schema.TaggedErrorClass()("NotFound", { + id: Schema.String +}) {} +``` + +## Optional Fields + +Be precise about optionality. + +Important rule from the vendored docs: + +- `Schema.optional(schema)` means `T | undefined` +- `Schema.optionalKey(schema)` means an exact optional property in a struct + +Prefer `optionalKey` for object fields. + +Prefer: + +```ts +const Query = Schema.Struct({ + search: Schema.optionalKey(Schema.String) +}) +``` + +Use `optional` when the value itself should be `A | undefined`, not just an omitted field. + +## Unions + +Use `Schema.Union([...])` for ordinary unions. + +```ts +const Id = Schema.Union([ + Schema.String, + Schema.Number +]) +``` + +Prefer tagged unions for domain variants. + +```ts +class Created extends Schema.TaggedClass()("Created", { + id: Schema.String +}) {} + +class Deleted extends Schema.TaggedClass()("Deleted", { + id: Schema.String +}) {} + +const TodoEvent = Schema.Union([Created, Deleted]) +``` + +Why: + +- decoding and branching are clearer +- `_tag`-based matching aligns with Effect code style + +## Recursive Schemas + +Use `Schema.suspend` for recursive schemas. + +```ts +type Tree = { + readonly name: string + readonly children: ReadonlyArray +} + +const Tree: Schema.Schema = Schema.Struct({ + name: Schema.String, + children: Schema.Array(Schema.suspend((): Schema.Schema => Tree)) +}) +``` + +Use it whenever a schema refers to itself, directly or indirectly. + +Without `suspend`, recursive definitions will not work correctly. + +## Transformations + +Transformations are one of the most important Schema features. + +Use them when decoded and encoded shapes differ. + +This is the main tool that avoids needless schema duplication. + +If your instinct is “I need another schema because this boundary encodes the same value differently”, stop and first ask whether this should be one schema with a transformation instead. + +### `Schema.decodeTo` + +Use `decodeTo` when you want one schema to decode into another schema's type. + +```ts +const TrimmedString = Schema.String.pipe( + Schema.decodeTo(Schema.String, { + decode: (value) => value.trim(), + encode: (value) => value + }) +) +``` + +The vendored docs explicitly note that `decodeTo` is curried and should be used with `pipe`. + +### `Schema.encodeTo` + +Use `encodeTo` when the reverse direction reads more clearly. + +### `SchemaTransformation.transformOrFail` + +Use `transformOrFail` when the transformation itself is effectful or may fail. + +```ts +import * as Effect from "effect/Effect" +import * as SchemaTransformation from "effect/SchemaTransformation" + +const VerifiedString = Schema.String.pipe( + Schema.decodeTo( + Schema.String, + SchemaTransformation.transformOrFail({ + decode: (value) => Effect.succeed(value.trim()), + encode: (value) => Effect.succeed(value) + }) + ) +) +``` + +Use this when: + +- validation depends on services or effects +- decoding can fail with structured issues +- encoding also needs logic beyond identity + +## Field-Level Transformations + +Very often, the right answer is not a second object schema but a transformed field schema. + +Example shape: + +```ts +const Completed = Schema.BooleanFromBit + +const Todo = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Completed +}) +``` + +In this pattern: + +- the logical model still has `completed: boolean` +- the encoded SQL-facing representation can still be a bit +- the transformation lives at the field where it belongs + +This is usually better than defining `Todo` and `TodoSql` as separate object schemas. + +## Object-Level Transformations + +Use object-level transformations when the whole object encoding differs, not just one field. + +Good use cases: + +- external keys differ from internal keys +- several fields need coordinated transformation +- the encoded shape is a structurally different representation of the same model + +Still prefer a single logical schema plus a transformation pipeline over maintaining multiple duplicated top-level schemas. + +## Rename Keys + +Schema supports key renaming through struct transformations. + +The vendored `Schema.ts` implements key renaming by mapping fields and using decode/encode transformations with renamed key maps. + +Use key renaming when: + +- external payload keys differ from internal keys +- you want stable internal names while honoring external contract names + +Preferred pattern: + +- keep the internal decoded shape idiomatic +- use schema-level transformation or field-mapping to adapt external keys + +This is another example of avoiding duplication. If the only difference is key naming, do not define a second schema just to rename fields manually later. + +In practice, use struct field mapping helpers and transformation composition rather than manual post-parse object rewriting. + +## Opaque And Branded Types + +Use opaque or branded schemas when a value should stay distinct from its structural base type. + +### `Schema.brand` + +Use `brand` for refined nominal distinctions. + +```ts +const UserId = Schema.String.pipe( + Schema.brand("UserId") +) +``` + +This is useful for: + +- IDs +- validated domain scalars +- preventing accidental interchange of same-shaped values + +### `Schema.Opaque` + +Use `Opaque` when you want an opaque schema-backed type with the same structure as its underlying schema. + +This is especially useful when the type should remain distinct at the type level without changing its runtime shape. + +## Picking, Omitting, Partial Shapes, And Mutability + +Common struct operations include: + +- `pick` +- `omit` +- `partial` +- `mutable` + +Use them to derive variations instead of redefining near-identical schemas manually. + +Good examples: + +- request subset from a domain model +- patch/update payloads +- mutable representations for specific adapters + +Prefer deriving from one source schema rather than maintaining parallel copies. + +This is the second major tool for avoiding duplication: + +- use transformations when encoded and decoded representations differ +- use derivation when one schema is a subset, superset, or variation of another + +## Constraints And Validation + +Use schema checks and filters for validation. + +Examples from the module docs include: + +- `isMinLength` +- `isGreaterThan` +- `isPattern` +- `isUUID` + +Attach them with `.check(...)`. + +Use this when: + +- the validation is intrinsic to the schema +- the rule belongs to the data contract + +For business-rule validation that depends on services or current state, prefer effectful logic outside the schema or use effectful transformations. + +## Decoding And Encoding + +Common operations: + +- `Schema.decodeUnknownSync` +- `Schema.decodeUnknownEffect` +- `Schema.decodeUnknownExit` +- `Schema.encodeUnknownSync` +- `Schema.encodeUnknownEffect` + +Preferred rule: + +- use `decodeUnknownEffect` and `encodeUnknownEffect` in Effect code +- avoid throwing sync decode APIs in application flows unless you are intentionally at a sync boundary + +Good pattern: + +```ts +const decodeUser = Schema.decodeUnknownEffect(User) +``` + +## Schema Metadata And Derived Tooling + +Schema is also used for: + +- annotations and documentation metadata +- JSON Schema generation +- arbitrary generation for tests +- derived equivalence + +Useful operations from the module docs: + +- `.annotate(...)` +- `Schema.toJsonSchemaDocument(...)` +- `Schema.toArbitrary(...)` +- `Schema.toEquivalence(...)` + +Use annotations when the schema participates in: + +- API docs +- codegen +- contract generation + +## Common Repo Patterns + +Patterns visible in the vendored repo: + +- `Schema.Class` for named reusable contract types +- `Schema.Struct` for inline shapes and anonymous fragments +- `Schema.Union` for alternative payloads +- `Schema.optionalKey` for request/query/body optional fields +- `Schema.suspend` for recursive generated schemas +- `Schema.decodeTo` and `transformOrFail` for non-trivial decode/encode logic +- `Schema.TaggedErrorClass` for typed error payloads + +## Best Practices + +1. Prefer `Class` variants over plain `Struct` for named reusable schemas. +2. Prefer tagged variants for unions and errors. +3. Prefer `optionalKey` for optional object properties. +4. Do not duplicate schemas unless there is a real semantic difference. +5. Prefer schema-level transformations over ad hoc post-parse object rewriting. +6. Prefer deriving schema variants with `pick`, `omit`, `partial`, and `mutable` instead of duplicating definitions. +7. Prefer field-level transformations when only a field encoding differs. +8. Prefer branded or opaque types for important domain identifiers. +9. Prefer `decodeUnknownEffect` in application code. +10. Keep internal decoded shapes idiomatic and use schema transforms for external representation differences. + +## Anti-Patterns + +- using plain `Struct` for every reusable domain model even when `Class` would give a clearer named type +- duplicating whole schemas when only one field encoding differs +- creating `Foo` and `FooSql` schemas for the same logical model when a transformation would do +- using `optional` when you actually want an optional key +- duplicating near-identical schemas instead of deriving variants +- rewriting keys manually after decode instead of using schema transformations +- hand-validating external data after decode when the constraint belongs in the schema +- exposing unvalidated external payloads deep into business logic + +## Good Repo Examples To Study + +- `./.repos/effect/packages/tools/ai-codegen/src/Config.ts` +- `./.repos/effect/packages/platform-node/test/fixtures/rpc-schemas.ts` +- `./.repos/effect/packages/platform-browser/test/IndexedDbQueryBuilder.test.ts` +- `./.repos/effect/packages/tools/openapi-generator/src/JsonSchemaGenerator.ts` +- `./.repos/effect/packages/effect/src/Schema.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-sql.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-sql.md new file mode 100644 index 000000000..ccaa8d0a8 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-sql.md @@ -0,0 +1,536 @@ +# SQL Guide + +This guide is based on the vendored Effect SQL modules in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/Migrator.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlResolver.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlSchema.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlModel.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` + +## Preferred Rule + +When a project uses Effect, prefer the Effect SQL modules over directly coupling business code to a native SQL driver API. + +Prefer: + +- `effect/unstable/sql/SqlClient` +- `effect/unstable/sql/Migrator` +- `effect/unstable/sql/SqlResolver` +- `effect/unstable/sql/SqlSchema` +- `effect/unstable/sql/SqlModel` + +Over: + +- embedding raw driver calls directly in business services +- hand-rolling transactions in service methods +- ad hoc migration scripts disconnected from the Effect runtime + +Why: + +- transactions are integrated into the Effect model +- spans and SQL observability are built in +- SQL errors stay typed and consistent +- schema decoding and request resolution compose better +- layers and services stay portable across runtimes and tests + +## Mental Model + +The Effect SQL stack is organized around: + +- `SqlClient` as the main database capability +- `withTransaction` for transaction boundaries +- `SqlResolver` for request-style batched and validated access +- `SqlSchema` and `SqlModel` for schema-aware query/model patterns +- `Migrator` for managed migrations + +The business layer should depend on Effect SQL abstractions, not on a raw driver object. + +## `SqlClient` + +`SqlClient` is the primary service for executing SQL. + +Repo reference: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` + +Use it when: + +- you need to execute queries +- you need transactions +- you want SQL operations to participate in Effect spans and context + +Important capabilities from the repo: + +- transaction support with `withTransaction` +- connection reservation with `reserve` +- reactive queries +- integration with transaction-scoped context + +## Typed Queries + +Prefer typed SQL queries instead of leaving row shapes implicit. + +The repo uses typed query literals like: + +```ts +const rows = yield* sql<{ id: number; name: string }>`SELECT * FROM test` +``` + +This is the first level of typed SQL usage and is already better than untyped row access. + +Use typed query literals when: + +- the row shape is small and obvious +- the query is local and does not justify a reusable schema +- you want immediate row typing without introducing extra helpers + +Avoid: + +- leaving query results untyped and then recovering shape with unsafe assertions +- using `as` on rows after query execution + +## Schema Integration + +Prefer integrating SQL with Schema whenever the row shape matters or the query result crosses a meaningful boundary. + +Why: + +- Schema validates the shape rather than trusting the database blindly +- row decoding stays explicit and typed +- the same schema can often be reused for transport, domain, or contract layers +- this avoids unsafe row assertions such as `as TodoRow` + +### Prefer schema-decoded results over `as`-based rows + +Avoid this pattern: + +```ts +const row = yield* sql`SELECT id, title FROM todos WHERE id = ${id}` +const todo = row[0] as TodoRow +``` + +Prefer: + +- a typed SQL query plus Schema decoding +- or a SQL helper such as `SqlResolver`, `SqlSchema`, or `SqlModel` when appropriate + +Example boundary decode: + +```ts +const TodoRow = Schema.Struct({ + id: Schema.Number, + title: Schema.String, + completed: Schema.Boolean +}) + +const decodeTodoRows = Schema.decodeUnknownEffect(Schema.Array(TodoRow)) +``` + +Then keep the query and decoding together in one SQL-aware operation. + +### `SqlResolver` + Schema + +`SqlResolver` is one of the clearest examples of SQL and Schema integration in the vendored repo. + +It uses: + +- a `Request` schema for validating resolver input +- a `Result` schema for validating query output + +Example shape from the repo tests: + +```ts +const resolver = SqlResolver.findById({ + Id: Schema.Number, + Result: Schema.Struct({ + id: Schema.Number, + name: Schema.String + }), + ResultId: (row) => row.id, + execute: (ids) => sql`SELECT * FROM test WHERE id IN ${sql.in(ids)}` +}) +``` + +This is a preferred pattern when: + +- multiple IDs are fetched together +- request batching is useful +- the query contract should be schema-validated + +### `SqlSchema` and `SqlModel` + +Use `SqlSchema` and `SqlModel` when you want tighter schema integration with SQL itself. + +They are preferred over hand-written row types when: + +- the row shape is central to the module +- you want reusable schema-aware model logic +- manual row mapping is becoming repetitive + +## Prefer SQL Services Over Native Driver Services + +Avoid this pattern: + +```ts +class TodoService extends Context.Service()("TodoService") { + static readonly layer = Layer.effect(this)( + Effect.acquireRelease( + Effect.try({ try: () => new Database("todos.sqlite") }) + ) + ) +} +``` + +Why this is usually a bad pattern in an Effect codebase: + +- the service is now tightly coupled to one runtime-specific database client +- you lose the shared SQL abstraction that the Effect repo already provides +- transaction and query conventions become ad hoc +- it is easier to drift away from typed SQL errors, SQL tracing, and reusable query helpers + +Prefer a layer that provides `SqlClient`, and let domain services depend on that. + +## Domain Services Should Depend On `SqlClient` + +Good pattern: + +```ts +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as SqlClient from "effect/unstable/sql/SqlClient" + +class TodoRepo extends Context.Service()("TodoRepo", { + make: Effect.succeed({ + getById: Effect.fn("TodoRepo.getById")(function*(id: number) { + const sql = yield* SqlClient.SqlClient + return yield* sql`SELECT id, title, completed FROM todos WHERE id = ${id}` + }) + }) +}) {} +``` + +Why this is better: + +- the domain service depends on an Effect SQL capability +- SQL stays observable and transactional +- the SQL client implementation can be provided separately from the business service + +## Transactions + +Use `SqlClient.withTransaction` for transaction boundaries. + +Repo reference: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` + +Prefer: + +```ts +const createAndAudit = Effect.fn("TodoRepo.createAndAudit")(function*(title: string) { + const sql = yield* SqlClient.SqlClient + + return yield* sql.withTransaction( + Effect.gen(function*() { + yield* sql`INSERT INTO todos ${sql.insert({ title, completed: false })}` + yield* sql`INSERT INTO audit_log ${sql.insert({ event: "todo_created" })}` + }) + ) +}) +``` + +Avoid: + +- manual `BEGIN` / `COMMIT` / `ROLLBACK` in application service code +- driver-specific transaction logic spread across multiple service methods + +## Query Composition + +Prefer keeping query logic inside SQL-aware services or repositories. + +Good patterns: + +- keep queries close to the service that owns the behavior +- use named business operations for multi-step workflows +- use small local helpers only when they improve clarity + +Avoid exposing one exported accessor function per SQL service method if it only forwards to the service. + +Bad: + +```ts +export const createTodo = Effect.fn(function*(title: string) { + const todos = yield* TodoRepo + return yield* todos.create(title) +}) +``` + +Prefer: + +- use the service method directly within the owning workflow +- or export a real business operation that adds behavior beyond simple forwarding + +## SQL Resolvers + +Use `SqlResolver` when request-style batching or schema-validated request/response handling is useful. + +Repo reference: + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlResolver.ts` + +It is especially good for: + +- batched lookup patterns +- `findById`-style resolvers +- grouped query resolution +- integrating SQL with request batching patterns + +Important repo pattern: + +- request schema validates inputs +- result schema validates outputs +- execution remains effectful and transactional + +This is one of the strongest typed-query patterns in the repo and should be preferred over ad hoc batched row mapping when the query fits the resolver model. + +## SQL Schemas And Models + +When schema-aware SQL helpers fit the task, prefer them over hand-mapped rows. + +Look at: + +- `SqlSchema` +- `SqlModel` + +These are good fits when: + +- the row shape matters strongly +- schema-based decode/encode should stay aligned with database access +- you want to reduce ad hoc row mapping logic + +Avoid overusing manual `type Row = { ... }` plus custom conversion if the schema-aware modules already express the shape clearly. + +Preferred order for query typing: + +1. schema-aware SQL module such as `SqlResolver`, `SqlSchema`, or `SqlModel` when it fits +2. typed query literals plus Schema decoding when the query is local +3. only use manual row mapping when the first two options are clearly heavier than the problem + +## Migrations + +Use `Migrator` for migrations. + +Repo reference: + +- `./.repos/effect/packages/effect/src/unstable/sql/Migrator.ts` + +The repo shows these best practices: + +- maintain a dedicated migrations table +- load migrations through a managed loader +- run migrations through the Effect runtime +- keep migration execution observable with logs and spans +- use SQL client transaction and locking semantics handled by the migrator + +Important behavior in the vendored repo: + +- migrations table creation is dialect-aware +- duplicate migration IDs are detected +- concurrent migration runs are guarded +- each migration is logged and wrapped in a span + +### Concrete Migration Loader Example + +The runtime-specific SQL migrator packages expose `fromRecord(...)` to define migrations from an ordered record. + +Example shape: + +```ts +import * as SqliteMigrator from "@effect/sql-sqlite-bun/SqliteMigrator" +import * as Effect from "effect/Effect" + +const migrations = SqliteMigrator.fromRecord({ + "1_create_todos": Effect.gen(function*() { + yield* sql` + CREATE TABLE todos ( + id INTEGER PRIMARY KEY NOT NULL, + title TEXT NOT NULL, + completed INTEGER NOT NULL DEFAULT 0 + ) + `.withoutTransform + }), + "2_add_todo_index": Effect.gen(function*() { + yield* sql` + CREATE INDEX todos_completed_idx ON todos (completed) + `.withoutTransform + }) +}) +``` + +This matches the model used by the vendored migrator implementation: + +- each migration has a numeric prefix and descriptive name +- each migration resolves to an `Effect` +- migrations are ordered by ID + +### Running Migrations Directly + +Use the runtime-specific `run(...)` helper when you want a startup effect that runs migrations explicitly. + +Example shape: + +```ts +import * as SqliteMigrator from "@effect/sql-sqlite-bun/SqliteMigrator" + +const runMigrations = SqliteMigrator.run({ + loader: migrations +}) +``` + +This is a good fit when: + +- startup explicitly runs migrations before launching the main app +- deployment tooling runs migrations as a separate command +- you want migration results as an ordinary `Effect` + +The return value includes the applied migration IDs and names. + +### Running Migrations As A Layer + +Use the runtime-specific `layer(...)` helper when migrations should run as part of top-level infrastructure setup. + +Example shape: + +```ts +import * as SqliteMigrator from "@effect/sql-sqlite-bun/SqliteMigrator" + +const MigrationLayer = SqliteMigrator.layer({ + loader: migrations +}) +``` + +From the vendored packages, this is implemented as `Layer.effectDiscard(run(options))`. + +That means: + +- the layer performs the migration effect +- it does not provide a new service of its own +- it is intended to be composed into startup infrastructure + +### Concrete Startup Composition Example + +Preferred shape: + +```ts +const SqlLayer = SqliteClient.layer({ filename: "todos.sqlite" }) + +const MigrationLayer = SqliteMigrator.layer({ + loader: migrations +}) + +const MigratedSqlLayer = Layer.merge( + SqlLayer, + MigrationLayer.pipe(Layer.provide(SqlLayer)) +) +``` + +This keeps the structure explicit: + +- one layer provides the SQL client +- one layer runs migrations +- the merged layer represents a migrated SQL environment + +If the same migrated SQL environment is reused in tests, create it once and reuse the layer value. + +### Migration Best Practices + +- use stable numeric migration IDs +- keep migration files ordered and unique +- do not hand-roll a separate migrations subsystem if `Migrator` already fits the project +- keep migration execution at startup or a dedicated operational boundary +- do not bury migration execution inside arbitrary service construction unless startup is explicitly the right place + +### Preferred Migration Boundary + +Good pattern: + +- construct the SQL layer +- run migrations once at startup or deployment entry +- then run the main application + +Concrete shapes: + +- separate startup effect: `SqliteMigrator.run({ loader })` +- startup layer: `SqliteMigrator.layer({ loader })` +- migrated environment layer: merge the SQL client layer with the migration layer provided by that client layer + +Avoid: + +- opportunistic migrations inside ordinary request handlers +- schema creation hidden inside unrelated business service constructors + +## Errors + +Prefer SQL errors that stay inside the Effect SQL model as long as possible. + +Use domain-level translation only where it helps the business boundary. + +Good: + +- SQL layer or repository works with `SqlError` and schema decode failures where appropriate +- higher-level service translates expected cases into domain errors when needed + +Avoid: + +- converting every SQL error immediately into a string +- hiding SQL failure details too early + +## Observability + +The vendored SQL modules already integrate with spans and transaction context. + +This is another reason to prefer them over raw driver usage. + +Good pattern: + +- use `SqlClient` and `withTransaction` +- keep business operations wrapped with `Effect.fn` +- add explicit spans only where business-level detail matters beyond the built-in SQL spans + +## Layering Pattern + +Preferred layering shape: + +1. runtime-specific database layer provides `SqlClient` +2. migrations run at startup boundary +3. domain repository/service depends on `SqlClient` +4. top-level application layer composes the database layer with the business layers + +This keeps: + +- driver choice at the edge +- SQL capability in the middle +- business logic above it + +## Anti-Patterns + +- embedding a native driver directly in business services when Effect SQL modules are available +- hand-rolling transactions with raw SQL statements in service methods +- hiding schema creation inside unrelated service constructors +- exporting one trivial accessor function per repository/service method +- converting SQL errors to strings too early +- bypassing `Migrator` when the project already uses Effect SQL +- creating ad hoc migration effects without a stable loader shape when `fromRecord(...)` already fits the project +- scattering migration execution across multiple subsystems instead of one explicit startup boundary + +## Good Repo Examples To Study + +- `./.repos/effect/packages/effect/src/unstable/sql/SqlClient.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/Migrator.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlResolver.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlSchema.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlModel.ts` +- `./.repos/effect/packages/effect/src/unstable/sql/SqlError.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/guide-testing.md b/tools/agent-os/.agents/skills/effect-ts/references/guide-testing.md new file mode 100644 index 000000000..2f777cbc4 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/guide-testing.md @@ -0,0 +1,504 @@ +# Testing Guide + +This guide is based on the vendored `@effect/vitest` package in `./.repos/effect`. + +Key source files: + +- `./.repos/effect/packages/vitest/src/index.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` +- `./.repos/effect/packages/vitest/test/index.test.ts` +- `./.repos/effect/packages/vitest/typetest/index.tst.ts` + +## Preferred Rule + +When testing Effect code with Vitest, prefer `@effect/vitest` over manually calling `Effect.runPromise`, `Effect.runSync`, or ad hoc runtime setup inside ordinary Vitest tests. + +Layer provisioning in tests should follow these rules: + +1. If multiple tests should share the same layered setup, use `layer(...)`. +2. If a nested group needs extra dependencies, use `it.layer(...)`. +3. If tests need isolated layer instances per test, use multiple separate `it.layer(...)` calls. +4. Do not default to local `.pipe(Effect.provide(...))` inside test bodies. + +Use: + +- `it.effect` for Effect-based tests with test services +- `it.live` for Effect-based tests that should use live services +- `layer(...)` and `it.layer(...)` for shared layered test setup +- `it.effect.prop` for Effect-based property tests + +Do not default to `.pipe(Effect.provide(SomeLayer))` inside test bodies when `layer(...)` or `it.layer(...)` expresses the setup more clearly. + +## Imports + +Preferred imports for Effect tests: + +```ts +import { assert, describe, it, layer } from "@effect/vitest" +import { Effect } from "effect" +``` + +`@effect/vitest` re-exports Vitest, so it is the normal entrypoint for test APIs in an Effect codebase. + +## Core Test Modes + +## `it.effect` + +Use `it.effect` for most Effect tests. + +It automatically: + +- runs the effect +- scopes it correctly +- provides the default test environment + +The default test environment includes: + +- `TestConsole` +- `TestClock` + +This comes directly from the internal implementation. + +Example: + +```ts +import { assert, it } from "@effect/vitest" +import { Effect } from "effect" + +it.effect("loads a user", () => + Effect.gen(function*() { + yield* Effect.void + assert.isTrue(true) + }) +) +``` + +Use `it.effect` when: + +- the test uses ordinary Effect code +- the test benefits from `TestClock` +- the test benefits from `TestConsole` +- the test should run in a scoped Effect runtime + +## `it.live` + +Use `it.live` when the test should use live services instead of the default test environment. + +Example: + +```ts +it.live("uses live services", () => + Effect.gen(function*() { + yield* Effect.void + }) +) +``` + +Use `it.live` when: + +- you need the real `Clock` +- the test should interact with live runtime services +- you deliberately do not want the test environment overrides + +Rule of thumb: + +- default: `it.effect` +- opt into `it.live` only when you actually need live behavior + +## Effect Test Structure + +Preferred pattern: + +```ts +it.effect("does something", () => + Effect.gen(function*() { + const value = yield* someEffect + assert.strictEqual(value, 1) + }) +) +``` + +Prefer `Effect.gen` inside `it.effect` and `it.live` for readability. + +Avoid: + +- `it("...", async () => ...)` for Effect programs +- `Effect.runPromise(...)` inside plain Vitest tests +- manual runtime setup for routine Effect tests + +## Assertions + +Use `assert` for Effect tests. + +The vendored repo also uses `expect` in some tests, but for skill guidance prefer `assert` in Effect-based tests because it keeps tests more uniform and explicit inside Effect programs. + +Examples: + +```ts +assert.isTrue(value === 1) +assert.strictEqual(a + b, b + a) +assert.include(text, substring) +``` + +## Test Context + +`it.effect` and `it.live` test functions can receive a Vitest `TestContext`. + +Example: + +```ts +it.effect("uses context", (ctx) => + Effect.gen(function*() { + ctx.onTestFailed(() => { + // cleanup or diagnostics + }) + }) +) +``` + +Use this when you need: + +- failure hooks +- abort signals +- ordinary Vitest test context integration + +## `each`, `skip`, `skipIf`, `runIf`, `only`, `fails` + +The Effect testers support the standard test variants: + +- `it.effect.each(...)` +- `it.live.each(...)` +- `it.effect.skip(...)` +- `it.effect.skipIf(...)` +- `it.effect.runIf(...)` +- `it.effect.only(...)` +- `it.live.fails(...)` + +Examples from the vendored tests show these are first-class parts of the API. + +Use them exactly as you would with Vitest, but return `Effect` from the test body. + +## Property Testing + +There are two different property-test entrypoints and they do not have identical behavior. + +## Top-level `it.prop` + +Use `it.prop` for non-Effect property tests. + +Example: + +```ts +import { it } from "@effect/vitest" +import { FastCheck } from "effect/testing" + +const realNumber = FastCheck.float({ noNaN: true, noDefaultInfinity: true }) + +it.prop("symmetry", [realNumber, FastCheck.integer()], ([a, b]) => a + b === b + a) +``` + +Important limitation from the internal implementation: + +- top-level `it.prop` does not support `Schema` arbitraries yet +- if you pass a `Schema`, it throws + +So use top-level `it.prop` only with explicit `FastCheck` arbitraries. + +## `it.effect.prop` + +Use `it.effect.prop` for property tests that return `Effect`. + +This is the more powerful property-testing mode for Effect code. + +Example: + +```ts +import { assert, it } from "@effect/vitest" +import { Effect } from "effect" +import { FastCheck } from "effect/testing" + +const realNumber = FastCheck.float({ noNaN: true, noDefaultInfinity: true }) + +it.effect.prop("symmetry", [realNumber, FastCheck.integer()], ([a, b]) => + Effect.gen(function*() { + assert.strictEqual(a + b, b + a) + }) +) +``` + +Unlike top-level `it.prop`, `it.effect.prop` does support `Schema` inputs by converting them with `Schema.toArbitrary`. + +So prefer `it.effect.prop` when: + +- the property test needs `Effect` +- you want to use `Schema` values as arbitraries +- the test needs Effect services or scope + +## `layer(...)` + +Use top-level `layer(...)` to share a `Layer` across a group of tests. + +Hard rule: + +- use `layer(...)` when tests should share one layered setup +- do not use it when each test needs its own isolated layer instance + +This is one of the most important `@effect/vitest` features. + +Example: + +```ts +import { describe, it, layer } from "@effect/vitest" +import { Context, Effect, Layer } from "effect" + +class Foo extends Context.Service()("Foo") { + static readonly layer = Layer.succeed(Foo)("foo") +} + +describe("foo", () => { + layer(Foo.layer)((it) => { + it.effect("gets foo", () => + Effect.gen(function*() { + const foo = yield* Foo + return foo + }) + ) + }) +}) +``` + +What it does internally: + +- builds the layer once for the test group +- memoizes the built layer with a `MemoMap` +- keeps the scope open for the group +- closes the scope in `afterAll` + +This makes it the preferred way to share layer setup across related tests. + +This is also the preferred alternative to calling `Effect.provide(...)` inside each individual test body. + +## Anti-Pattern: Local `Effect.provide(...)` In Tests + +If multiple tests use the same layer, do not write tests like this: + +```ts +it.effect("creates and lists todos", () => + Effect.gen(function*() { + const service = yield* TodoService + yield* service.create("write tests") + yield* service.create("ship feature") + }).pipe( + Effect.provide(TodoService.inMemoryLayer) + ) +) +``` + +Why this is the wrong pattern: + +- it repeats provisioning in every test +- it hides the layered test setup inside each test body +- it fights the `@effect/vitest` layer helpers +- it makes shared setup and teardown less explicit +- it bypasses the clearer grouped-layer style that the library is designed for + +Prefer: + +```ts +describe("TodoService", () => { + layer(TodoService.inMemoryLayer)((it) => { + it.effect("creates and lists todos", () => + Effect.gen(function*() { + const service = yield* TodoService + yield* service.create("write tests") + yield* service.create("ship feature") + }) + ) + + it.effect("updates completion and deletes todos", () => + Effect.gen(function*() { + const service = yield* TodoService + const todo = yield* service.create("close issue") + yield* service.setCompleted(todo.id, true) + yield* service.remove(todo.id) + }) + ) + }) +}) +``` + +Rule: + +- if a test or group of tests depends on a layer, prefer `layer(...)` +- if a nested group needs extra dependencies, prefer `it.layer(...)` +- if tests need isolated layer instances per test, use multiple separate `it.layer(...)` calls +- use local `Effect.provide(...)` in tests only for true one-off edge cases, not as the normal pattern + +This matches the general layer guidance: provisioning belongs at the boundary, and in `@effect/vitest` the test boundary should usually be expressed with `layer(...)` rather than ad hoc local provisioning. + +## `it.layer(...)` + +Use `it.layer(...)` inside an existing layered test group to add nested layer context. + +Hard rule: + +- use `it.layer(...)` when a specific test or nested test block should get its own isolated layered setup +- if isolation matters, prefer multiple separate `it.layer(...)` calls over one shared `layer(...)` group + +Example: + +```ts +layer(Foo.layer)((it) => { + it.layer(Bar.layer)("nested", (it) => { + it.effect("gets both", () => + Effect.gen(function*() { + const foo = yield* Foo + const bar = yield* Bar + return [foo, bar] + }) + ) + }) +}) +``` + +Important behavior from the implementation: + +- nested `it.layer(...)` reuses the parent memo map +- nested `it.layer(...)` does not accept `memoMap` or `excludeTestServices` +- nested layering is meant to extend the current layered test environment, not redefine its runtime policy + +Use multiple `it.layer(...)` blocks when each test should get its own isolated layered setup instead of sharing one top-level `layer(...)` group. + +## `layer` Options + +The top-level `layer(...)` helper accepts: + +- `timeout` +- `memoMap` +- `excludeTestServices` + +### `excludeTestServices` + +By default, `layer(...)` merges your layer with the test environment (`TestClock` and `TestConsole`). + +Use `excludeTestServices: true` when you want your layer group to run without those test-service overrides. + +This is useful for tests that should keep live runtime behavior. + +### `memoMap` + +Use `memoMap` only when you have a specific reason to coordinate layer memoization manually across test setups. + +Most tests should let `@effect/vitest` manage it. + +## Scoped Resources In Tests + +`@effect/vitest` is designed to work correctly with scoped effects and layered resources. + +The vendored tests explicitly verify resource release through `afterAll`. + +Use this normally: + +- define scoped services in layers +- use `layer(...)` to share them +- let the helper own scope setup and teardown + +Avoid manually managing large scopes in test code unless the test specifically needs that control. + +## `flakyTest` + +Use `flakyTest` for tests that need bounded retrying. + +Repo behavior: + +- wraps the test in `Effect.scoped` +- retries using a schedule +- retries for up to a timeout window +- converts final failure to defect with `Effect.orDie` + +Use this only for truly flaky integration-style conditions, not as a substitute for deterministic tests. + +## `makeMethods` And `describeWrapped` + +These exist for custom integration and wrapper scenarios. + +### `makeMethods` + +Use `makeMethods` when you need to extend or wrap a custom Vitest `it` instance while preserving Effect-aware helpers. + +### `describeWrapped` + +Use `describeWrapped` when you want a `describe` wrapper that hands you the augmented Effect-aware `it` methods directly. + +Most test code can just use imported `describe` and `it` from `@effect/vitest`. + +## Equality Testers + +`addEqualityTesters()` exists as part of the public API. + +Use it only when you have a concrete need to install equality testers for your Vitest environment. + +It is not needed for ordinary test structure. + +## Recommended Patterns + +### Pattern: normal Effect test + +```ts +it.effect("does work", () => + Effect.gen(function*() { + const value = yield* Effect.succeed(1) + assert.strictEqual(value, 1) + }) +) +``` + +### Pattern: shared layer for a test group + +```ts +layer(AppLayer)("app", (it) => { + it.effect("uses app services", () => + Effect.gen(function*() { + yield* Effect.void + }) + ) +}) +``` + +### Pattern: property test with Effect + +```ts +it.effect.prop("law", [FastCheck.integer()], ([n]) => + Effect.gen(function*() { + assert.strictEqual(n + 0, n) + }) +) +``` + +### Pattern: use `TestClock` + +```ts +it.effect("uses TestClock", () => + Effect.gen(function*() { + const fiber = yield* Effect.forkChild(Effect.sleep("1 second")) + yield* TestClock.adjust("1 second") + yield* Fiber.join(fiber) + }) +) +``` + +## Anti-Patterns + +- using plain `it(...)` with `Effect.runPromise(...)` for normal Effect tests +- using `it.live` by default when `it.effect` is sufficient +- manually building and tearing down large layer graphs instead of using `layer(...)` +- using top-level `it.prop` with `Schema` inputs +- using `flakyTest` to hide deterministic failures +- duplicating runtime setup instead of sharing a layer + +## Good Repo Examples To Study + +- `./.repos/effect/packages/vitest/test/index.test.ts` +- `./.repos/effect/packages/vitest/src/index.ts` +- `./.repos/effect/packages/vitest/src/internal/internal.ts` +- `./.repos/effect/packages/vitest/typetest/index.tst.ts` diff --git a/tools/agent-os/.agents/skills/effect-ts/references/setup.md b/tools/agent-os/.agents/skills/effect-ts/references/setup.md new file mode 100644 index 000000000..be471a678 --- /dev/null +++ b/tools/agent-os/.agents/skills/effect-ts/references/setup.md @@ -0,0 +1,89 @@ +# Effect Source Setup + +This setup task is required when `./.repos/effect` is missing from the root of the repository where this skill is used. + +## Prompt + +The local Effect source checkout was not found at `./.repos/effect`. + +Choose one of these setup options before continuing: + +1. Add `https://github.com/Effect-TS/effect-smol` as a git subtree with squashed history at `./.repos/effect` +2. Add `https://github.com/Effect-TS/effect-smol` as a git submodule at `./.repos/effect` +3. Use `git clone` into `./.repos/effect`, ignore it via `.gitignore`, and add a prepare script that bootstraps it when missing + +## Supported Options + +### 1. Git Subtree + +Use this when the repository should vendor the Effect source directly while keeping history compact. + +- Repo path: `./.repos/effect` +- Source: `https://github.com/Effect-TS/effect-smol` +- Preferred shape: subtree with squashed history + +### 2. Git Submodule + +Use this when the repository should track the Effect source explicitly as a separate Git dependency. + +- Repo path: `./.repos/effect` +- Source: `https://github.com/Effect-TS/effect-smol` +- Preferred shape: standard Git submodule + +### 3. Local Clone + Gitignore + Prepare Task + +Use this when the repository should avoid vendoring or submodule management, but still provide a reproducible local setup. + +- Repo path: `./.repos/effect` +- Source: `https://github.com/Effect-TS/effect-smol` +- Add `.repos/effect` to the repository `.gitignore` +- Add a `prepare` task that clones the repo automatically when the directory is missing + +#### Concrete Shape + +Use this exact shape for the setup. Do not invent a different script. + +`package.json`: + +```json +{ + "scripts": { + "prepare": "./scripts/prepare-effect.sh" + } +} +``` + +`.gitignore`: + +```gitignore +.repos/effect +``` + +`scripts/prepare-effect.sh`: + +```sh +#!/usr/bin/env sh + +set -eu + +repo_dir=".repos/effect" +repo_url="https://github.com/Effect-TS/effect-smol" + +if [ -d "$repo_dir/.git" ]; then + exit 0 +fi + +mkdir -p ".repos" +git clone "$repo_url" "$repo_dir" +``` + +#### Notes + +- This keeps `./.repos/effect` available for local research without forcing it into version control +- The script is only responsible for ensuring the checkout exists; it does not update or reset an existing clone +- If you choose this option, the setup task should add this exact script, wire it via `prepare`, and add `.repos/effect` to `.gitignore` + +## Guidance + +- Do not continue with Effect-specific work until one of the setup options is chosen. +- Prefer the option that matches the host repository's dependency management style. diff --git a/tools/agent-os/.gitignore b/tools/agent-os/.gitignore index c2658d7d1..4304d7d0d 100644 --- a/tools/agent-os/.gitignore +++ b/tools/agent-os/.gitignore @@ -1 +1,2 @@ node_modules/ +.repos/effect diff --git a/tools/agent-os/AGENTS.md b/tools/agent-os/AGENTS.md index 8ff026edf..872166ee7 100644 --- a/tools/agent-os/AGENTS.md +++ b/tools/agent-os/AGENTS.md @@ -11,6 +11,14 @@ Keep compatibility with the root distro's `data/`, `state/`, briefs, status even Use the Bun channel declared in `.bun-version`. Keep `.bun-version`, `packageManager`, and the matching Bun type definitions aligned on the latest stable release. +## Effect + +Use Effect for effectful workflows whose failure modes, dependencies, cleanup, concurrency, retries, or observability benefit from typed composition. +Keep small total functions and straightforward value transformations in plain TypeScript. +Load the scoped `effect-ts` skill installed from `Effect-TS/skills` before changing Effect code. +Use the ignored `.repos/effect` checkout for source verification when the skill's focused guides do not answer the question. +Do not copy that checkout into the container image or commit it as a source snapshot. + ## Addresses Use Multiaddr strings as the single locator for supervised mates. @@ -23,7 +31,9 @@ The address grammar and validation contract are owned by `src/address.ts` and it ## Command Execution -Use Bun Shell's `$` for finite calls to trusted CLIs such as `kubectl`, `herdr`, `treehouse`, and Git. +Use the official `@akua-dev/sdk` for Akua operations implemented inside this TypeScript tool. +Keep the bundled `akua` CLI available for Firstmate's direct and interactive use. +Use Bun Shell's `$` for other finite calls to trusted CLIs such as `kubectl`, `herdr`, `treehouse`, and Git. Interpolate dynamic values directly so Bun treats each value as a literal argument. Never use `{ raw: value }` for dynamic data. Avoid `bash -c` and validate values that an external command could interpret as flags. @@ -32,6 +42,12 @@ Set environment and working directory per command instead of mutating the global Use `Bun.spawn()` for interactive agents, servers, watches, live logs, cancellation, and other long-running processes. Existing Bash scripts may be invoked explicitly during migration, but new orchestration and state transitions belong in TypeScript. +## Akua packages + +The prepared mate package lives at `packages/mate/` and renders ordinary Kubernetes YAML. +Treat it as an editable convenience, not a required abstraction or controller. +Firstmate may use `agent-os mate render`, invoke `akua render` itself, edit the result, or author equivalent YAML directly. + ## Scope Discipline Prefer a thin adapter over a new framework. diff --git a/tools/agent-os/README.md b/tools/agent-os/README.md index f37699a54..7776509b3 100644 --- a/tools/agent-os/README.md +++ b/tools/agent-os/README.md @@ -23,6 +23,11 @@ bun run check bun run src/cli.ts address local task-x bun run src/cli.ts address k8s task-x bun run src/cli.ts address inspect /k8s/in-cluster/namespace/agent-os/mate/task-x/herdr +bun run src/cli.ts mate render task-x --out /tmp/task-x ``` +Akua package operations inside this tool use the official `@akua-dev/sdk` in process. +The prepared package under `packages/mate/` remains optional and renders editable Kubernetes YAML. +The scoped official `Effect-TS/skills` installation guides typed Effect workflows; its Effect source checkout is local and ignored rather than committed as a snapshot. + See `AGENTS.md` in this directory for the tool-specific development rules. diff --git a/tools/agent-os/bun.lock b/tools/agent-os/bun.lock index 717d11312..c0ea2c18d 100644 --- a/tools/agent-os/bun.lock +++ b/tools/agent-os/bun.lock @@ -5,7 +5,9 @@ "": { "name": "@akua-dev/agent-os-tool", "dependencies": { + "@akua-dev/sdk": "0.8.24", "@multiformats/multiaddr": "13.0.3", + "effect": "3.21.4", }, "devDependencies": { "@types/bun": "1.3.14", @@ -14,10 +16,32 @@ }, }, "packages": { + "@akua-dev/native": ["@akua-dev/native@0.8.24", "", { "dependencies": { "@akua-dev/native-engines": "0.8.24" }, "optionalDependencies": { "@akua-dev/native-darwin-arm64": "0.8.24", "@akua-dev/native-darwin-x64": "0.8.24", "@akua-dev/native-linux-arm64-gnu": "0.8.24", "@akua-dev/native-linux-arm64-musl": "0.8.24", "@akua-dev/native-linux-x64-gnu": "0.8.24", "@akua-dev/native-linux-x64-musl": "0.8.24", "@akua-dev/native-win32-x64-msvc": "0.8.24" } }, "sha512-DZsUSd0SaLrqBvbmsEmPD9rUwxrUZSC/QTNfclzRgmKxnK2y5x90pT+J4L/Aoa3lCFWpCp9MOM9pZ7ztjHL1Nw=="], + + "@akua-dev/native-darwin-arm64": ["@akua-dev/native-darwin-arm64@0.8.24", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6Wa2zDcrRWQUPouHKmZUom4kNmffvWYxcrvnaeoVwbQFj3TXWKAaCzcVYjZGeLcV8HxMkTrYx4rt/NH5GRLkWA=="], + + "@akua-dev/native-darwin-x64": ["@akua-dev/native-darwin-x64@0.8.24", "", { "os": "darwin", "cpu": "x64" }, "sha512-hhXiqfpw9itn6cdNhC0IVW1WUaPBmpYmBPW7hfkBuaTlP/2wPyAUx+Wu0cJWpna1YCUg3WzcUOj5rm/lyciP9w=="], + + "@akua-dev/native-engines": ["@akua-dev/native-engines@0.8.24", "", {}, "sha512-3rKiWQn1pIURERUmxNiqPw7SxtMrNwX5tDxsCRxsAZ7Ovzfzcwx/VcycfuY+z1DNhEJrkKChkYiMT3N/EZanmg=="], + + "@akua-dev/native-linux-arm64-gnu": ["@akua-dev/native-linux-arm64-gnu@0.8.24", "", { "os": "linux", "cpu": "arm64" }, "sha512-NTeuJRCjy68JJT5LuHwPHf11mnZ1a5hNeKNfoBYMzUeViN3BdnBuszjNxwYHAgQYEN8+Rf5EGtTMdOTm0xDy/Q=="], + + "@akua-dev/native-linux-arm64-musl": ["@akua-dev/native-linux-arm64-musl@0.8.24", "", { "os": "linux", "cpu": "arm64" }, "sha512-QbGJfTY2Ilv+FSVaIU5MnlHjprbKVM41Ya+YfanHJsTBwO0dUw6J9C/fNVKTHeQM98vLVBIWdLiAPl73jSI/Qw=="], + + "@akua-dev/native-linux-x64-gnu": ["@akua-dev/native-linux-x64-gnu@0.8.24", "", { "os": "linux", "cpu": "x64" }, "sha512-jJDnr4MGYxB/vflhGklC1dUxHsO7HNuWexDFGB/vPhzo6RGsAWsApnrqPYYfBDmsk0nvvZKO9OWTB+e837gCzA=="], + + "@akua-dev/native-linux-x64-musl": ["@akua-dev/native-linux-x64-musl@0.8.24", "", { "os": "linux", "cpu": "x64" }, "sha512-21EsbM2fArJjX4RKXeVXDPxSBHkeMtYVCvT9vDj4S1lo5Ld0xRM/utm6EQkfbShdpwWZRhkv/t7evgdVM8Zb2A=="], + + "@akua-dev/native-win32-x64-msvc": ["@akua-dev/native-win32-x64-msvc@0.8.24", "", { "os": "win32", "cpu": "x64" }, "sha512-boG/GW6KTucoq7PUlU92tvB91DGxl4hNdQICcsXwgO7npzLqNEhCzKYnpuwKgqhK6vSpoPd0/jZdmu6EwUyzKQ=="], + + "@akua-dev/sdk": ["@akua-dev/sdk@0.8.24", "", { "dependencies": { "@akua-dev/native": "0.8.24", "@standard-schema/spec": "^1.1.0", "ajv": "^8.18.0", "yaml": "^2.8.3" } }, "sha512-8OM8luAqmxnA4kRJUGbf4Jhr4YuSf/lXKAOUD4kZTUepoRvgYP3xHlXJRCVd1Qv00L1NmeluXG8sr5rh/soGIg=="], + "@chainsafe/is-ip": ["@chainsafe/is-ip@2.1.0", "", {}, "sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w=="], "@multiformats/multiaddr": ["@multiformats/multiaddr@13.0.3", "", { "dependencies": { "@chainsafe/is-ip": "^2.0.1", "multiformats": "^14.0.0", "uint8-varint": "^3.0.0", "uint8arrays": "^6.1.1" } }, "sha512-mEqqJ4r3a/uuFMTpRkU316wGNIDQNhuVWpm+ebKTQeYsfv9jXbPONWM6VVnj3KGUrwfsX7GZOyp4TFqEA2SPCw=="], + "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], @@ -62,10 +86,26 @@ "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], + "ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "effect": ["effect@3.21.4", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" } }, "sha512-B89v/xSgPbl1J2Ai2u18jxq3odpFauU1rC6/eSs4FeNHi72kwKdJp12VGigvRV2lK+kRnx+OOz41XV8guZd4gQ=="], + + "fast-check": ["fast-check@3.23.2", "", { "dependencies": { "pure-rand": "^6.1.0" } }, "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A=="], + + "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], + + "fast-uri": ["fast-uri@3.1.3", "", {}, "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg=="], + + "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], + "multiformats": ["multiformats@14.0.4", "", {}, "sha512-+SXItmOUWzT0kjlIfkZ4NawJaC9jW/mPlyn902V9Qgpc7YDwdEiwqbiZxTyvC0XWh1jZguXca3A/WQ+UOPRLUA=="], + "pure-rand": ["pure-rand@6.1.0", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="], + + "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], + "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "uint8-varint": ["uint8-varint@3.0.0", "", { "dependencies": { "uint8arraylist": "^3.0.1", "uint8arrays": "^6.1.0" } }, "sha512-S4DdpXBaLwKcFo7f0bWzWfHjbZ/i3QhM842qn+ZvHjxqFCfUcEB9SQNcmI69S+zMlcmIcKxsk9Iyw77S2Kxv6Q=="], @@ -75,5 +115,7 @@ "uint8arrays": ["uint8arrays@6.1.1", "", { "dependencies": { "multiformats": "^14.0.0" } }, "sha512-iz7JN0XCSZYA111lhFG2Ui9EhFvTNekqSRHw3lvMHq+dzwWy1OQftxFQREEh4rffU0oSoXdQHsk2TiHKVm4fsA=="], "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + + "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], } } diff --git a/tools/agent-os/package.json b/tools/agent-os/package.json index 1dd58ead0..ba519baaa 100644 --- a/tools/agent-os/package.json +++ b/tools/agent-os/package.json @@ -8,11 +8,14 @@ }, "scripts": { "check": "bun run typecheck && bun test", + "prepare": "./scripts/prepare-effect.sh", "test": "bun test", "typecheck": "tsc --noEmit" }, "dependencies": { - "@multiformats/multiaddr": "13.0.3" + "@akua-dev/sdk": "0.8.24", + "@multiformats/multiaddr": "13.0.3", + "effect": "3.21.4" }, "devDependencies": { "@types/bun": "1.3.14", diff --git a/tools/agent-os/packages/mate/README.md b/tools/agent-os/packages/mate/README.md new file mode 100644 index 000000000..5576ebec4 --- /dev/null +++ b/tools/agent-os/packages/mate/README.md @@ -0,0 +1,25 @@ +# Agent OS mate package + +This Akua package is a prepared convenience for rendering one general-purpose mate Pod and its persistent home. +It is not a controller, required operating model, or replacement for Kubernetes YAML. + +Render the example directly: + +```sh +akua render --package package.k --inputs inputs.example.yaml --out /tmp/scout-1 +``` + +Or use the Agent OS convenience command from any directory: + +```sh +agent-os mate render scout-1 --namespace agent-os-demo --image agent-os:dev --out /tmp/scout-1 +``` + +Inspect and edit the rendered YAML before applying it: + +```sh +kubectl apply -f /tmp/scout-1 +``` + +Firstmate may change this package, supply additional inputs, edit the rendered files, or author equivalent YAML directly. +The package intentionally grants no Kubernetes token and no privileged, host-namespace, or host-mount access. diff --git a/tools/agent-os/packages/mate/akua.toml b/tools/agent-os/packages/mate/akua.toml new file mode 100644 index 000000000..06303c226 --- /dev/null +++ b/tools/agent-os/packages/mate/akua.toml @@ -0,0 +1,5 @@ +[package] +name = "agent-os-mate" +version = "0.1.0" +edition = "akua.dev/v1alpha1" +strict_signing = true diff --git a/tools/agent-os/packages/mate/inputs.example.yaml b/tools/agent-os/packages/mate/inputs.example.yaml new file mode 100644 index 000000000..89d21d97f --- /dev/null +++ b/tools/agent-os/packages/mate/inputs.example.yaml @@ -0,0 +1,5 @@ +mateId: scout-1 +namespace: agent-os-demo +address: /k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr +image: agent-os:dev +imagePullPolicy: Never diff --git a/tools/agent-os/packages/mate/package.k b/tools/agent-os/packages/mate/package.k new file mode 100644 index 000000000..f12829369 --- /dev/null +++ b/tools/agent-os/packages/mate/package.k @@ -0,0 +1,101 @@ +import akua.ctx + +schema Input: + """Inputs for one general-purpose Kubernetes-backed Agent OS mate.""" + + mateId: str + namespace: str = "agent-os" + address: str + image: str = "agent-os:dev" + imagePullPolicy: "Always" | "IfNotPresent" | "Never" = "IfNotPresent" + storage: str = "10Gi" + cpuRequest: str = "250m" + memoryRequest: str = "512Mi" + cpuLimit: str = "2" + memoryLimit: str = "4Gi" + + check: + len(mateId) <= 43, "mateId must leave room for the agent-os-mate- resource prefix" + +input: Input = ctx.input() + +labels = { + "app.kubernetes.io/name" = "agent-os" + "app.kubernetes.io/component" = "mate" + "agent-os.akua.dev/mate" = input.mateId +} + +resources = [ + { + apiVersion = "v1" + kind = "PersistentVolumeClaim" + metadata = { + name = "agent-os-mate-${input.mateId}-home" + namespace = input.namespace + labels = labels + } + spec = { + accessModes = ["ReadWriteOnce"] + resources = { + requests = { storage = input.storage } + } + } + }, + { + apiVersion = "v1" + kind = "Pod" + metadata = { + name = "agent-os-mate-${input.mateId}" + namespace = input.namespace + labels = labels + annotations = { + "agent-os.akua.dev/address" = input.address + } + } + spec = { + automountServiceAccountToken = False + securityContext = { + fsGroup = 0 + runAsGroup = 0 + runAsUser = 0 + } + initContainers = [{ + name = "agent-os-init" + image = input.image + imagePullPolicy = input.imagePullPolicy + command = ["/opt/agent-os/bin/agent-os-init.sh"] + volumeMounts = [{ + name = "home" + mountPath = "/persistent-agent" + }] + }] + containers = [{ + name = "mate" + image = input.image + imagePullPolicy = input.imagePullPolicy + env = [ + { name = "FM_HOME", value = "/home/agent" }, + { name = "HOME", value = "/home/agent" }, + ] + resources = { + requests = { + cpu = input.cpuRequest + memory = input.memoryRequest + } + limits = { + cpu = input.cpuLimit + memory = input.memoryLimit + } + } + volumeMounts = [ + { name = "home", mountPath = "/home/agent" }, + { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, + ] + }] + volumes = [{ + name = "home" + persistentVolumeClaim = { claimName = "agent-os-mate-${input.mateId}-home" } + }] + } + }, +] diff --git a/tools/agent-os/scripts/prepare-effect.sh b/tools/agent-os/scripts/prepare-effect.sh new file mode 100755 index 000000000..24a9d3145 --- /dev/null +++ b/tools/agent-os/scripts/prepare-effect.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +set -eu + +repo_dir=".repos/effect" +repo_url="https://github.com/Effect-TS/effect-smol" + +if [ -d "$repo_dir/.git" ]; then + exit 0 +fi + +mkdir -p ".repos" +git clone "$repo_url" "$repo_dir" diff --git a/tools/agent-os/skills-lock.json b/tools/agent-os/skills-lock.json new file mode 100644 index 000000000..192e1e1d3 --- /dev/null +++ b/tools/agent-os/skills-lock.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "skills": { + "effect-ts": { + "source": "Effect-TS/skills", + "sourceType": "github", + "skillPath": "skills/effect-ts/SKILL.md", + "computedHash": "301521e167410dd42d834dd0e7cedbe9cfffa519f630a12241dbea7b1d932324" + } + } +} diff --git a/tools/agent-os/src/cli.ts b/tools/agent-os/src/cli.ts index 5cd221ed4..48830c7d0 100755 --- a/tools/agent-os/src/cli.ts +++ b/tools/agent-os/src/cli.ts @@ -1,17 +1,20 @@ #!/usr/bin/env bun import { parseArgs } from "node:util"; +import { Effect } from "effect"; import { formatKubernetesAgentAddress, formatLocalAgentAddress, parseAgentAddress, } from "./address.ts"; +import { renderMatePackage } from "./mate-package.ts"; const usage = `Usage: agent-os address local agent-os address k8s [--cluster ] [--namespace ] - agent-os address inspect `; + agent-os address inspect + agent-os mate render [--cluster ] [--namespace ] [--image ] [--out ]`; function requireSingleArgument(args: string[]): string { if (args.length !== 1 || args[0] == null) { @@ -67,18 +70,68 @@ function runAddressCommand(args: string[]): void { } } -function main(): void { - const [command, ...rest] = Bun.argv.slice(2); +async function runMateCommand(args: string[]): Promise { + const [command, ...rest] = args; - if (command !== "address") { + if (command !== "render") { throw new Error(usage); } - runAddressCommand(rest); + const { positionals, values } = parseArgs({ + allowPositionals: true, + args: rest, + options: { + cluster: { + default: "in-cluster", + type: "string", + }, + image: { + default: "agent-os:dev", + type: "string", + }, + namespace: { + default: "agent-os", + type: "string", + }, + out: { + type: "string", + }, + }, + strict: true, + }); + const mateId = requireSingleArgument(positionals); + const result = await Effect.runPromise( + renderMatePackage({ + cluster: values.cluster, + image: values.image, + mateId, + namespace: values.namespace, + outDir: values.out, + }).pipe( + Effect.catchAll((error) => Effect.fail(new Error(error.message))), + ), + ); + + console.log(JSON.stringify(result, null, 2)); +} + +async function main(): Promise { + const [command, ...rest] = Bun.argv.slice(2); + + switch (command) { + case "address": + runAddressCommand(rest); + return; + case "mate": + await runMateCommand(rest); + return; + default: + throw new Error(usage); + } } try { - main(); + await main(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; diff --git a/tools/agent-os/src/json-modules.d.ts b/tools/agent-os/src/json-modules.d.ts new file mode 100644 index 000000000..b14866845 --- /dev/null +++ b/tools/agent-os/src/json-modules.d.ts @@ -0,0 +1,4 @@ +declare module "*.json" { + const value: { readonly $defs: Readonly> }; + export default value; +} diff --git a/tools/agent-os/src/mate-package.ts b/tools/agent-os/src/mate-package.ts new file mode 100644 index 000000000..9d9e0306d --- /dev/null +++ b/tools/agent-os/src/mate-package.ts @@ -0,0 +1,151 @@ +import { + Akua, + AkuaRateLimitedError, + AkuaUserError, + type RenderOptions, + type RenderSummary, +} from "@akua-dev/sdk"; +import { Data, Effect } from "effect"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { formatKubernetesAgentAddress } from "./address.ts"; + +export interface RenderMatePackageOptions { + cluster?: string; + image?: string; + mateId: string; + namespace?: string; + outDir?: string; +} + +export interface RenderedMatePackage { + address: string; + outDir: string; +} + +export class MatePackageInputError extends Data.TaggedError( + "MatePackageInputError", +)<{ + readonly cause: unknown; + readonly message: string; +}> {} + +export class AkuaRenderError extends Data.TaggedError("AkuaRenderError")<{ + readonly cause: unknown; + readonly message: string; +}> {} + +export class AkuaPackageUserError extends Data.TaggedError( + "AkuaPackageUserError", +)<{ + readonly cause: Error; + readonly message: string; +}> {} + +export class AkuaPackageRateLimitedError extends Data.TaggedError( + "AkuaPackageRateLimitedError", +)<{ + readonly cause: Error; + readonly message: string; +}> {} + +export interface AkuaRenderer { + render(options: RenderOptions): Promise; +} + +const packageDirectory = resolve( + dirname(fileURLToPath(import.meta.url)), + "../packages/mate", +); + +const defaultAkua = new Akua(); + +function mapAkuaError(cause: unknown) { + if (cause instanceof AkuaRateLimitedError) { + return new AkuaPackageRateLimitedError({ + cause, + message: "Akua rate-limited the mate package render", + }); + } + if (cause instanceof AkuaUserError) { + return new AkuaPackageUserError({ + cause, + message: cause.message, + }); + } + return new AkuaRenderError({ + cause, + message: "Akua could not render the mate package", + }); +} + +export const renderMatePackage = Effect.fn("agent-os.renderMatePackage")( + function* ( + options: RenderMatePackageOptions, + akua: AkuaRenderer = defaultAkua, + ) { + const cluster = options.cluster ?? "in-cluster"; + const namespace = options.namespace ?? "agent-os"; + const address = formatKubernetesAgentAddress(options.mateId, { + cluster, + namespace, + }); + const outDir = resolve( + options.outDir ?? resolve(process.cwd(), "deploy", options.mateId), + ); + const temporaryDirectory = yield* Effect.tryPromise({ + try: () => mkdtemp(resolve(tmpdir(), "agent-os-mate-")), + catch: (cause) => + new MatePackageInputError({ + cause, + message: "Could not create temporary Akua inputs", + }), + }); + + const render = Effect.gen(function* () { + const inputsPath = resolve(temporaryDirectory, "inputs.json"); + yield* Effect.tryPromise({ + try: () => + writeFile( + inputsPath, + JSON.stringify( + { + address, + image: options.image ?? "agent-os:dev", + mateId: options.mateId, + namespace, + }, + null, + 2, + ), + ), + catch: (cause) => + new MatePackageInputError({ + cause, + message: "Could not write temporary Akua inputs", + }), + }); + yield* Effect.tryPromise({ + try: () => + akua.render({ + inputs: inputsPath, + out: outDir, + package: resolve(packageDirectory, "package.k"), + timeout: "30s", + }), + catch: mapAkuaError, + }); + + return { address, outDir } satisfies RenderedMatePackage; + }); + + return yield* render.pipe( + Effect.ensuring( + Effect.promise(() => rm(temporaryDirectory, { force: true, recursive: true })), + ), + ); + }, +); diff --git a/tools/agent-os/test/mate-package.test.ts b/tools/agent-os/test/mate-package.test.ts new file mode 100644 index 000000000..97b72dfee --- /dev/null +++ b/tools/agent-os/test/mate-package.test.ts @@ -0,0 +1,81 @@ +import { Effect } from "effect"; +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; + +import { + type AkuaRenderer, + renderMatePackage, +} from "../src/mate-package.ts"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => + rm(directory, { force: true, recursive: true }), + ), + ); +}); + +describe("agent-os mate render", () => { + test("passes typed inputs to the Akua SDK and reports the address", async () => { + const root = await mkdtemp(resolve(tmpdir(), "agent-os-mate-test-")); + temporaryDirectories.push(root); + const outDir = resolve(root, "rendered"); + let capturedInputs: unknown; + const akua: AkuaRenderer = { + async render(options) { + capturedInputs = JSON.parse( + await readFile(options.inputs ?? "", "utf8"), + ); + return { + files: [], + format: "raw-manifests", + hash: "sha256:test", + manifests: 2, + target: options.out ?? outDir, + }; + }, + }; + + const output = await Effect.runPromise( + renderMatePackage( + { + image: "ghcr.io/akua-dev/agent-os:test", + mateId: "scout-1", + namespace: "agent-os-demo", + outDir, + }, + akua, + ), + ); + + expect(output).toEqual({ + address: + "/k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr", + outDir, + }); + expect(capturedInputs).toEqual({ + address: + "/k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr", + image: "ghcr.io/akua-dev/agent-os:test", + mateId: "scout-1", + namespace: "agent-os-demo", + }); + }); + + test("keeps unexpected SDK failures in the typed error channel", async () => { + const akua: AkuaRenderer = { + render: () => Promise.reject(new Error("native addon failed")), + }; + const exit = await Effect.runPromiseExit( + renderMatePackage({ mateId: "scout-1" }, akua), + ); + + expect(exit._tag).toBe("Failure"); + expect(String(exit)).toContain("AkuaRenderError"); + expect(String(exit)).toContain("Akua could not render the mate package"); + }); +}); From 4d4294b15cbc51606110c29f38194b7b2bdd03a7 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 10:21:11 +0200 Subject: [PATCH 11/56] feat: bundle k9s in agent image --- .agents/skills/kubernetes-fleet/SKILL.md | 1 + Dockerfile | 16 ++++++++++++++++ THIRD_PARTY_NOTICES.md | 7 +++++++ docs/kubernetes.md | 2 +- tests/agent-os-container.test.sh | 4 ++++ 5 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index d39f7c1b1..4947dd9d0 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -15,6 +15,7 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. - The image includes Akua's CLI for direct use and the Agent OS tool uses `@akua-dev/sdk` in process. +- Use the bundled K9s terminal UI when it makes live Kubernetes inspection faster than individual `kubectl` reads. - Use `agent-os mate render ` when the prepared package is useful, then inspect or edit its ordinary YAML before applying it. - The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. diff --git a/Dockerfile b/Dockerfile index 2a1979ece..23f4753ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ ARG TREEHOUSE_VERSION=2.0.0 ARG NO_MISTAKES_VERSION=1.34.0 ARG BUN_VERSION=1.3.14 ARG AKUA_VERSION=0.8.25 +ARG K9S_VERSION=0.51.0 RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -106,6 +107,21 @@ RUN set -eu; \ mkdir -p /usr/share/licenses/akua; \ curl -fsSL "https://raw.githubusercontent.com/akua-dev/akua/v${AKUA_VERSION}/LICENSE" -o /usr/share/licenses/akua/LICENSE +RUN set -eu; \ + case "$TARGETARCH" in \ + amd64) sha=c3752ad51a5a4015a113819c4eeb6e55a4d0e4b8e652494797532f6fc8161dd7 ;; \ + arm64) sha=3ee05c82e5f9198928a4e86133608ba6a2c10a2244d6a7789e820f78319d640c ;; \ + *) echo "unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \ + esac; \ + asset="k9s_Linux_${TARGETARCH}.tar.gz"; \ + curl -fsSL "https://github.com/derailed/k9s/releases/download/v${K9S_VERSION}/${asset}" -o "/tmp/${asset}"; \ + echo "$sha /tmp/${asset}" | sha256sum -c -; \ + tar -xzf "/tmp/${asset}" -C /tmp k9s; \ + install -m 0755 /tmp/k9s /usr/local/bin/k9s; \ + rm -f "/tmp/${asset}" /tmp/k9s; \ + mkdir -p /usr/share/licenses/k9s; \ + curl -fsSL "https://raw.githubusercontent.com/derailed/k9s/v${K9S_VERSION}/LICENSE" -o /usr/share/licenses/k9s/LICENSE + RUN npm install --global \ @earendil-works/pi-coding-agent@0.80.6 \ gh-axi@0.1.27 \ diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 0049de365..7e07a1233 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -15,3 +15,10 @@ Akua is available under the Apache License 2.0. The image includes Akua's license at `/usr/share/licenses/akua/LICENSE`. The exact source is available at . Agent OS invokes Akua through its documented CLI. + +## K9s + +The Agent OS demo image includes an unmodified K9s 0.51.0 executable as a separate program. +K9s is available under the Apache License 2.0. +The image includes K9s's license at `/usr/share/licenses/k9s/LICENSE`. +The exact source is available at . diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 553bba78e..afad836fa 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -35,7 +35,7 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` -The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, treehouse, no-mistakes, and every required AXI CLI. +The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, `kubectl`, K9s, treehouse, no-mistakes, and every required AXI CLI. Authenticate GitHub inside the primary with `gh auth login`. Authenticate Pi using `/login` and the provider flow you choose. Host credentials are intentionally excluded from the image and are never copied automatically. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 1ceefb506..c37ea3b2b 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -13,6 +13,7 @@ assert_grep 'ARG TREEHOUSE_VERSION=2.0.0' "$ROOT/Dockerfile" "image must pin tre assert_grep 'ARG NO_MISTAKES_VERSION=1.34.0' "$ROOT/Dockerfile" "image must pin no-mistakes 1.34.0" assert_grep 'ARG BUN_VERSION=1.3.14' "$ROOT/Dockerfile" "image must pin stable Bun 1.3.14" assert_grep 'ARG AKUA_VERSION=0.8.25' "$ROOT/Dockerfile" "image must pin Akua 0.8.25" +assert_grep 'ARG K9S_VERSION=0.51.0' "$ROOT/Dockerfile" "image must pin K9s 0.51.0" assert_grep '"@akua-dev/sdk": "0.8.24"' "$ROOT/tools/agent-os/package.json" "tool must pin the Akua SDK" assert_grep 'sha256sum -c -' "$ROOT/Dockerfile" "downloaded runtime binaries must be checksum verified" assert_grep '@earendil-works/pi-coding-agent@0.80.6' "$ROOT/Dockerfile" "image must pin Pi 0.80.6" @@ -30,6 +31,7 @@ assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo "persistent tool prefixes must lead PATH" assert_grep '/opt/image-usr-local' "$ROOT/Dockerfile" "image must retain a seed copy of /usr/local" assert_grep 'akua-dev/akua/releases/download/v${AKUA_VERSION}' "$ROOT/Dockerfile" "image must install Akua from its release" +assert_grep 'derailed/k9s/releases/download/v${K9S_VERSION}' "$ROOT/Dockerfile" "image must install K9s from its release" assert_grep 'ln -s /opt/agent-os/tools/agent-os/src/cli.ts /usr/local/bin/agent-os' "$ROOT/Dockerfile" \ "image must expose the Agent OS tool" assert_grep 'bun install --frozen-lockfile --production --ignore-scripts' "$ROOT/Dockerfile" \ @@ -46,6 +48,8 @@ assert_grep 'https://github.com/ogulcancelik/herdr/tree/v0.7.3' "$ROOT/THIRD_PAR "Herdr's exact corresponding source must be named" assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_NOTICES.md" \ "Akua's exact source must be named" +assert_grep 'https://github.com/derailed/k9s/tree/v0.51.0' "$ROOT/THIRD_PARTY_NOTICES.md" \ + "K9s's exact source must be named" bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" pass "container files pin dependencies and exclude host credentials" From 9837e8ad654847dffb1622656826f3e3acb6eb94 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 10:50:23 +0200 Subject: [PATCH 12/56] refactor: keep mate workflows tool-native --- .agents/skills/kubernetes-fleet/SKILL.md | 11 +- bin/agent-os-container-entrypoint.sh | 2 + bin/agent-os-init.sh | 1 + bin/agent-os-kubeconfig.sh | 35 ++++ docs/agent-evals.md | 48 ++++++ docs/kubernetes.md | 42 ++++- tests/agent-os-container.test.sh | 15 +- tests/agent-os-kubeconfig.test.sh | 36 +++++ tools/agent-os/AGENTS.md | 18 ++- tools/agent-os/README.md | 4 +- tools/agent-os/bun.lock | 42 ----- tools/agent-os/package.json | 4 +- tools/agent-os/packages/mate/README.md | 10 +- .../packages/mate/inputs.example.yaml | 1 + tools/agent-os/packages/mate/package.k | 20 ++- tools/agent-os/src/cli.ts | 65 +------- tools/agent-os/src/json-modules.d.ts | 4 - tools/agent-os/src/mate-package.ts | 151 ------------------ tools/agent-os/test/mate-package.test.ts | 81 ---------- 19 files changed, 223 insertions(+), 367 deletions(-) create mode 100755 bin/agent-os-kubeconfig.sh create mode 100644 docs/agent-evals.md create mode 100755 tests/agent-os-kubeconfig.test.sh delete mode 100644 tools/agent-os/src/json-modules.d.ts delete mode 100644 tools/agent-os/src/mate-package.ts delete mode 100644 tools/agent-os/test/mate-package.test.ts diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index 4947dd9d0..6ecbd247c 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -14,16 +14,21 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. -- The image includes Akua's CLI for direct use and the Agent OS tool uses `@akua-dev/sdk` in process. +- The image includes Akua's CLI and the prepared mate package lives at `tools/agent-os/packages/mate/`. - Use the bundled K9s terminal UI when it makes live Kubernetes inspection faster than individual `kubectl` reads. -- Use `agent-os mate render ` when the prepared package is useful, then inspect or edit its ordinary YAML before applying it. +- Render the package with `akua render`, inspect or edit its ordinary YAML when useful, and apply it with `kubectl`. +- Treat AI credentials as explicit per-mate grants, never as ambient inheritance merely because Firstmate can read them. +- Create or select a Kubernetes Secret only after the grant is authorized, then pass its name as the package's `piAuthSecret` input. +- Give every launched Herdr agent a task-unique name, close only a confirmed dead restored pane before reuse, and never replace a live agent. +- Grade completion by the promised artifact or delivered Git state; Herdr `idle` alone is not a completion signal. - The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. - Use `bin/agent-os-crewmate.sh create ` to create a separate Pod and persistent home. - Use `status` to inspect it and `delete` only after unique work is checkpointed or delivered. - Never mount the primary home into a child Pod. - The demo child receives no Kubernetes ServiceAccount token by default. +- A Pod with an authorized ServiceAccount gets a token-file-backed `in-cluster` kubeconfig automatically, so Firstmate does not need to copy a bearer token into a temporary kubeconfig. - The OrbStack primary's cluster-admin binding is a local-demo trust decision, not a production-safe default. -- Pin an explicit host context with `AGENT_OS_CONTEXT`; the launcher refuses ambient host contexts. +- Pin an explicit host context for host-side `kubectl`; inside an authorized Pod use the generated `in-cluster` context. Use ordinary `kubectl exec`, Herdr, files, and Git to supervise the child. Do not add a custom inter-agent chat protocol or Task/Run service. diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index 0cf4d203e..a94ddaaf6 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -16,6 +16,8 @@ mkdir -p \ "$HOME/.local/share" \ "$HOME/.bun" \ "$HOME/.cargo" + +"$(dirname "$0")/agent-os-kubeconfig.sh" if [ ! -e "$FM_HOME/config/backend" ]; then printf 'herdr\n' > "$FM_HOME/config/backend" fi diff --git a/bin/agent-os-init.sh b/bin/agent-os-init.sh index 4b88ae911..385f5bd7d 100755 --- a/bin/agent-os-init.sh +++ b/bin/agent-os-init.sh @@ -10,6 +10,7 @@ mkdir -p \ "$PERSISTENT_ROOT/.cache" \ "$PERSISTENT_ROOT/.local/bin" \ "$PERSISTENT_ROOT/.local/share" \ + "$PERSISTENT_ROOT/.pi/agent" \ "$PERSISTENT_ROOT/.bun" \ "$PERSISTENT_ROOT/.cargo" \ "$PERSISTENT_ROOT/usr-local" diff --git a/bin/agent-os-kubeconfig.sh b/bin/agent-os-kubeconfig.sh new file mode 100755 index 000000000..aeef3c2c6 --- /dev/null +++ b/bin/agent-os-kubeconfig.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Create a token-file-backed kubeconfig when this container has a ServiceAccount. +set -eu + +HOME=${HOME:-/home/agent} +service_account_dir=${AGENT_OS_SERVICE_ACCOUNT_DIR:-/var/run/secrets/kubernetes.io/serviceaccount} +kubeconfig=${AGENT_OS_KUBECONFIG_PATH:-$HOME/.kube/config} + +[ -r "$service_account_dir/token" ] || exit 0 +[ -r "$service_account_dir/ca.crt" ] || exit 0 +[ -e "$kubeconfig" ] && exit 0 + +: "${KUBERNETES_SERVICE_HOST:?KUBERNETES_SERVICE_HOST is required with a ServiceAccount token}" +mkdir -p "$(dirname "$kubeconfig")" +umask 077 +cat > "$kubeconfig" < "$tmp/serviceaccount/token" +printf '%s\n' demo-ca > "$tmp/serviceaccount/ca.crt" + +HOME="$tmp/home" \ +AGENT_OS_SERVICE_ACCOUNT_DIR="$tmp/serviceaccount" \ +KUBERNETES_SERVICE_HOST=kubernetes.default.svc \ +KUBERNETES_SERVICE_PORT_HTTPS=443 \ +AGENT_OS_NAMESPACE=agent-os-eval \ + "$ROOT/bin/agent-os-kubeconfig.sh" + +config="$tmp/home/.kube/config" +[ -f "$config" ] || fail "in-cluster kubeconfig was not created" +assert_grep 'current-context: in-cluster' "$config" "kubeconfig must select the local cluster alias" +assert_grep 'namespace: agent-os-eval' "$config" "kubeconfig must select the Agent OS namespace" +assert_grep "tokenFile: $tmp/serviceaccount/token" "$config" "kubeconfig must follow the rotating token file" +assert_no_grep 'demo-token' "$config" "kubeconfig must not copy token contents" +[ "$(stat -f '%Lp' "$config" 2>/dev/null || stat -c '%a' "$config")" = 600 ] || fail "kubeconfig must be mode 600" + +printf '%s\n' sentinel > "$config" +HOME="$tmp/home" \ +AGENT_OS_SERVICE_ACCOUNT_DIR="$tmp/serviceaccount" \ +KUBERNETES_SERVICE_HOST=kubernetes.default.svc \ + "$ROOT/bin/agent-os-kubeconfig.sh" +[ "$(cat "$config")" = sentinel ] || fail "existing kubeconfig must not be overwritten" + +pass "Agent OS creates a rotation-safe in-cluster kubeconfig without exposing token contents" diff --git a/tools/agent-os/AGENTS.md b/tools/agent-os/AGENTS.md index 872166ee7..ed564429e 100644 --- a/tools/agent-os/AGENTS.md +++ b/tools/agent-os/AGENTS.md @@ -5,7 +5,7 @@ They govern development of the operating tool and must not be copied into the ro ## Purpose -This tool incrementally moves complex fleet mechanics from Bash into a small Bun and TypeScript CLI. +This tool owns only stable Agent OS primitives that the underlying general-purpose tools do not already provide. It extends Firstmate's existing files and lifecycle rather than introducing a parallel state store, controller, or agent model. Keep compatibility with the root distro's `data/`, `state/`, briefs, status events, and guarded teardown behavior. Use the Bun channel declared in `.bun-version`. @@ -31,26 +31,30 @@ The address grammar and validation contract are owned by `src/address.ts` and it ## Command Execution -Use the official `@akua-dev/sdk` for Akua operations implemented inside this TypeScript tool. -Keep the bundled `akua` CLI available for Firstmate's direct and interactive use. -Use Bun Shell's `$` for other finite calls to trusted CLIs such as `kubectl`, `herdr`, `treehouse`, and Git. +Keep the bundled `akua` CLI available for Firstmate's direct use instead of proxying it through this tool. +Use Bun Shell's `$` for finite calls to trusted CLIs only when implementing a genuinely missing Agent OS primitive. Interpolate dynamic values directly so Bun treats each value as a literal argument. Never use `{ raw: value }` for dynamic data. Avoid `bash -c` and validate values that an external command could interpret as flags. Keep non-zero exits throwing by default and use `.nothrow()` only for an expected negative probe. Set environment and working directory per command instead of mutating the global `$` instance. Use `Bun.spawn()` for interactive agents, servers, watches, live logs, cancellation, and other long-running processes. -Existing Bash scripts may be invoked explicitly during migration, but new orchestration and state transitions belong in TypeScript. +Existing Bash scripts may be invoked explicitly during migration, but do not move adaptable orchestration into TypeScript merely because it is currently written in Bash or instructions. ## Akua packages The prepared mate package lives at `packages/mate/` and renders ordinary Kubernetes YAML. Treat it as an editable convenience, not a required abstraction or controller. -Firstmate may use `agent-os mate render`, invoke `akua render` itself, edit the result, or author equivalent YAML directly. +Firstmate may invoke `akua render`, edit the result, change the package, or author equivalent YAML directly. ## Scope Discipline -Prefer a thin adapter over a new framework. +Prefer no adapter when an agent can reliably compose the underlying tools itself. +When a stable safety boundary truly needs code, prefer the thinnest adapter that owns only that boundary. +Do not wrap a capable general-purpose CLI merely to encode one Agent OS workflow. +Prefer instructions, skills, and direct composition of Akua, Kubernetes, Herdr, Git, and other trusted tools. +Add TypeScript only for a stable primitive the underlying tools do not already provide, such as the Agent OS address grammar, or when repeated failures prove a reusable safety boundary is missing. +Keep policy and desired outcomes in instructions instead of hardcoding how an adaptable agent must sequence tools. Do not add a database, daemon, custom inter-agent protocol, Kubernetes controller, or duplicate task specification. Keep mates general-purpose and keep parent-only communication unchanged across local and remote addresses. Use Kubernetes APIs and later Agent Sandbox only as runtime resolvers behind the same logical address. diff --git a/tools/agent-os/README.md b/tools/agent-os/README.md index 7776509b3..f0362dc01 100644 --- a/tools/agent-os/README.md +++ b/tools/agent-os/README.md @@ -23,11 +23,9 @@ bun run check bun run src/cli.ts address local task-x bun run src/cli.ts address k8s task-x bun run src/cli.ts address inspect /k8s/in-cluster/namespace/agent-os/mate/task-x/herdr -bun run src/cli.ts mate render task-x --out /tmp/task-x ``` -Akua package operations inside this tool use the official `@akua-dev/sdk` in process. -The prepared package under `packages/mate/` remains optional and renders editable Kubernetes YAML. +The prepared Akua package under `packages/mate/` remains optional and renders editable Kubernetes YAML through the bundled `akua` CLI. The scoped official `Effect-TS/skills` installation guides typed Effect workflows; its Effect source checkout is local and ignored rather than committed as a snapshot. See `AGENTS.md` in this directory for the tool-specific development rules. diff --git a/tools/agent-os/bun.lock b/tools/agent-os/bun.lock index c0ea2c18d..717d11312 100644 --- a/tools/agent-os/bun.lock +++ b/tools/agent-os/bun.lock @@ -5,9 +5,7 @@ "": { "name": "@akua-dev/agent-os-tool", "dependencies": { - "@akua-dev/sdk": "0.8.24", "@multiformats/multiaddr": "13.0.3", - "effect": "3.21.4", }, "devDependencies": { "@types/bun": "1.3.14", @@ -16,32 +14,10 @@ }, }, "packages": { - "@akua-dev/native": ["@akua-dev/native@0.8.24", "", { "dependencies": { "@akua-dev/native-engines": "0.8.24" }, "optionalDependencies": { "@akua-dev/native-darwin-arm64": "0.8.24", "@akua-dev/native-darwin-x64": "0.8.24", "@akua-dev/native-linux-arm64-gnu": "0.8.24", "@akua-dev/native-linux-arm64-musl": "0.8.24", "@akua-dev/native-linux-x64-gnu": "0.8.24", "@akua-dev/native-linux-x64-musl": "0.8.24", "@akua-dev/native-win32-x64-msvc": "0.8.24" } }, "sha512-DZsUSd0SaLrqBvbmsEmPD9rUwxrUZSC/QTNfclzRgmKxnK2y5x90pT+J4L/Aoa3lCFWpCp9MOM9pZ7ztjHL1Nw=="], - - "@akua-dev/native-darwin-arm64": ["@akua-dev/native-darwin-arm64@0.8.24", "", { "os": "darwin", "cpu": "arm64" }, "sha512-6Wa2zDcrRWQUPouHKmZUom4kNmffvWYxcrvnaeoVwbQFj3TXWKAaCzcVYjZGeLcV8HxMkTrYx4rt/NH5GRLkWA=="], - - "@akua-dev/native-darwin-x64": ["@akua-dev/native-darwin-x64@0.8.24", "", { "os": "darwin", "cpu": "x64" }, "sha512-hhXiqfpw9itn6cdNhC0IVW1WUaPBmpYmBPW7hfkBuaTlP/2wPyAUx+Wu0cJWpna1YCUg3WzcUOj5rm/lyciP9w=="], - - "@akua-dev/native-engines": ["@akua-dev/native-engines@0.8.24", "", {}, "sha512-3rKiWQn1pIURERUmxNiqPw7SxtMrNwX5tDxsCRxsAZ7Ovzfzcwx/VcycfuY+z1DNhEJrkKChkYiMT3N/EZanmg=="], - - "@akua-dev/native-linux-arm64-gnu": ["@akua-dev/native-linux-arm64-gnu@0.8.24", "", { "os": "linux", "cpu": "arm64" }, "sha512-NTeuJRCjy68JJT5LuHwPHf11mnZ1a5hNeKNfoBYMzUeViN3BdnBuszjNxwYHAgQYEN8+Rf5EGtTMdOTm0xDy/Q=="], - - "@akua-dev/native-linux-arm64-musl": ["@akua-dev/native-linux-arm64-musl@0.8.24", "", { "os": "linux", "cpu": "arm64" }, "sha512-QbGJfTY2Ilv+FSVaIU5MnlHjprbKVM41Ya+YfanHJsTBwO0dUw6J9C/fNVKTHeQM98vLVBIWdLiAPl73jSI/Qw=="], - - "@akua-dev/native-linux-x64-gnu": ["@akua-dev/native-linux-x64-gnu@0.8.24", "", { "os": "linux", "cpu": "x64" }, "sha512-jJDnr4MGYxB/vflhGklC1dUxHsO7HNuWexDFGB/vPhzo6RGsAWsApnrqPYYfBDmsk0nvvZKO9OWTB+e837gCzA=="], - - "@akua-dev/native-linux-x64-musl": ["@akua-dev/native-linux-x64-musl@0.8.24", "", { "os": "linux", "cpu": "x64" }, "sha512-21EsbM2fArJjX4RKXeVXDPxSBHkeMtYVCvT9vDj4S1lo5Ld0xRM/utm6EQkfbShdpwWZRhkv/t7evgdVM8Zb2A=="], - - "@akua-dev/native-win32-x64-msvc": ["@akua-dev/native-win32-x64-msvc@0.8.24", "", { "os": "win32", "cpu": "x64" }, "sha512-boG/GW6KTucoq7PUlU92tvB91DGxl4hNdQICcsXwgO7npzLqNEhCzKYnpuwKgqhK6vSpoPd0/jZdmu6EwUyzKQ=="], - - "@akua-dev/sdk": ["@akua-dev/sdk@0.8.24", "", { "dependencies": { "@akua-dev/native": "0.8.24", "@standard-schema/spec": "^1.1.0", "ajv": "^8.18.0", "yaml": "^2.8.3" } }, "sha512-8OM8luAqmxnA4kRJUGbf4Jhr4YuSf/lXKAOUD4kZTUepoRvgYP3xHlXJRCVd1Qv00L1NmeluXG8sr5rh/soGIg=="], - "@chainsafe/is-ip": ["@chainsafe/is-ip@2.1.0", "", {}, "sha512-KIjt+6IfysQ4GCv66xihEitBjvhU/bixbbbFxdJ1sqCp4uJ0wuZiYBPhksZoy4lfaF0k9cwNzY5upEW/VWdw3w=="], "@multiformats/multiaddr": ["@multiformats/multiaddr@13.0.3", "", { "dependencies": { "@chainsafe/is-ip": "^2.0.1", "multiformats": "^14.0.0", "uint8-varint": "^3.0.0", "uint8arrays": "^6.1.1" } }, "sha512-mEqqJ4r3a/uuFMTpRkU316wGNIDQNhuVWpm+ebKTQeYsfv9jXbPONWM6VVnj3KGUrwfsX7GZOyp4TFqEA2SPCw=="], - "@standard-schema/spec": ["@standard-schema/spec@1.1.0", "", {}, "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=="], - "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/node": ["@types/node@26.1.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw=="], @@ -86,26 +62,10 @@ "@typescript/typescript-win32-x64": ["@typescript/typescript-win32-x64@7.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g=="], - "ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], - "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], - "effect": ["effect@3.21.4", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "fast-check": "^3.23.1" } }, "sha512-B89v/xSgPbl1J2Ai2u18jxq3odpFauU1rC6/eSs4FeNHi72kwKdJp12VGigvRV2lK+kRnx+OOz41XV8guZd4gQ=="], - - "fast-check": ["fast-check@3.23.2", "", { "dependencies": { "pure-rand": "^6.1.0" } }, "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A=="], - - "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], - - "fast-uri": ["fast-uri@3.1.3", "", {}, "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg=="], - - "json-schema-traverse": ["json-schema-traverse@1.0.0", "", {}, "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug=="], - "multiformats": ["multiformats@14.0.4", "", {}, "sha512-+SXItmOUWzT0kjlIfkZ4NawJaC9jW/mPlyn902V9Qgpc7YDwdEiwqbiZxTyvC0XWh1jZguXca3A/WQ+UOPRLUA=="], - "pure-rand": ["pure-rand@6.1.0", "", {}, "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA=="], - - "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], - "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], "uint8-varint": ["uint8-varint@3.0.0", "", { "dependencies": { "uint8arraylist": "^3.0.1", "uint8arrays": "^6.1.0" } }, "sha512-S4DdpXBaLwKcFo7f0bWzWfHjbZ/i3QhM842qn+ZvHjxqFCfUcEB9SQNcmI69S+zMlcmIcKxsk9Iyw77S2Kxv6Q=="], @@ -115,7 +75,5 @@ "uint8arrays": ["uint8arrays@6.1.1", "", { "dependencies": { "multiformats": "^14.0.0" } }, "sha512-iz7JN0XCSZYA111lhFG2Ui9EhFvTNekqSRHw3lvMHq+dzwWy1OQftxFQREEh4rffU0oSoXdQHsk2TiHKVm4fsA=="], "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], - - "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], } } diff --git a/tools/agent-os/package.json b/tools/agent-os/package.json index ba519baaa..cf3f52ee6 100644 --- a/tools/agent-os/package.json +++ b/tools/agent-os/package.json @@ -13,9 +13,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@akua-dev/sdk": "0.8.24", - "@multiformats/multiaddr": "13.0.3", - "effect": "3.21.4" + "@multiformats/multiaddr": "13.0.3" }, "devDependencies": { "@types/bun": "1.3.14", diff --git a/tools/agent-os/packages/mate/README.md b/tools/agent-os/packages/mate/README.md index 5576ebec4..71c5fe0b0 100644 --- a/tools/agent-os/packages/mate/README.md +++ b/tools/agent-os/packages/mate/README.md @@ -9,12 +9,6 @@ Render the example directly: akua render --package package.k --inputs inputs.example.yaml --out /tmp/scout-1 ``` -Or use the Agent OS convenience command from any directory: - -```sh -agent-os mate render scout-1 --namespace agent-os-demo --image agent-os:dev --out /tmp/scout-1 -``` - Inspect and edit the rendered YAML before applying it: ```sh @@ -23,3 +17,7 @@ kubectl apply -f /tmp/scout-1 Firstmate may change this package, supply additional inputs, edit the rendered files, or author equivalent YAML directly. The package intentionally grants no Kubernetes token and no privileged, host-namespace, or host-mount access. + +AI credentials are an explicit grant. +Set `piAuthSecret` to the name of an existing Kubernetes Secret whose `auth.json` key contains the selected Pi credential set. +The package mounts only that Secret reference read-only; it never discovers or copies credentials by itself. diff --git a/tools/agent-os/packages/mate/inputs.example.yaml b/tools/agent-os/packages/mate/inputs.example.yaml index 89d21d97f..3f0136f73 100644 --- a/tools/agent-os/packages/mate/inputs.example.yaml +++ b/tools/agent-os/packages/mate/inputs.example.yaml @@ -3,3 +3,4 @@ namespace: agent-os-demo address: /k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr image: agent-os:dev imagePullPolicy: Never +piAuthSecret: agent-os-mate-scout-1-pi-auth diff --git a/tools/agent-os/packages/mate/package.k b/tools/agent-os/packages/mate/package.k index f12829369..2ed2259c6 100644 --- a/tools/agent-os/packages/mate/package.k +++ b/tools/agent-os/packages/mate/package.k @@ -8,6 +8,7 @@ schema Input: address: str image: str = "agent-os:dev" imagePullPolicy: "Always" | "IfNotPresent" | "Never" = "IfNotPresent" + piAuthSecret: str = "" storage: str = "10Gi" cpuRequest: str = "250m" memoryRequest: str = "512Mi" @@ -25,6 +26,21 @@ labels = { "agent-os.akua.dev/mate" = input.mateId } +piAuthMounts = [{ + name = "pi-auth" + mountPath = "/home/agent/.pi/agent/auth.json" + subPath = "auth.json" + readOnly = True +}] if input.piAuthSecret else [] + +piAuthVolumes = [{ + name = "pi-auth" + secret = { + secretName = input.piAuthSecret + defaultMode = 256 + } +}] if input.piAuthSecret else [] + resources = [ { apiVersion = "v1" @@ -90,12 +106,12 @@ resources = [ volumeMounts = [ { name = "home", mountPath = "/home/agent" }, { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, - ] + ] + piAuthMounts }] volumes = [{ name = "home" persistentVolumeClaim = { claimName = "agent-os-mate-${input.mateId}-home" } - }] + }] + piAuthVolumes } }, ] diff --git a/tools/agent-os/src/cli.ts b/tools/agent-os/src/cli.ts index 48830c7d0..5cd221ed4 100755 --- a/tools/agent-os/src/cli.ts +++ b/tools/agent-os/src/cli.ts @@ -1,20 +1,17 @@ #!/usr/bin/env bun import { parseArgs } from "node:util"; -import { Effect } from "effect"; import { formatKubernetesAgentAddress, formatLocalAgentAddress, parseAgentAddress, } from "./address.ts"; -import { renderMatePackage } from "./mate-package.ts"; const usage = `Usage: agent-os address local agent-os address k8s [--cluster ] [--namespace ] - agent-os address inspect - agent-os mate render [--cluster ] [--namespace ] [--image ] [--out ]`; + agent-os address inspect `; function requireSingleArgument(args: string[]): string { if (args.length !== 1 || args[0] == null) { @@ -70,68 +67,18 @@ function runAddressCommand(args: string[]): void { } } -async function runMateCommand(args: string[]): Promise { - const [command, ...rest] = args; +function main(): void { + const [command, ...rest] = Bun.argv.slice(2); - if (command !== "render") { + if (command !== "address") { throw new Error(usage); } - const { positionals, values } = parseArgs({ - allowPositionals: true, - args: rest, - options: { - cluster: { - default: "in-cluster", - type: "string", - }, - image: { - default: "agent-os:dev", - type: "string", - }, - namespace: { - default: "agent-os", - type: "string", - }, - out: { - type: "string", - }, - }, - strict: true, - }); - const mateId = requireSingleArgument(positionals); - const result = await Effect.runPromise( - renderMatePackage({ - cluster: values.cluster, - image: values.image, - mateId, - namespace: values.namespace, - outDir: values.out, - }).pipe( - Effect.catchAll((error) => Effect.fail(new Error(error.message))), - ), - ); - - console.log(JSON.stringify(result, null, 2)); -} - -async function main(): Promise { - const [command, ...rest] = Bun.argv.slice(2); - - switch (command) { - case "address": - runAddressCommand(rest); - return; - case "mate": - await runMateCommand(rest); - return; - default: - throw new Error(usage); - } + runAddressCommand(rest); } try { - await main(); + main(); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; diff --git a/tools/agent-os/src/json-modules.d.ts b/tools/agent-os/src/json-modules.d.ts deleted file mode 100644 index b14866845..000000000 --- a/tools/agent-os/src/json-modules.d.ts +++ /dev/null @@ -1,4 +0,0 @@ -declare module "*.json" { - const value: { readonly $defs: Readonly> }; - export default value; -} diff --git a/tools/agent-os/src/mate-package.ts b/tools/agent-os/src/mate-package.ts deleted file mode 100644 index 9d9e0306d..000000000 --- a/tools/agent-os/src/mate-package.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { - Akua, - AkuaRateLimitedError, - AkuaUserError, - type RenderOptions, - type RenderSummary, -} from "@akua-dev/sdk"; -import { Data, Effect } from "effect"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -import { formatKubernetesAgentAddress } from "./address.ts"; - -export interface RenderMatePackageOptions { - cluster?: string; - image?: string; - mateId: string; - namespace?: string; - outDir?: string; -} - -export interface RenderedMatePackage { - address: string; - outDir: string; -} - -export class MatePackageInputError extends Data.TaggedError( - "MatePackageInputError", -)<{ - readonly cause: unknown; - readonly message: string; -}> {} - -export class AkuaRenderError extends Data.TaggedError("AkuaRenderError")<{ - readonly cause: unknown; - readonly message: string; -}> {} - -export class AkuaPackageUserError extends Data.TaggedError( - "AkuaPackageUserError", -)<{ - readonly cause: Error; - readonly message: string; -}> {} - -export class AkuaPackageRateLimitedError extends Data.TaggedError( - "AkuaPackageRateLimitedError", -)<{ - readonly cause: Error; - readonly message: string; -}> {} - -export interface AkuaRenderer { - render(options: RenderOptions): Promise; -} - -const packageDirectory = resolve( - dirname(fileURLToPath(import.meta.url)), - "../packages/mate", -); - -const defaultAkua = new Akua(); - -function mapAkuaError(cause: unknown) { - if (cause instanceof AkuaRateLimitedError) { - return new AkuaPackageRateLimitedError({ - cause, - message: "Akua rate-limited the mate package render", - }); - } - if (cause instanceof AkuaUserError) { - return new AkuaPackageUserError({ - cause, - message: cause.message, - }); - } - return new AkuaRenderError({ - cause, - message: "Akua could not render the mate package", - }); -} - -export const renderMatePackage = Effect.fn("agent-os.renderMatePackage")( - function* ( - options: RenderMatePackageOptions, - akua: AkuaRenderer = defaultAkua, - ) { - const cluster = options.cluster ?? "in-cluster"; - const namespace = options.namespace ?? "agent-os"; - const address = formatKubernetesAgentAddress(options.mateId, { - cluster, - namespace, - }); - const outDir = resolve( - options.outDir ?? resolve(process.cwd(), "deploy", options.mateId), - ); - const temporaryDirectory = yield* Effect.tryPromise({ - try: () => mkdtemp(resolve(tmpdir(), "agent-os-mate-")), - catch: (cause) => - new MatePackageInputError({ - cause, - message: "Could not create temporary Akua inputs", - }), - }); - - const render = Effect.gen(function* () { - const inputsPath = resolve(temporaryDirectory, "inputs.json"); - yield* Effect.tryPromise({ - try: () => - writeFile( - inputsPath, - JSON.stringify( - { - address, - image: options.image ?? "agent-os:dev", - mateId: options.mateId, - namespace, - }, - null, - 2, - ), - ), - catch: (cause) => - new MatePackageInputError({ - cause, - message: "Could not write temporary Akua inputs", - }), - }); - yield* Effect.tryPromise({ - try: () => - akua.render({ - inputs: inputsPath, - out: outDir, - package: resolve(packageDirectory, "package.k"), - timeout: "30s", - }), - catch: mapAkuaError, - }); - - return { address, outDir } satisfies RenderedMatePackage; - }); - - return yield* render.pipe( - Effect.ensuring( - Effect.promise(() => rm(temporaryDirectory, { force: true, recursive: true })), - ), - ); - }, -); diff --git a/tools/agent-os/test/mate-package.test.ts b/tools/agent-os/test/mate-package.test.ts deleted file mode 100644 index 97b72dfee..000000000 --- a/tools/agent-os/test/mate-package.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Effect } from "effect"; -import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { resolve } from "node:path"; - -import { - type AkuaRenderer, - renderMatePackage, -} from "../src/mate-package.ts"; - -const temporaryDirectories: string[] = []; - -afterEach(async () => { - await Promise.all( - temporaryDirectories.splice(0).map((directory) => - rm(directory, { force: true, recursive: true }), - ), - ); -}); - -describe("agent-os mate render", () => { - test("passes typed inputs to the Akua SDK and reports the address", async () => { - const root = await mkdtemp(resolve(tmpdir(), "agent-os-mate-test-")); - temporaryDirectories.push(root); - const outDir = resolve(root, "rendered"); - let capturedInputs: unknown; - const akua: AkuaRenderer = { - async render(options) { - capturedInputs = JSON.parse( - await readFile(options.inputs ?? "", "utf8"), - ); - return { - files: [], - format: "raw-manifests", - hash: "sha256:test", - manifests: 2, - target: options.out ?? outDir, - }; - }, - }; - - const output = await Effect.runPromise( - renderMatePackage( - { - image: "ghcr.io/akua-dev/agent-os:test", - mateId: "scout-1", - namespace: "agent-os-demo", - outDir, - }, - akua, - ), - ); - - expect(output).toEqual({ - address: - "/k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr", - outDir, - }); - expect(capturedInputs).toEqual({ - address: - "/k8s/in-cluster/namespace/agent-os-demo/mate/scout-1/herdr", - image: "ghcr.io/akua-dev/agent-os:test", - mateId: "scout-1", - namespace: "agent-os-demo", - }); - }); - - test("keeps unexpected SDK failures in the typed error channel", async () => { - const akua: AkuaRenderer = { - render: () => Promise.reject(new Error("native addon failed")), - }; - const exit = await Effect.runPromiseExit( - renderMatePackage({ mateId: "scout-1" }, akua), - ); - - expect(exit._tag).toBe("Failure"); - expect(String(exit)).toContain("AkuaRenderError"); - expect(String(exit)).toContain("Akua could not render the mate package"); - }); -}); From 7c73fccf3dcf4cb969baf841b996f819e639ec85 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 12:01:10 +0200 Subject: [PATCH 13/56] feat: add Akua Firstmate bootstrap package --- .../akua-intelligence-bootstrap/SKILL.md | 84 ++++++++ .agents/skills/kubernetes-fleet/SKILL.md | 1 + .github/workflows/agent-os-image.yml | 61 ++++++ AGENTS.md | 1 + README.md | 6 +- docs/acceptance.md | 26 +++ docs/evidence/2026-07-12-firstmate-package.md | 70 +++++++ tests/agent-os-container.test.sh | 6 + tests/agent-os-packages.test.sh | 36 ++++ tools/agent-os/packages/firstmate/README.md | 21 ++ tools/agent-os/packages/firstmate/akua.toml | 5 + .../packages/firstmate/inputs.example.yaml | 5 + tools/agent-os/packages/firstmate/package.k | 182 ++++++++++++++++++ 13 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 .agents/skills/akua-intelligence-bootstrap/SKILL.md create mode 100644 .github/workflows/agent-os-image.yml create mode 100644 docs/acceptance.md create mode 100644 docs/evidence/2026-07-12-firstmate-package.md create mode 100755 tests/agent-os-packages.test.sh create mode 100644 tools/agent-os/packages/firstmate/README.md create mode 100644 tools/agent-os/packages/firstmate/akua.toml create mode 100644 tools/agent-os/packages/firstmate/inputs.example.yaml create mode 100644 tools/agent-os/packages/firstmate/package.k diff --git a/.agents/skills/akua-intelligence-bootstrap/SKILL.md b/.agents/skills/akua-intelligence-bootstrap/SKILL.md new file mode 100644 index 000000000..d32a28491 --- /dev/null +++ b/.agents/skills/akua-intelligence-bootstrap/SKILL.md @@ -0,0 +1,84 @@ +--- +name: akua-intelligence-bootstrap +description: "Bootstrap or recover a dedicated Agent OS intelligence cluster through Akua's native API, MCP, CLI, Akua packages, and Kubernetes without adding an Agent OS provisioning service." +user-invocable: false +metadata: + internal: true +--- + +# Akua intelligence bootstrap + +Load this skill before provisioning, recovering, or handing off a Firstmate in an Akua-managed intelligence cluster. + +## Boundary + +- The Akua workspace and Hetzner project must be dedicated to Agent OS infrastructure and contain no product, production, or customer resources. +- Provisioning machines, creating or revoking credentials, retrieving kubeconfig, and granting cluster-admin require the captain's explicit approval for the exact run. +- An available credential proves capability, not approval. +- Use Akua's native Platform MCP, REST API, CLI, and packages directly. +- Do not build or invoke an Agent OS provisioning wrapper, controller, Install, or GitOps workflow. +- Use a fresh idempotency key for each intended create and reuse that key only when retrying the same intended operation. +- Never put an Akua token, Hetzner token, kubeconfig, or authorization header in Git, a prompt, a status file, command arguments, or retained evidence. + +## Native API surface + +The current public API base is `https://api.akua.dev/v1`. +When using REST, keep a complete HTTP authorization header in a mode-`0400` file and pass it with curl's native `-H @file` support. +Write secret-bearing JSON through protected files or stdin, never shell arguments. + +Use the Platform MCP `search` tool again before a mutation so current schemas outrank this routing table. + +| Outcome | Native endpoint | +| --- | --- | +| Store the approved Hetzner token | `POST /secrets`, kind `cloud_provider/hcloud` | +| Validate the provider token | `POST /secrets/validate_token` | +| Create managed KaaS | `POST /clusters` | +| Wait for cluster creation | `POST /operations/{id}:wait?timeout=60` | +| Create Hetzner compute configuration | `POST /compute/configs` | +| Inspect available capacity and price | `GET /compute/instance_types?config={config}` | +| Create a worker | `POST /compute/machines` | +| Inspect worker state | `GET /compute/machines/{providerId}` | +| Retrieve the approved owner kubeconfig | `GET /clusters/{id}/kubeconfig` | +| Create the clustered-Firstmate token | `POST /api_tokens` | +| Inventory tokens | `GET /api_tokens` | +| Revoke the local bootstrap token | `DELETE /api_tokens/{id}` | + +## Bootstrap procedure + +1. Record the approved workspace ID, intended region, machine constraints, expiration or cleanup condition, and evidence directory without recording secrets. +2. Read the live workspace, Secret, cluster, compute configuration, machine, and token inventories before creating anything. +3. Stop if the workspace or Hetzner project contains production resources or if an existing matching resource makes intent ambiguous. +4. Validate the provider token without storing it, then create or select the `cloud_provider/hcloud` Secret and require server validation state `valid` before creating compute. +5. Create the managed cluster with an idempotency key, preserve its operation ID, and wait for terminal `SUCCEEDED` state. +6. Create the compute configuration from the validated Secret, inspect available instance types, and select the smallest type satisfying the approved CPU, RAM, disk, architecture, price, and region constraints. +7. Create the worker, then verify both Akua machine state and Kubernetes Node readiness. +8. Retrieve kubeconfig only into a protected temporary file and verify the cluster identity before applying anything. +9. Build or select a published Agent OS image by immutable digest. +10. Create a distinct clustered-Firstmate API token and immediately convert it into a Kubernetes Secret named by `akuaAuthSecret`. + The Secret's `authorization` key contains the complete header consumed through `AKUA_AUTH_HEADER_FILE`. +11. Render `tools/agent-os/packages/firstmate/package.k` with `akua render`, inspect the ordinary YAML, and apply it with `kubectl`. +12. Verify the StatefulSet is ready, Herdr responds, the persistent home is writable, the `in-cluster` context is cluster-admin only in this intelligence cluster, and the Pod can list its Akua workspace through `curl -H @"$AKUA_AUTH_HEADER_FILE"`. +13. From the clustered Firstmate, inventory all workspace token IDs and prove the new token can perform the required Akua and Kubernetes reads. +14. Only after that handoff succeeds, revoke the known local bootstrap token and prove it now receives an authentication failure. +15. Destroy every protected temporary file and record only non-secret resource IDs, operation IDs, timestamps, states, image digest, test results, costs, and interventions. + +## Recovery and cleanup + +- A Pod restart reuses the Firstmate PVC and mounted Kubernetes Secret. +- A worker replacement must preserve or reattach every PVC holding unique work before deleting the old worker. +- Do not claim cluster-loss recovery until an external encrypted backup has been restored in a clean cluster. +- Machine, cluster, Secret, token, and workspace deletion are separate destructive actions. +- Execute only the approved cleanup scope and verify retained resources afterward. + +## Completion evidence + +The bootstrap is complete only when current external state proves all of these: + +- Akua operation IDs reached `SUCCEEDED`; +- the worker is ready in Akua and Kubernetes; +- the Firstmate StatefulSet and Herdr server are ready; +- the home and installed-tool paths survive a Pod replacement; +- the clustered token works from the Pod; +- the bootstrap token is revoked and fails authentication; +- no product or production resource exists in the intelligence workspace; and +- the evidence record contains no secret values. diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index 6ecbd247c..37782cf7e 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -15,6 +15,7 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Keep every crewmate general-purpose; its brief, tools, and authority specialize it for the current task. - A crewmate normally communicates only with its parent through its terminal, status files, reports, and delivered Git state. - The image includes Akua's CLI and the prepared mate package lives at `tools/agent-os/packages/mate/`. +- The optional persistent controller package lives at `tools/agent-os/packages/firstmate/`; load `akua-intelligence-bootstrap` before using it against Akua-managed infrastructure. - Use the bundled K9s terminal UI when it makes live Kubernetes inspection faster than individual `kubectl` reads. - Render the package with `akua render`, inspect or edit its ordinary YAML when useful, and apply it with `kubectl`. - Treat AI credentials as explicit per-mate grants, never as ambient inheritance merely because Firstmate can read them. diff --git a/.github/workflows/agent-os-image.yml b/.github/workflows/agent-os-image.yml new file mode 100644 index 000000000..a529a5864 --- /dev/null +++ b/.github/workflows/agent-os-image.yml @@ -0,0 +1,61 @@ +name: Agent OS image + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + branches: [main] + +permissions: + contents: read + packages: write + +concurrency: + group: agent-os-image-${{ github.ref }} + cancel-in-progress: true + +env: + IMAGE: ghcr.io/akua-dev/agent-os + +jobs: + build: + name: Build linux/amd64 and linux/arm64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: docker/setup-qemu-action@v3 + + - uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image tags + id: metadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE }} + tags: | + type=sha,format=long + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=tag + + - name: Build and optionally publish + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: mode=max + sbom: true diff --git a/AGENTS.md b/AGENTS.md index c231466ce..c493feb10 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -765,6 +765,7 @@ These skills are not captain-invocable; they are conditional operating reference - `firstmate-codexapp` - load before coordinating a visible Codex Desktop thread, evaluating a Codex App backend request, or reconciling Codex Desktop host-tool smoke evidence for Firstmate work. - `firstmate-coding-guidelines` - load before changing firstmate's shared, tracked material, as defined by section 1's list, whether editing directly or briefing a crewmate for a firstmate-repo task. - `kubernetes-fleet` - load before creating, supervising, recovering, or deleting Kubernetes-backed crewmates. +- `akua-intelligence-bootstrap` - load before provisioning, recovering, or handing off a Firstmate in an Akua-managed intelligence cluster. ## 14. X mode diff --git a/README.md b/README.md index 83881cd91..64b92bd9c 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,10 @@ Launching a supported harness inside it instantiates your first mate - and makes This Agent OS fork also packages the distro for Kubernetes: one persistent first mate can allocate isolated, persistent crewmate containers using ordinary `kubectl`, while Herdr keeps every agent terminal visible. Start locally with -the [OrbStack Kubernetes demo](docs/kubernetes.md). +the [OrbStack Kubernetes demo](docs/kubernetes.md), then use the optional +[Firstmate Akua package](tools/agent-os/packages/firstmate/README.md) and the +[focused bootstrap guidance](.agents/skills/akua-intelligence-bootstrap/SKILL.md) +for a dedicated Akua-managed intelligence cluster. ## Features @@ -188,6 +191,7 @@ Firstmate's skills live in two separate places with different audiences: ## Documentation - [docs/kubernetes.md](docs/kubernetes.md) - run the Agent OS controller and isolated crewmates on local OrbStack Kubernetes. +- [docs/acceptance.md](docs/acceptance.md) - requirement-to-evidence ledger for the full distributed Agent OS proof. - [docs/architecture.md](docs/architecture.md) - how the crew, supervision, worktrees, secondmates, and project modes work. - [docs/configuration.md](docs/configuration.md) - environment variables, `FM_HOME`, runtime backend selection, optional X mode, the files you set, and harness support. - [docs/wedge-alarm.md](docs/wedge-alarm.md) - configure the active alert for a wedged away-mode escalation delivery. diff --git a/docs/acceptance.md b/docs/acceptance.md new file mode 100644 index 000000000..7d7463890 --- /dev/null +++ b/docs/acceptance.md @@ -0,0 +1,26 @@ +# Agent OS acceptance ledger + +This ledger maps the Cortex distributed-Firstmate acceptance contract to current claim-matched evidence. +Update it from real runs; planned behavior and model narration never count as proof. + +| Requirement | Current evidence | State | +| --- | --- | --- | +| Persistent Firstmate and Herdr in Kubernetes | OrbStack demo plus generic package render and disposable-cluster run in `docs/evidence/2026-07-12-firstmate-package.md` | Proven locally | +| Firstmate cluster-admin limited to the intelligence cluster | Dedicated local namespace and explicit demo ClusterRoleBinding | Proven locally only | +| Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | +| Distinct clustered token and bootstrap-token revocation | Explicit Secret mount in the Firstmate package and handoff procedure | Not yet run | +| Firstmate-native Akua worker lifecycle | Native endpoint routing in the bootstrap skill | Not yet run | +| Same-Pod general-purpose crewmate | Existing Firstmate Herdr backend | Not yet captured as acceptance evidence | +| Separate persistent crewmate Pod with explicit authority | Mate package, no ServiceAccount token by default, explicit Pi Secret grant | Package proven; full task run incomplete | +| Parent-only supervision and human attach | Firstmate contract, Herdr CLI, live workspaces | Mechanism proven; distributed task trace incomplete | +| Real issue to tested review-ready PR | Existing Firstmate delivery machinery | Not yet proven through the Kubernetes fleet | +| Parent and child restart without unique-work loss | Parent PVC restart proven | Child unique-work recovery not yet proven | +| Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | +| Optional GitHub Issue/Project client | Cortex contract only | Deferred until terminal-native path passes | +| Repeatable end-to-end eval and recordable demo | Deterministic unit checks and local smoke evidence | Full critical-path run not yet captured | +| Public installable image and release | Multi-architecture GHCR workflow added; local image only | Publication not yet proven | + +## Definition of done + +Agent OS is finished only when every row is backed by current external evidence and no row remains partial or unproven. +The complete run must record time-to-cluster, time-to-Firstmate, time-to-crewmate, time-to-PR, recovery time, model and infrastructure cost, human interventions, immutable image digest, resource and operation IDs, and sanitized failure evidence. diff --git a/docs/evidence/2026-07-12-firstmate-package.md b/docs/evidence/2026-07-12-firstmate-package.md new file mode 100644 index 000000000..aa49fc105 --- /dev/null +++ b/docs/evidence/2026-07-12-firstmate-package.md @@ -0,0 +1,70 @@ +# Firstmate package verification + +Date: 2026-07-12 +Akua CLI: 0.8.20 +Kubernetes: OrbStack 1.34.8 +Agent OS image: `sha256:f87ae0f5f93d2781700a7c3a0828833433565322b7bb3e8f1d49244b40b8580e` + +## Claim + +The optional Firstmate Akua package renders ordinary resources and can start a persistent, cluster-admin Firstmate with a read-only Akua authorization mount in a dedicated Kubernetes namespace. + +## Render evidence + +Command: + +```sh +akua render --no-json --no-interactive \ + --package tools/agent-os/packages/firstmate/package.k \ + --inputs tools/agent-os/packages/firstmate/inputs.example.yaml \ + --out /tmp/agent-os-firstmate-render +``` + +Result: + +```text +rendered: 6 manifest(s) (sha256:4965345d355a00530a887f8f24454d5e858664c7fa4328ba584c15f96703ca0d) +Namespace +ServiceAccount +PersistentVolumeClaim +Service +ClusterRoleBinding +StatefulSet +``` + +## Disposable-cluster evidence + +The live eval used namespace `agent-os-package-eval`, the local immutable image above, `imagePullPolicy: Never`, and a synthetic authorization Secret. +It rendered with `createNamespace: false`, applied five manifests, waited for the StatefulSet, and ran the checks inside the Pod. + +Commands: + +```sh +kubectl --context orbstack apply -f /tmp/agent-os-firstmate-eval +kubectl --context orbstack -n agent-os-package-eval \ + rollout status statefulset/agent-os-firstmate --timeout=120s +kubectl --context orbstack -n agent-os-package-eval \ + exec statefulset/agent-os-firstmate -- herdr status --json +kubectl --context orbstack -n agent-os-package-eval \ + exec statefulset/agent-os-firstmate -- kubectl auth can-i '*' '*' --all-namespaces +``` + +Result: + +```text +rendered: 5 manifest(s) (sha256:ecf9cfb0052ca90677f14987ff823637aa27d3e914be9b7ab66906b55e99acc4) +partitioned roll out complete: 1 new pods have been updated +herdr=ready +home=writable +auth_mount=readonly +context=in-cluster +cluster_admin=yes +``` + +The write probe against `/var/run/secrets/agent-os/akua/authorization` failed with `Read-only file system`. +The eval namespace and its cluster-scoped binding were deleted after the checks. + +## Scope + +This proves the Kubernetes package contract locally. +It does not prove Akua workspace authentication, managed KaaS creation, Hetzner worker lifecycle, successor-token handoff, public-image availability, or bootstrap-token revocation. diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index be15dd2d1..53c260ac7 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -62,6 +62,12 @@ assert_grep 'https://github.com/akua-dev/akua/tree/v0.8.25' "$ROOT/THIRD_PARTY_N "Akua's exact source must be named" assert_grep 'https://github.com/derailed/k9s/tree/v0.51.0' "$ROOT/THIRD_PARTY_NOTICES.md" \ "K9s's exact source must be named" +assert_grep 'ghcr.io/akua-dev/agent-os' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release workflow must publish the image expected by the Akua packages" +assert_grep 'linux/amd64,linux/arm64' "$ROOT/.github/workflows/agent-os-image.yml" \ + "release workflow must build the two supported container architectures" +assert_grep "push: \${{ github.event_name != 'pull_request' }}" "$ROOT/.github/workflows/agent-os-image.yml" \ + "pull requests must build but never publish images" bash -n "$ROOT/bin/agent-os-container-entrypoint.sh" bash -n "$ROOT/bin/agent-os-kubeconfig.sh" diff --git a/tests/agent-os-packages.test.sh b/tests/agent-os-packages.test.sh new file mode 100755 index 000000000..0b82902bc --- /dev/null +++ b/tests/agent-os-packages.test.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Static contracts for the optional Akua packages used by Agent OS. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +FIRSTMATE="$ROOT/tools/agent-os/packages/firstmate" + +[ -f "$FIRSTMATE/package.k" ] || fail "firstmate package must exist" +[ -f "$FIRSTMATE/inputs.example.yaml" ] || fail "firstmate example inputs must exist" + +assert_grep 'ghcr.io/akua-dev/agent-os:latest' "$FIRSTMATE/package.k" \ + "firstmate package must default to the public Agent OS image" +assert_grep 'kind = "StatefulSet"' "$FIRSTMATE/package.k" \ + "firstmate package must create a restartable controller" +assert_grep 'kind = "PersistentVolumeClaim"' "$FIRSTMATE/package.k" \ + "firstmate package must persist its home" +assert_grep 'kind = "ServiceAccount"' "$FIRSTMATE/package.k" \ + "firstmate package must create its cluster identity" +assert_grep 'name = "cluster-admin"' "$FIRSTMATE/package.k" \ + "dedicated intelligence clusters must support the explicit Firstmate admin grant" +assert_grep 'akuaAuthSecret: str = ""' "$FIRSTMATE/package.k" \ + "Akua workspace access must be an explicit Secret reference" +assert_grep 'mountPath = "/var/run/secrets/agent-os/akua"' "$FIRSTMATE/package.k" \ + "Akua authorization material must use a runtime Secret mount" +assert_grep 'readOnly = True' "$FIRSTMATE/package.k" \ + "Akua authorization material must be mounted read-only" +assert_grep 'herdr", "status", "--json"' "$FIRSTMATE/package.k" \ + "Firstmate readiness must prove the Herdr server is responsive" +assert_grep 'mountPath = "/usr/local"' "$FIRSTMATE/package.k" \ + "Firstmate-installed tools must persist" +assert_no_grep 'token:' "$FIRSTMATE/inputs.example.yaml" \ + "example inputs must never contain an Akua token value" + +pass "Akua packages keep Firstmate persistent and authority explicit" diff --git a/tools/agent-os/packages/firstmate/README.md b/tools/agent-os/packages/firstmate/README.md new file mode 100644 index 000000000..59245da08 --- /dev/null +++ b/tools/agent-os/packages/firstmate/README.md @@ -0,0 +1,21 @@ +# Firstmate package + +This optional Akua package renders ordinary Kubernetes resources for one persistent Firstmate in a dedicated intelligence cluster. +It creates a namespace, ServiceAccount, persistent home, headless Service, StatefulSet, and - when explicitly enabled - the intelligence-cluster `cluster-admin` binding. + +Render it directly with Akua: + +```sh +akua render \ + --package package.k \ + --inputs inputs.example.yaml \ + --out /tmp/agent-os-firstmate +``` + +Inspect the YAML before applying it with `kubectl`. +The package is an editable convenience, not a controller or required workflow. +Use an immutable image digest for a real intelligence cluster; the `latest` default is only a discoverable starting point. + +`akuaAuthSecret` names an existing Kubernetes Secret with an `authorization` key containing a complete HTTP authorization header for the dedicated Agent OS workspace token. +The value is mounted read-only at runtime and never belongs in inputs, rendered YAML, Git, logs, or command arguments. +Leave the input empty when Firstmate should not receive Akua workspace access. diff --git a/tools/agent-os/packages/firstmate/akua.toml b/tools/agent-os/packages/firstmate/akua.toml new file mode 100644 index 000000000..245f6a593 --- /dev/null +++ b/tools/agent-os/packages/firstmate/akua.toml @@ -0,0 +1,5 @@ +[package] +name = "agent-os-firstmate" +version = "0.1.0" +edition = "akua.dev/v1alpha1" +strict_signing = true diff --git a/tools/agent-os/packages/firstmate/inputs.example.yaml b/tools/agent-os/packages/firstmate/inputs.example.yaml new file mode 100644 index 000000000..30b57eb03 --- /dev/null +++ b/tools/agent-os/packages/firstmate/inputs.example.yaml @@ -0,0 +1,5 @@ +namespace: agent-os +image: ghcr.io/akua-dev/agent-os:latest +imagePullPolicy: IfNotPresent +clusterAdmin: true +akuaAuthSecret: agent-os-firstmate-akua-auth diff --git a/tools/agent-os/packages/firstmate/package.k b/tools/agent-os/packages/firstmate/package.k new file mode 100644 index 000000000..ff2f6f233 --- /dev/null +++ b/tools/agent-os/packages/firstmate/package.k @@ -0,0 +1,182 @@ +import akua.ctx + +schema Input: + """Inputs for one persistent Firstmate in a dedicated intelligence cluster.""" + + namespace: str = "agent-os" + createNamespace: bool = True + image: str = "ghcr.io/akua-dev/agent-os:latest" + imagePullPolicy: "Always" | "IfNotPresent" | "Never" = "IfNotPresent" + clusterAdmin: bool = True + akuaAuthSecret: str = "" + storage: str = "20Gi" + cpuRequest: str = "500m" + memoryRequest: str = "1Gi" + cpuLimit: str = "4" + memoryLimit: str = "8Gi" + +input: Input = ctx.input() + +labels = { + "app.kubernetes.io/name" = "agent-os" + "app.kubernetes.io/component" = "firstmate" +} + +akuaAuthMounts = [{ + name = "akua-auth" + mountPath = "/var/run/secrets/agent-os/akua" + readOnly = True +}] if input.akuaAuthSecret else [] + +akuaAuthVolumes = [{ + name = "akua-auth" + secret = { + secretName = input.akuaAuthSecret + defaultMode = 256 + } +}] if input.akuaAuthSecret else [] + +akuaAuthEnv = [{ + name = "AKUA_AUTH_HEADER_FILE" + value = "/var/run/secrets/agent-os/akua/authorization" +}] if input.akuaAuthSecret else [] + +namespaceResources = [{ + apiVersion = "v1" + kind = "Namespace" + metadata = { + name = input.namespace + labels = { "app.kubernetes.io/part-of" = "agent-os" } + } +}] if input.createNamespace else [] + +adminResources = [{ + apiVersion = "rbac.authorization.k8s.io/v1" + kind = "ClusterRoleBinding" + metadata = { + name = "agent-os-firstmate-${input.namespace}" + labels = labels + } + roleRef = { + apiGroup = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cluster-admin" + } + subjects = [{ + kind = "ServiceAccount" + name = "agent-os-firstmate" + namespace = input.namespace + }] +}] if input.clusterAdmin else [] + +resources = namespaceResources + [ + { + apiVersion = "v1" + kind = "ServiceAccount" + metadata = { + name = "agent-os-firstmate" + namespace = input.namespace + labels = labels + } + }, + { + apiVersion = "v1" + kind = "PersistentVolumeClaim" + metadata = { + name = "agent-os-firstmate-home" + namespace = input.namespace + labels = labels + } + spec = { + accessModes = ["ReadWriteOnce"] + resources = { requests = { storage = input.storage } } + } + }, + { + apiVersion = "v1" + kind = "Service" + metadata = { + name = "agent-os-firstmate" + namespace = input.namespace + labels = labels + } + spec = { + clusterIP = "None" + selector = labels + } + }, +] + adminResources + [ + { + apiVersion = "apps/v1" + kind = "StatefulSet" + metadata = { + name = "agent-os-firstmate" + namespace = input.namespace + labels = labels + } + spec = { + serviceName = "agent-os-firstmate" + replicas = 1 + selector = { matchLabels = labels } + template = { + metadata = { + labels = labels + annotations = { + "agent-os.akua.dev/address" = "/k8s/in-cluster/namespace/${input.namespace}/mate/firstmate/herdr" + } + } + spec = { + serviceAccountName = "agent-os-firstmate" + securityContext = { + fsGroup = 0 + runAsGroup = 0 + runAsUser = 0 + } + initContainers = [{ + name = "agent-os-init" + image = input.image + imagePullPolicy = input.imagePullPolicy + command = ["/opt/agent-os/bin/agent-os-init.sh"] + volumeMounts = [{ + name = "home" + mountPath = "/persistent-agent" + }] + }] + containers = [{ + name = "firstmate" + image = input.image + imagePullPolicy = input.imagePullPolicy + env = [ + { name = "FM_HOME", value = "/home/agent" }, + { name = "HOME", value = "/home/agent" }, + { name = "AKUA_API_URL", value = "https://api.akua.dev/v1" }, + ] + akuaAuthEnv + readinessProbe = { + exec = { command = ["herdr", "status", "--json"] } + initialDelaySeconds = 2 + periodSeconds = 5 + } + resources = { + requests = { + cpu = input.cpuRequest + memory = input.memoryRequest + } + limits = { + cpu = input.cpuLimit + memory = input.memoryLimit + } + } + volumeMounts = [ + { name = "home", mountPath = "/home/agent" }, + { name = "home", mountPath = "/usr/local", subPath = "usr-local" }, + ] + akuaAuthMounts + }] + volumes = [{ + name = "home" + persistentVolumeClaim = { claimName = "agent-os-firstmate-home" } + }] + akuaAuthVolumes + } + } + } + }, +] From aa94fa982c900ca90f76c57c1d1169d32dfeb1c9 Mon Sep 17 00:00:00 2001 From: Robin Date: Sun, 12 Jul 2026 12:06:29 +0200 Subject: [PATCH 14/56] fix: probe model capacity and portable evidence --- .agents/skills/akua-intelligence-bootstrap/SKILL.md | 9 ++++++--- .agents/skills/kubernetes-fleet/SKILL.md | 1 + docs/acceptance.md | 1 + tests/agent-os-kubeconfig.test.sh | 6 +++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.agents/skills/akua-intelligence-bootstrap/SKILL.md b/.agents/skills/akua-intelligence-bootstrap/SKILL.md index d32a28491..8da717917 100644 --- a/.agents/skills/akua-intelligence-bootstrap/SKILL.md +++ b/.agents/skills/akua-intelligence-bootstrap/SKILL.md @@ -58,9 +58,11 @@ Use the Platform MCP `search` tool again before a mutation so current schemas ou The Secret's `authorization` key contains the complete header consumed through `AKUA_AUTH_HEADER_FILE`. 11. Render `tools/agent-os/packages/firstmate/package.k` with `akua render`, inspect the ordinary YAML, and apply it with `kubectl`. 12. Verify the StatefulSet is ready, Herdr responds, the persistent home is writable, the `in-cluster` context is cluster-admin only in this intelligence cluster, and the Pod can list its Akua workspace through `curl -H @"$AKUA_AUTH_HEADER_FILE"`. -13. From the clustered Firstmate, inventory all workspace token IDs and prove the new token can perform the required Akua and Kubernetes reads. -14. Only after that handoff succeeds, revoke the known local bootstrap token and prove it now receives an authentication failure. -15. Destroy every protected temporary file and record only non-secret resource IDs, operation IDs, timestamps, states, image digest, test results, costs, and interventions. +13. Probe the approved primary model route with a bounded request and record only provider, model, result, timing, and cost. + If there is no independently approved fallback provider, record the single-provider availability risk instead of presenting the fleet as quota-resilient. +14. From the clustered Firstmate, inventory all workspace token IDs and prove the new token can perform the required Akua and Kubernetes reads. +15. Only after that handoff succeeds, revoke the known local bootstrap token and prove it now receives an authentication failure. +16. Destroy every protected temporary file and record only non-secret resource IDs, operation IDs, timestamps, states, image digest, test results, costs, and interventions. ## Recovery and cleanup @@ -77,6 +79,7 @@ The bootstrap is complete only when current external state proves all of these: - Akua operation IDs reached `SUCCEEDED`; - the worker is ready in Akua and Kubernetes; - the Firstmate StatefulSet and Herdr server are ready; +- at least one approved model route completes a bounded request; - the home and installed-tool paths survive a Pod replacement; - the clustered token works from the Pod; - the bootstrap token is revoked and fails authentication; diff --git a/.agents/skills/kubernetes-fleet/SKILL.md b/.agents/skills/kubernetes-fleet/SKILL.md index 37782cf7e..cd4529f21 100644 --- a/.agents/skills/kubernetes-fleet/SKILL.md +++ b/.agents/skills/kubernetes-fleet/SKILL.md @@ -20,6 +20,7 @@ Load this skill only when the current firstmate is running in Kubernetes or is e - Render the package with `akua render`, inspect or edit its ordinary YAML when useful, and apply it with `kubectl`. - Treat AI credentials as explicit per-mate grants, never as ambient inheritance merely because Firstmate can read them. - Create or select a Kubernetes Secret only after the grant is authorized, then pass its name as the package's `piAuthSecret` input. +- Probe the selected model route before launching work; when quota is unavailable, use another explicitly granted provider or report the capacity blocker instead of repeatedly spawning agents. - Give every launched Herdr agent a task-unique name, close only a confirmed dead restored pane before reuse, and never replace a live agent. - Grade completion by the promised artifact or delivered Git state; Herdr `idle` alone is not a completion signal. - The package is optional; direct `akua render`, raw YAML, and `kubectl` remain supported. diff --git a/docs/acceptance.md b/docs/acceptance.md index 7d7463890..90a7ab1ce 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -10,6 +10,7 @@ Update it from real runs; planned behavior and model narration never count as pr | Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | | Distinct clustered token and bootstrap-token revocation | Explicit Secret mount in the Firstmate package and handoff procedure | Not yet run | | Firstmate-native Akua worker lifecycle | Native endpoint routing in the bootstrap skill | Not yet run | +| Replaceable usable model supply | Pi auth is explicit, but the live demo has one Codex provider and current quota exhaustion | Blocked until an approved route has capacity | | Same-Pod general-purpose crewmate | Existing Firstmate Herdr backend | Not yet captured as acceptance evidence | | Separate persistent crewmate Pod with explicit authority | Mate package, no ServiceAccount token by default, explicit Pi Secret grant | Package proven; full task run incomplete | | Parent-only supervision and human attach | Firstmate contract, Herdr CLI, live workspaces | Mechanism proven; distributed task trace incomplete | diff --git a/tests/agent-os-kubeconfig.test.sh b/tests/agent-os-kubeconfig.test.sh index 80aa70097..a06d76f07 100755 --- a/tests/agent-os-kubeconfig.test.sh +++ b/tests/agent-os-kubeconfig.test.sh @@ -24,7 +24,11 @@ assert_grep 'current-context: in-cluster' "$config" "kubeconfig must select the assert_grep 'namespace: agent-os-eval' "$config" "kubeconfig must select the Agent OS namespace" assert_grep "tokenFile: $tmp/serviceaccount/token" "$config" "kubeconfig must follow the rotating token file" assert_no_grep 'demo-token' "$config" "kubeconfig must not copy token contents" -[ "$(stat -f '%Lp' "$config" 2>/dev/null || stat -c '%a' "$config")" = 600 ] || fail "kubeconfig must be mode 600" +case $(uname -s) in + Darwin) mode=$(stat -f '%Lp' "$config") ;; + *) mode=$(stat -c '%a' "$config") ;; +esac +[ "$mode" = 600 ] || fail "kubeconfig must be mode 600" printf '%s\n' sentinel > "$config" HOME="$tmp/home" \ From 82db241b137bc97af2e863054c91e26a62834195 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 10:48:55 +0200 Subject: [PATCH 15/56] Prove persistent Kubernetes mate recovery --- .agents/skills/harness-adapters/SKILL.md | 2 + AGENTS.md | 1 + bin/agent-os-container-entrypoint.sh | 64 ++++++++++ deploy/orbstack/primary.yaml | 4 + docs/acceptance.md | 14 +-- .../evidence/2026-07-13-same-pod-firstmate.md | 97 ++++++++++++++ .../2026-07-13-separate-pod-recovery.md | 119 ++++++++++++++++++ docs/kubernetes.md | 11 +- tests/agent-os-container.test.sh | 12 ++ 9 files changed, 316 insertions(+), 8 deletions(-) create mode 100644 docs/evidence/2026-07-13-same-pod-firstmate.md create mode 100644 docs/evidence/2026-07-13-separate-pod-recovery.md diff --git a/.agents/skills/harness-adapters/SKILL.md b/.agents/skills/harness-adapters/SKILL.md index 67bffd886..704f457d3 100644 --- a/.agents/skills/harness-adapters/SKILL.md +++ b/.agents/skills/harness-adapters/SKILL.md @@ -87,6 +87,8 @@ The supported launch-profile flags below were verified locally on 2026-06-30 wit | pi | `--model ` | `--thinking ` | Verified on pi 0.80.2. `max` prints an invalid-thinking warning, so firstmate omits Pi effort when the requested effort is `max`. | | opencode | `--model ` | none for firstmate's interactive launch | Verified on opencode 1.17.6. `opencode run` has `--variant`, but firstmate launches the interactive `opencode --prompt` path, which has no verified effort flag. | +When a Pi provider route is known, pass a provider-qualified model id such as `openai-codex/gpt-5.6-terra`; unqualified names may resolve to an unauthenticated provider. + When a requested effort value is outside the harness-specific accepted set, `fm-spawn` records the requested `effort=` in meta but emits no effort flag for that harness. This preserves launch success instead of passing a known-bad value. diff --git a/AGENTS.md b/AGENTS.md index c493feb10..e0dd21f18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,7 @@ Never add an agent name as co-author. `FM_HOME` selects the operational home for a firstmate instance. When it is unset, most scripts use this repo root as the home, which is today's behavior. When it is set, scripts still use their own `bin/` from the repo they live in, but operational dirs come from `$FM_HOME`: `state/`, `data/`, `config/`, and `projects/`. +When `FM_HOME` is set, never create or read operational state through repo-relative `state/`, `data/`, `config/`, or `projects/` paths; use `$FM_HOME/...` or firstmate scripts that resolve it. Existing overrides remain compatible: `FM_STATE_OVERRIDE` can still point at a custom state dir, and `FM_ROOT_OVERRIDE` still behaves like the old whole-root override when `FM_HOME` is unset. `bin/fm-send.sh` is the fail-closed exception: it requires `FM_HOME` to be set so target resolution is always scoped to an explicit firstmate home. Each secondmate gets its own persistent `FM_HOME`, so its local state, backlog, projects, and session lock are isolated from the main firstmate. diff --git a/bin/agent-os-container-entrypoint.sh b/bin/agent-os-container-entrypoint.sh index a94ddaaf6..a8958c471 100755 --- a/bin/agent-os-container-entrypoint.sh +++ b/bin/agent-os-container-entrypoint.sh @@ -22,6 +22,70 @@ if [ ! -e "$FM_HOME/config/backend" ]; then printf 'herdr\n' > "$FM_HOME/config/backend" fi +# The local evaluation overlay may pin every spawned Pi agent while a model is +# under test. This is deliberately opt-in so the reusable image and Akua +# packages remain model-agnostic. Converge both dispatch files on every Pod +# start: crew-dispatch governs crewmates/scouts, while secondmate-harness is the +# separate profile used to launch Secondmates. +if [ -n "${AGENT_OS_TEST_PI_MODEL:-}" ]; then + case "$AGENT_OS_TEST_PI_MODEL" in + */*) ;; + *) + echo "error: AGENT_OS_TEST_PI_MODEL must include its provider" >&2 + exit 2 + ;; + esac + case "$AGENT_OS_TEST_PI_MODEL" in + *[!A-Za-z0-9._:/-]*) + echo "error: AGENT_OS_TEST_PI_MODEL must be one provider-qualified token" >&2 + exit 2 + ;; + esac + case "${AGENT_OS_TEST_PI_EFFORT:-}" in + low|medium|high|xhigh) ;; + *) + echo "error: AGENT_OS_TEST_PI_EFFORT must be low, medium, high, or xhigh" >&2 + exit 2 + ;; + esac + cat > "$FM_HOME/config/crew-dispatch.json" < "$FM_HOME/config/secondmate-harness" + + pi_provider=${AGENT_OS_TEST_PI_MODEL%%/*} + pi_model=${AGENT_OS_TEST_PI_MODEL#*/} + pi_settings_dir="$HOME/.pi/agent" + pi_settings="$pi_settings_dir/settings.json" + pi_settings_tmp="$pi_settings.tmp.$$" + mkdir -p "$pi_settings_dir" + umask 077 + if [ -s "$pi_settings" ] && jq -e 'type == "object"' "$pi_settings" >/dev/null 2>&1; then + jq \ + --arg provider "$pi_provider" \ + --arg model "$pi_model" \ + --arg thinking "$AGENT_OS_TEST_PI_EFFORT" \ + '. + {defaultProvider: $provider, defaultModel: $model, defaultThinkingLevel: $thinking}' \ + "$pi_settings" > "$pi_settings_tmp" + else + jq -n \ + --arg provider "$pi_provider" \ + --arg model "$pi_model" \ + --arg thinking "$AGENT_OS_TEST_PI_EFFORT" \ + '{defaultProvider: $provider, defaultModel: $model, defaultThinkingLevel: $thinking}' \ + > "$pi_settings_tmp" + fi + mv "$pi_settings_tmp" "$pi_settings" +fi + for tool in gh-axi chrome-devtools-axi lavish-axi; do marker="$HOME/.config/agent-os/setup-$tool" if [ ! -e "$marker" ]; then diff --git a/deploy/orbstack/primary.yaml b/deploy/orbstack/primary.yaml index 1a1717d70..4debe6814 100644 --- a/deploy/orbstack/primary.yaml +++ b/deploy/orbstack/primary.yaml @@ -57,6 +57,10 @@ spec: value: /home/agent - name: HOME value: /home/agent + - name: AGENT_OS_TEST_PI_MODEL + value: openai-codex/gpt-5.6-terra + - name: AGENT_OS_TEST_PI_EFFORT + value: low readinessProbe: exec: command: diff --git a/docs/acceptance.md b/docs/acceptance.md index 90a7ab1ce..b69ba6a5f 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -10,16 +10,16 @@ Update it from real runs; planned behavior and model narration never count as pr | Direct Akua-managed KaaS and Hetzner bootstrap | Public endpoint study and `akua-intelligence-bootstrap` skill | Not yet run | | Distinct clustered token and bootstrap-token revocation | Explicit Secret mount in the Firstmate package and handoff procedure | Not yet run | | Firstmate-native Akua worker lifecycle | Native endpoint routing in the bootstrap skill | Not yet run | -| Replaceable usable model supply | Pi auth is explicit, but the live demo has one Codex provider and current quota exhaustion | Blocked until an approved route has capacity | -| Same-Pod general-purpose crewmate | Existing Firstmate Herdr backend | Not yet captured as acceptance evidence | -| Separate persistent crewmate Pod with explicit authority | Mate package, no ServiceAccount token by default, explicit Pi Secret grant | Package proven; full task run incomplete | -| Parent-only supervision and human attach | Firstmate contract, Herdr CLI, live workspaces | Mechanism proven; distributed task trace incomplete | +| Replaceable usable model supply | `openai-codex/gpt-5.4-mini` completed the same-Pod task; Terra-low completed the separate-Pod and recovery tasks; the current local overlay converges direct Pi, crewmate, and Secondmate defaults to `openai-codex/gpt-5.6-terra` with low thinking | One provider with multiple models proven locally; replacement provider still unproven | +| Same-Pod general-purpose crewmate | Firstmate-supervised Pi scout, real Kubernetes test, in-cluster context check, and report in `docs/evidence/2026-07-13-same-pod-firstmate.md` | Proven locally | +| Separate persistent crewmate Pod with explicit authority | Terra-low task, explicit resources, read-only Pi Secret, no ServiceAccount token, and retained PVC in `docs/evidence/2026-07-13-separate-pod-recovery.md` | Proven locally | +| Parent-only supervision and human attach | Same-Pod and separate-Pod tasks accepted only Firstmate briefs and steers; a connected Herdr client observed the live workspaces | Proven locally | | Real issue to tested review-ready PR | Existing Firstmate delivery machinery | Not yet proven through the Kubernetes fleet | -| Parent and child restart without unique-work loss | Parent PVC restart proven | Child unique-work recovery not yet proven | +| Parent and child restart without unique-work loss | Primary session recovery plus separate-Pod replacement with unique artifact, report, tools, and exact Pi session resumed | Proven locally | | Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | | Optional GitHub Issue/Project client | Cortex contract only | Deferred until terminal-native path passes | -| Repeatable end-to-end eval and recordable demo | Deterministic unit checks and local smoke evidence | Full critical-path run not yet captured | -| Public installable image and release | Multi-architecture GHCR workflow added; local image only | Publication not yet proven | +| Repeatable end-to-end eval and recordable demo | Same-Pod and separate-Pod run records now capture model, time, cost estimate, interventions, resource IDs, and failures | Local agent lifecycle captured; Akua/GitHub/product path incomplete | +| Public installable image and release | Multi-architecture workflow run `29188585384` succeeded; local image only because pull-request builds do not publish | Publication not yet proven | ## Definition of done diff --git a/docs/evidence/2026-07-13-same-pod-firstmate.md b/docs/evidence/2026-07-13-same-pod-firstmate.md new file mode 100644 index 000000000..45a063e80 --- /dev/null +++ b/docs/evidence/2026-07-13-same-pod-firstmate.md @@ -0,0 +1,97 @@ +# Same-Pod Firstmate verification + +Date: 2026-07-13 +Kubernetes context: `orbstack` +Namespace: `agent-os-demo` +Pod: `agent-os-firstmate-0` +Image ID: `docker-pullable://agent-os@sha256:1e6696938aafcec21748b83e3d68773d8b3cf1ccb967974f2cfbeda0c4fbed65` +Herdr: 0.7.3, protocol 16 +Pi: 0.80.6 + +## Claim + +A persistent Firstmate running in Herdr inside the Kubernetes Pod can create, supervise, recover, and finish a general-purpose Pi crewmate in another Herdr task pane in the same Pod. +The crewmate receives work only from Firstmate, runs real verification commands, writes its report to the persistent Firstmate home, and remains observable through Herdr. + +## Topology + +- Primary Firstmate: Herdr workspace `w3`, pane `w3:p1`, persistent `FM_HOME=/home/agent`. +- Crewmate task: `aeval-p4`, Herdr endpoint `default:w2:p6`. +- Model: `openai-codex/gpt-5.4-mini`, medium thinking. +- Worktree: `/home/agent/.treehouse/agent-os-eval-f1de56/2/agent-os-eval`. +- Report: `/home/agent/data/aeval-p4/report.md`. +- Both agents ran in `agent-os-firstmate-0`; no separate task service or communication protocol was used. + +## Task evidence + +Firstmate created a minimal local-only evaluation repository, registered it, scaffolded a scout brief, and dispatched the crewmate through `fm-spawn.sh` with the Herdr backend. +The successful spawn recorded: + +```text +spawned aeval-p4 harness=pi kind=scout mode=local-only yolo=off window=default:w2:p6 worktree=/home/agent/.treehouse/agent-os-eval-f1de56/2/agent-os-eval +``` + +The crewmate ran the real kubeconfig regression test: + +```sh +bash /opt/agent-os/tests/agent-os-kubeconfig.test.sh +``` + +```text +ok - Agent OS creates a rotation-safe in-cluster kubeconfig without exposing token contents +``` + +It also checked the active Kubernetes context: + +```text +CURRENT NAME CLUSTER AUTHINFO NAMESPACE +* in-cluster in-cluster in-cluster agent-os-demo +``` + +The task status reached: + +```text +working: inspecting project and kube integration +resolved: recovered with provider-qualified openai-codex/gpt-5.4-mini +done: report written with verification and evidence +``` + +The recorded task interval was `2026-07-13T08:18:30Z` to `2026-07-13T08:20:47Z`, or 2 minutes 17 seconds. +Pi displayed an approximate crewmate-session cost of `$0.195`; this is a model UI estimate, not a provider invoice. + +## Parent-only supervision and human observation + +The child received its initial brief from `fm-spawn.sh` and later steers through Firstmate's `fm-send.sh` path. +The external evaluator made only read-only Herdr observations of the child pane and never sent a prompt or command directly to the child. +Herdr server logs recorded a connected client while the fleet was live, proving that a human could attach and observe the same workspaces. + +## Failure and recovery evidence + +The first spawn used the unqualified model name `gpt-5.4-mini`. +Pi resolved it to the unauthenticated `azure-openai-responses` provider and failed with: + +```text +Error: No API key found for azure-openai-responses. +``` + +Firstmate preserved the task brief, state, report path, and worktree allocation, then respawned the same task id with `openai-codex/gpt-5.4-mini`. +The qualified route completed successfully. + +A connected Herdr client also closed the original primary workspace during the run. +The primary Pi session, persistent Firstmate home, task metadata, brief, and status survived. +A resumed primary reacquired the fleet through `bin/fm-session-start.sh` and supervised the task to completion. +The resumed primary had to remain in its own Herdr workspace; placing it inside the task tab made ordinary dead-task replacement close the supervisor with that tab. + +Three operator interventions were required: qualify the provider route, restart the primary after its workspace closed, and move the resumed primary into a dedicated workspace. +Firstmate then handled child steering, report acceptance, and agent exit itself. + +## Lessons promoted into the system + +- Operational state must always resolve through `$FM_HOME`; repo-relative `data/` and `projects/` created non-persistent duplicates during the first attempt. +- Pi dispatch must use a provider-qualified model id when authentication depends on the provider route. +- A primary Firstmate Herdr pane must not share a task tab that the backend may replace during recovery. + +## Scope + +This proves the same-Pod task path, parent-only task communication, human observation, provider-qualified model recovery, and primary-session recovery locally. +It does not prove a separately resourced crewmate Pod, unique child work surviving a Pod restart, Akua-managed infrastructure, GitHub delivery, or production access boundaries. diff --git a/docs/evidence/2026-07-13-separate-pod-recovery.md b/docs/evidence/2026-07-13-separate-pod-recovery.md new file mode 100644 index 000000000..972d38614 --- /dev/null +++ b/docs/evidence/2026-07-13-separate-pod-recovery.md @@ -0,0 +1,119 @@ +# Separate-Pod mate and recovery verification + +Date: 2026-07-13 +Kubernetes context: `orbstack` +Namespace: `agent-os-demo` +Package: `tools/agent-os/packages/mate/package.k` +Model policy: `openai-codex/gpt-5.6-terra`, low thinking + +## Claim + +Firstmate can use ordinary Akua package rendering, Kubernetes, and Herdr to create and supervise a separately resourced persistent mate Pod with explicit AI credentials and no ambient Kubernetes authority. +The mate can complete real work, survive Pod replacement without losing its unique artifact or Pi session, resume the exact persisted session, append recovery evidence, and return control to Firstmate without a custom controller or communication protocol. + +## Resource and authority boundary + +- Pod: `agent-os-mate-separate-eval`. +- Address: `/k8s/in-cluster/ns/agent-os-demo/pod/agent-os-mate-separate-eval`. +- Requests: `500m` CPU and `1Gi` memory. +- Limits: `2` CPU and `4Gi` memory. +- PVC: `agent-os-mate-separate-eval-home`, UID `2c141e3b-0585-4648-96e8-2bc496450a91`, retained and Bound. +- Pi Secret: `agent-os-mate-separate-eval-pi-auth`, UID `c4698208-bb08-437c-87e9-9346e8b14ed2`, retained. +- `automountServiceAccountToken`: `false`. +- No ServiceAccount token existed in the container. +- `/home/agent/.pi/agent/auth.json` was an explicit read-only Secret mount. +- The package added no privileged setting, host namespace, or host mount. + +Parent-side live verification returned: + +```text +NO_SA_TOKEN +/home/agent/.pi/agent/auth.json ro,relatime +{"server":"running","running":true,"socket":"/home/agent/.config/herdr/herdr.sock","compatible":true} +``` + +The child initially misread `/proc/mounts` and claimed that no auth material was mounted. +Firstmate caught the discrepancy and retained the authoritative parent-side `/proc/self/mountinfo` evidence above. + +## Initial task + +Firstmate launched `separate-eval-terra` through the mate's Herdr server with `openai-codex/gpt-5.6-terra` and low thinking. +The child created `/home/agent/unique-work.txt`: + +```text +separate-eval-20260713T083121Z-b8a6ef90edac4d55 +Short note: child evaluator artifact; parent-only communication. +``` + +Its immutable artifact hash was: + +```text +2aeb922cd51230ed7b62bc40cc7ce30d41e44b4df3963911fe860030c2f7765a +``` + +The task ran from `2026-07-13T08:31:12Z` through `2026-07-13T08:31:35Z` and wrote `/home/agent/separate-eval-report.md`. +The child communicated only through its initial Firstmate brief and returned its result through the persistent report and Herdr pane. + +## Pod and session recovery + +Firstmate exited the child, preserved the PVC and Secret, deleted only the Pod, and recreated it from the same Akua-rendered manifests. + +```text +Initial Pod UID: 960cc460-cd06-44d1-a8fa-da200d4384b9 +Replacement Pod UID: 1cb7ff92-5e0d-4ccd-9324-38515ae9823b +``` + +After replacement, Firstmate verified: + +```text +UNIQUE_SURVIVED +REPORT_SURVIVED +PI_SESSION_SURVIVED +/usr/local/bin/herdr +/usr/local/bin/pi +{"running":true,"socket":"/home/agent/.config/herdr/herdr.sock"} +``` + +The persisted Pi session was `019f5a99-f031-7f51-8cc0-156777204dc5`. +Firstmate relaunched `pi --session 019f5a99-f031-7f51-8cc0-156777204dc5` on Terra-low through the replacement Pod's Herdr server. +The resumed agent rehashed the unique artifact, confirmed Herdr health, and appended a recovery record at `2026-07-13T08:33:26Z`. +It then exited, leaving no live Terra child agent. + +The approximate model cost displayed by the final recovery pane was `$0.127`; this is a Pi subscription estimate, not provider billing evidence. + +## Model-policy intervention + +An earlier `openai-codex/gpt-5.4-mini` child was stopped immediately when the captain changed the testing policy. +The primary was restarted on Terra-low, and `/home/agent/config/crew-dispatch.json` now pins all subsequent testing crewmates to: + +```json +{ + "rules": [], + "default": { + "harness": "pi", + "model": "openai-codex/gpt-5.6-terra", + "effort": "low" + } +} +``` + +The aborted pane restored as a stale Herdr entry after Pod replacement but had no live Pi process or accepted result. + +The policy was then closed over all three local launch paths: + +- Pi's persisted defaults are `openai-codex`, `gpt-5.6-terra`, and `low`, so a + direct primary `pi` session uses Terra-low; +- `crew-dispatch.json` pins ordinary crewmates and scouts; +- `secondmate-harness` contains + `pi openai-codex/gpt-5.6-terra low` for Secondmate launches. + +After rebuilding and replacing the Firstmate Pod, image +`docker-pullable://agent-os@sha256:94b7eb6c435f1a226e7279c40491e49c505b30bf54c65de1cd3d5b8e0a102611` +converged all three settings and Herdr returned healthy. The retained separate +mate PVC was also updated to the same direct-Pi defaults. No child agent was +left running after verification. + +## Scope + +This proves the separate-Pod authority boundary, parent-only supervision, persistent home, Pod replacement, unique-work survival, persistent toolchain, and exact Pi session recovery in the local intelligence cluster. +It does not prove Akua-managed KaaS or worker lifecycle, GitHub issue-to-PR delivery, product-cluster read/write boundaries, or a second model provider. diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 4a16458f3..271bc8f3c 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -50,12 +50,21 @@ Start Pi from the tracked distro and attach to Herdr from another terminal: ```sh # inside the primary shell cd /opt/agent-os -pi +pi --model openai-codex/gpt-5.6-terra --thinking low # on the host bin/agent-os-local.sh attach ``` +During the current local evaluation, the OrbStack manifest also converges +`config/crew-dispatch.json` and `config/secondmate-harness` on every primary Pod +start. Crewmates and Secondmates therefore launch through Pi with +`openai-codex/gpt-5.6-terra` and low thinking. The reusable image and Akua +packages remain model-agnostic when those local test environment variables are +absent. The same opt-in policy updates Pi's own `defaultProvider`, +`defaultModel`, and `defaultThinkingLevel`, so a plain `pi` primary session uses +Terra-low too. + The primary can create and manage isolated crewmates directly: ```sh diff --git a/tests/agent-os-container.test.sh b/tests/agent-os-container.test.sh index 53c260ac7..b82738e62 100755 --- a/tests/agent-os-container.test.sh +++ b/tests/agent-os-container.test.sh @@ -24,6 +24,10 @@ assert_grep 'quota-axi@0.1.5' "$ROOT/Dockerfile" "image must pin quota-axi 0.1.5 assert_grep 'ripgrep' "$ROOT/Dockerfile" "image must install ripgrep" assert_grep 'fd-find' "$ROOT/Dockerfile" "image must install fd" assert_grep 'FM_HOME=/home/agent' "$ROOT/Dockerfile" "image must declare the persistent firstmate home" +assert_grep 'never create or read operational state through repo-relative' "$ROOT/AGENTS.md" \ + "Firstmate must anchor operational state to FM_HOME" +assert_grep 'pass a provider-qualified model id' "$ROOT/.agents/skills/harness-adapters/SKILL.md" \ + "Pi dispatch must preserve the selected provider route" assert_grep 'XDG_CONFIG_HOME=/home/agent/.config' "$ROOT/Dockerfile" "image must persist XDG configuration" assert_grep 'NPM_CONFIG_PREFIX=/usr/local' "$ROOT/Dockerfile" "global npm installs must use persistent /usr/local" assert_grep 'PATH=/home/agent/.local/bin:/home/agent/.bun/bin:/home/agent/.cargo/bin:/usr/local/bin' "$ROOT/Dockerfile" \ @@ -41,6 +45,14 @@ assert_no_grep 'USER node' "$ROOT/Dockerfile" "Agent OS containers must start as assert_grep 'exec herdr server' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must keep Herdr as PID 1" assert_grep 'setup hooks' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must install persistent AXI hooks" assert_grep 'agent-os-kubeconfig.sh' "$ROOT/bin/agent-os-container-entrypoint.sh" "entrypoint must prepare in-cluster kubectl access" +assert_grep 'AGENT_OS_TEST_PI_MODEL' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "test-mode Pods must converge the Pi model policy" +assert_grep 'defaultThinkingLevel' "$ROOT/bin/agent-os-container-entrypoint.sh" \ + "test-mode Pods must also pin direct Pi sessions" +assert_grep 'openai-codex/gpt-5.6-terra' "$ROOT/deploy/orbstack/primary.yaml" \ + "the local test fleet must pin Terra" +assert_grep 'AGENT_OS_TEST_PI_EFFORT' "$ROOT/deploy/orbstack/primary.yaml" \ + "the local test fleet must pin Pi reasoning effort" assert_grep 'tokenFile:' "$ROOT/bin/agent-os-kubeconfig.sh" "kubeconfig must follow the projected token file" assert_no_grep 'set-credentials.*--token' "$ROOT/bin/agent-os-kubeconfig.sh" "kubeconfig must not copy a bearer token" assert_grep 'automountServiceAccountToken = False' "$ROOT/tools/agent-os/packages/mate/package.k" \ From 2f9d19a5d417f9a821cc4b476d07a882e1ea7371 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 10:52:24 +0200 Subject: [PATCH 16/56] Pin Terra in remote mate demo --- docs/kubernetes.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 271bc8f3c..e45df8603 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -103,7 +103,8 @@ Use ordinary Herdr and Kubernetes commands to launch, inspect, steer, and retrie ```sh kubectl -n agent-os-demo exec agent-os-mate-scout-1 -- \ - herdr agent start scout-1-research --cwd /home/agent --no-focus -- pi "$(cat /tmp/scout-1.prompt)" + herdr agent start scout-1-research --cwd /home/agent --no-focus -- \ + pi --model openai-codex/gpt-5.6-terra --thinking low "$(cat /tmp/scout-1.prompt)" kubectl -n agent-os-demo exec agent-os-mate-scout-1 -- \ test -s /home/agent/data/scout-1.md ``` From aa6cca59d921c7174799db3be694916e46cbfbf4 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 10:55:46 +0200 Subject: [PATCH 17/56] fix: isolate codex spawn auth boundary --- .agents/skills/harness-adapters/SKILL.md | 4 ++ bin/fm-spawn.sh | 4 +- tests/fm-secondmate-harness.test.sh | 23 +++++++++ tests/fm-spawn-dispatch-profile.test.sh | 62 ++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/.agents/skills/harness-adapters/SKILL.md b/.agents/skills/harness-adapters/SKILL.md index 67bffd886..2cecc9882 100644 --- a/.agents/skills/harness-adapters/SKILL.md +++ b/.agents/skills/harness-adapters/SKILL.md @@ -153,6 +153,10 @@ The decision persists for the repo, so later worktrees of the same project skip Resume after exit with `codex resume `. The session id is printed on quit. +Every Firstmate-launched Codex crewmate, scout, and secondmate must preserve `HOME=/Users/robin` and explicitly unset ambient `CODEX_HOME` at the Codex process boundary. +Apply the same canonical auth boundary to Codex launch, resume, and recovery work, and never request an Orca login. +Pi Quota Router credentials are separate and Pi-only. + **Primary-session guard fact (verified 2026-07-08, codex-cli 0.142.1).** The firstmate PRIMARY's own `.codex/hooks.json` registers a Stop hook that pipes Codex's Stop payload to `bin/fm-turnend-guard.sh`. Codex Stop hooks block on exit 2 and expose `stop_hook_active` for the same one-block loop safety Claude uses. diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 9a8a159a8..4ade0feaf 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -319,9 +319,9 @@ launch_template() { claude) printf '%s' 'CLAUDE_CODE_ENABLE_PROMPT_SUGGESTION=false claude --dangerously-skip-permissions __MODELFLAG____EFFORTFLAG__"$(cat __BRIEF__)"' ;; codex) if [ "$kind" = secondmate ]; then - printf '%s' 'codex __MODELFLAG____EFFORTFLAG__--dangerously-bypass-approvals-and-sandbox "$(cat __BRIEF__)"' + printf '%s' 'env -u CODEX_HOME codex __MODELFLAG____EFFORTFLAG__--dangerously-bypass-approvals-and-sandbox "$(cat __BRIEF__)"' else - printf '%s' 'codex __MODELFLAG____EFFORTFLAG__--dangerously-bypass-approvals-and-sandbox -c "notify=[\"bash\",\"-c\",\"touch __TURNEND__\"]" "$(cat __BRIEF__)"' + printf '%s' 'env -u CODEX_HOME codex __MODELFLAG____EFFORTFLAG__--dangerously-bypass-approvals-and-sandbox -c "notify=[\"bash\",\"-c\",\"touch __TURNEND__\"]" "$(cat __BRIEF__)"' fi ;; opencode) printf '%s' 'OPENCODE_CONFIG_CONTENT='\''{"permission":{"*":"allow"}}'\'' opencode __MODELFLAG__--prompt "$(cat __BRIEF__)"' ;; diff --git a/tests/fm-secondmate-harness.test.sh b/tests/fm-secondmate-harness.test.sh index 1a314c050..2d24ad885 100755 --- a/tests/fm-secondmate-harness.test.sh +++ b/tests/fm-secondmate-harness.test.sh @@ -625,6 +625,28 @@ test_spawn_explicit_harness_uses_explicit_profile_axes() { pass "C8 spawn: an explicit --harness still honors explicit model/effort flags" } +# Secondmates use the same Codex template but prepend their isolated FM_HOME. +# Pin that this extra prefix cannot allow an ambient runtime CODEX_HOME through. +test_secondmate_codex_normalizes_auth_boundary() { + local w sm sm_real launchlog launch out status + w="$TMP_ROOT/spawn-codex-auth-boundary" + sm="$w/sm" + launchlog="$w/launch.log" + mkdir -p "$w/home/config" + printf 'codex gpt-5 high\n' > "$w/home/config/secondmate-harness" + make_seeded_home "$sm" sm + sm_real=$(cd "$sm" && pwd -P) + + out=$(CODEX_HOME=/orca/runtime/home spawn_secondmate_capture "$w" sm "$sm" "$launchlog" 2>&1) + status=$? + expect_code 0 "$status" "Codex secondmate spawn with hostile CODEX_HOME should succeed" + launch=$(cat "$launchlog") + assert_contains "$launch" "FM_HOME='$sm_real' env -u CODEX_HOME codex --model 'gpt-5' -c 'model_reasoning_effort=\"high\"' --dangerously-bypass-approvals-and-sandbox" \ + "Codex secondmate launch must unset CODEX_HOME while preserving home, model, effort, and autonomy" + assert_not_contains "$launch" "notify=" "Codex secondmate launch must retain its no-notify behavior" + pass "Codex secondmate launch normalizes CODEX_HOME at the process boundary" +} + # The harness fallback chain (secondmate-harness -> crew-harness -> own) still # resolves correctly with no model/effort tokens anywhere in the chain, and a # crew/scout (non-secondmate) launch is entirely unaffected by this feature: no @@ -1030,6 +1052,7 @@ test_spawn_explicit_model_overrides_secondmate_harness_token test_spawn_explicit_effort_overrides_secondmate_harness_token test_spawn_explicit_harness_does_not_inherit_secondmate_harness_tokens test_spawn_explicit_harness_uses_explicit_profile_axes +test_secondmate_codex_normalizes_auth_boundary test_spawn_fallback_chain_and_crew_scout_unaffected test_bootstrap_sweep_propagates_and_reconverges test_bootstrap_sweep_propagates_when_tracked_current diff --git a/tests/fm-spawn-dispatch-profile.test.sh b/tests/fm-spawn-dispatch-profile.test.sh index 5b0bad7a3..9309fdfcd 100755 --- a/tests/fm-spawn-dispatch-profile.test.sh +++ b/tests/fm-spawn-dispatch-profile.test.sh @@ -257,6 +257,67 @@ test_codex_omits_invalid_max_effort() { pass "codex omits unsupported max effort instead of passing a bad config value" } +# Every Firstmate-owned Codex launch must reject an ambient runtime-specific +# CODEX_HOME at the final process boundary. The command capture pins ship, +# scout, and batch construction while the fake executable smoke proves the +# child process itself receives only the canonical HOME-bound auth state. +test_codex_normalizes_auth_boundary_for_ship_scout_and_batch() { + local rec ship scout batch_a batch_b out status launches ship_launch scout_launch envlog canonical_home + ship=auth-ship-z17 + scout=auth-scout-z18 + batch_a=auth-batch-a-z19 + batch_b=auth-batch-b-z20 + rec=$(make_spawn_case codex-auth-boundary codex "$ship" "$scout" "$batch_a" "$batch_b") + read_case_record "$rec" + + out=$(CODEX_HOME=/orca/runtime/home run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" \ + "$ship" "$PROJ_DIR" --harness codex --model gpt-5 --effort high) + status=$? + expect_code 0 "$status" "Codex ship spawn with hostile CODEX_HOME should succeed" + ship_launch=$(cat "$LAUNCH_LOG") + assert_contains "$ship_launch" "env -u CODEX_HOME codex --model 'gpt-5' -c 'model_reasoning_effort=\"high\"' --dangerously-bypass-approvals-and-sandbox" \ + "Codex ship launch must unset CODEX_HOME before preserving model, effort, and autonomy" + assert_contains "$ship_launch" "notify=[\\\"bash\\\",\\\"-c\\\",\\\"touch '" \ + "Codex ship launch must retain its notify turn-end behavior" + + out=$(CODEX_HOME=/orca/runtime/home run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" \ + "$scout" "$PROJ_DIR" --harness codex --scout) + status=$? + expect_code 0 "$status" "Codex scout spawn with hostile CODEX_HOME should succeed" + scout_launch=$(cat "$LAUNCH_LOG") + assert_contains "$scout_launch" "env -u CODEX_HOME codex --dangerously-bypass-approvals-and-sandbox" \ + "Codex scout launch must unset CODEX_HOME before preserving autonomy" + assert_contains "$scout_launch" "notify=[\\\"bash\\\",\\\"-c\\\",\\\"touch '" \ + "Codex scout launch must retain its notify turn-end behavior" + + out=$(CODEX_HOME=/orca/runtime/home run_spawn "$HOME_DIR" "$WT_DIR" "$FAKEBIN_DIR" "$LAUNCH_LOG" \ + "$batch_a=$PROJ_DIR" "$batch_b=$PROJ_DIR" --harness codex --model gpt-5 --effort high) + status=$? + expect_code 0 "$status" "Codex batch spawn with hostile CODEX_HOME should succeed" + launches=$(cat "$LAUNCH_LOG") + [ "$(printf '%s\n' "$launches" | grep -c '^env -u CODEX_HOME codex ' || true)" -eq 2 ] \ + || fail "every Codex batch launch must normalize CODEX_HOME: $launches" + + envlog="$CASE_DIR/fake-codex-env.log" + cat > "$FAKEBIN_DIR/codex" <<'SH' +#!/usr/bin/env bash +set -u +printf 'HOME=%s\n' "$HOME" > "$FM_FAKE_CODEX_ENV_LOG" +if [ "${CODEX_HOME+x}" = x ]; then + printf 'CODEX_HOME=set\n' >> "$FM_FAKE_CODEX_ENV_LOG" +else + printf 'CODEX_HOME=unset\n' >> "$FM_FAKE_CODEX_ENV_LOG" +fi +SH + chmod +x "$FAKEBIN_DIR/codex" + canonical_home="$CASE_DIR/canonical-home" + HOME="$canonical_home" CODEX_HOME=/orca/runtime/home PATH="$FAKEBIN_DIR:$PATH" \ + FM_FAKE_CODEX_ENV_LOG="$envlog" bash -c "$ship_launch" + assert_grep "HOME=$canonical_home" "$envlog" "Codex auth normalization must preserve HOME" + assert_grep 'CODEX_HOME=unset' "$envlog" "Codex child must see CODEX_HOME unset, not inherited" + pass "Codex ship, scout, and batch launches normalize CODEX_HOME while preserving launch behavior" +} + test_grok_threads_model_and_reasoning_effort() { local rec id out status launch id=profile-grok-z5 @@ -373,6 +434,7 @@ test_active_dispatch_profile_allows_raw_launch_command test_claude_threads_model_and_effort test_codex_threads_model_and_effort test_codex_omits_invalid_max_effort +test_codex_normalizes_auth_boundary_for_ship_scout_and_batch test_grok_threads_model_and_reasoning_effort test_grok_omits_invalid_max_reasoning_effort test_opencode_threads_model_and_ignores_effort_axis From 761223c533de5cba5a96681cf769ec12164a267e Mon Sep 17 00:00:00 2001 From: root Date: Mon, 13 Jul 2026 09:00:53 +0000 Subject: [PATCH 18/56] fix: select rebuilt local demo image --- AGENTS.md | 7 ++++++ bin/agent-os-local.sh | 15 ++++++++++++ docs/kubernetes.md | 2 ++ tests/agent-os-local.test.sh | 46 +++++++++++++++++++++++++++++++----- 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e0dd21f18..40666a79a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -788,3 +788,10 @@ On an `x-mention ` or `x-mode-error ...` `check:` wake, load `fmx-re It owns mention classification, acting on the request, reply composition, voice, thread-splitting, image attachments, dry-run preview, and the completion-follow-up procedure in full, including what an `x-mode-error` wake means instead. `docs/configuration.md` "X mode (.env)" has the wire-protocol reference. The one fact that must survive here because it fires on a generic terminal wake, not the mention wake itself: when an X-mode-linked task reaches a terminal state, post its final completion follow-up per section 8's wake-handling step before tearing down. + +## Maintaining this file + +Keep this file for knowledge useful to almost every future agent session in this project. +Do not repeat what the codebase already shows; point to the authoritative file or command instead. +Prefer rewriting or pruning existing entries over appending new ones. +When updating this file, preserve this bar for all agents and keep entries concise. diff --git a/bin/agent-os-local.sh b/bin/agent-os-local.sh index 03c1830dd..d0c4e7ad8 100755 --- a/bin/agent-os-local.sh +++ b/bin/agent-os-local.sh @@ -7,6 +7,7 @@ ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) CONTEXT=${AGENT_OS_CONTEXT:-orbstack} NAMESPACE=${AGENT_OS_NAMESPACE:-agent-os-demo} IMAGE=${AGENT_OS_IMAGE:-agent-os:dev} +IMAGE_IS_OVERRIDE=${AGENT_OS_IMAGE+x} COMMAND=${1:-} if [ "$CONTEXT" != orbstack ] && [ "${AGENT_OS_ALLOW_NON_ORBSTACK:-0}" != 1 ]; then @@ -14,11 +15,20 @@ if [ "$CONTEXT" != orbstack ] && [ "${AGENT_OS_ALLOW_NON_ORBSTACK:-0}" != 1 ]; t exit 2 fi +local_image_tag() { + local image_id + image_id=$(docker image inspect --format '{{.Id}}' "$IMAGE") + printf 'agent-os:local-%s\n' "${image_id#sha256:}" +} + cd "$ROOT" case "$COMMAND" in build) docker build -t "$IMAGE" . + if [ -z "$IMAGE_IS_OVERRIDE" ]; then + docker tag "$IMAGE" "$(local_image_tag)" + fi ;; deploy) if [ "$CONTEXT" = orbstack ]; then @@ -26,6 +36,11 @@ case "$COMMAND" in kubectl --context "$CONTEXT" wait --for=condition=Ready node/orbstack --timeout=120s fi kubectl --context "$CONTEXT" apply -k deploy/orbstack + if [ -z "$IMAGE_IS_OVERRIDE" ]; then + IMAGE=$(local_image_tag) + fi + kubectl --context "$CONTEXT" -n "$NAMESPACE" set image statefulset/agent-os-firstmate \ + "agent-os-init=$IMAGE" "firstmate=$IMAGE" ;; status) kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset agent-os-firstmate diff --git a/docs/kubernetes.md b/docs/kubernetes.md index e45df8603..12aa5ff82 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -35,6 +35,8 @@ bin/agent-os-local.sh status bin/agent-os-local.sh shell ``` +With the default local image, each `build` also creates a content-addressed local tag. The following `deploy` updates both StatefulSet containers to that tag, so a rebuilt `agent-os:dev` cannot reuse a stale OrbStack runtime digest. Set `AGENT_OS_IMAGE` to build and deploy an explicit image name without replacing or retagging that override. + The image includes Firstmate's complete required toolchain, including `gh`, `rg`, `fd`, Akua, `kubectl`, K9s, treehouse, no-mistakes, and every required AXI CLI. Authenticate GitHub inside the primary with `gh auth login`. Authenticate Pi using `/login` and the provider flow you choose. diff --git a/tests/agent-os-local.test.sh b/tests/agent-os-local.test.sh index 8c9ce018d..a95379b1c 100755 --- a/tests/agent-os-local.test.sh +++ b/tests/agent-os-local.test.sh @@ -22,11 +22,22 @@ SH } make_fake docker +cat > "$FAKEBIN/docker" <<'SH' +#!/usr/bin/env bash +printf '%s' "$(basename "$0")" >> "$AGENT_OS_TEST_LOG" +printf ' %s' "$@" >> "$AGENT_OS_TEST_LOG" +printf '\n' >> "$AGENT_OS_TEST_LOG" +if [ "${1:-}" = image ] && [ "${2:-}" = inspect ]; then + printf '%s\n' "${AGENT_OS_TEST_IMAGE_ID:?AGENT_OS_TEST_IMAGE_ID is required for image inspection}" +fi +SH +chmod +x "$FAKEBIN/docker" make_fake kubectl make_fake orbctl run_cli() { - PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" "$CLI" "$@" + PATH="$FAKEBIN:$PATH" AGENT_OS_TEST_LOG="$LOG" \ + AGENT_OS_TEST_IMAGE_ID="${AGENT_OS_TEST_IMAGE_ID:-sha256:default}" "$CLI" "$@" } assert_call() { @@ -52,11 +63,33 @@ test_deploy_starts_local_kubernetes_and_applies_kustomize() { pass "deploy starts and targets only OrbStack" } -test_build_uses_local_image_name() { +test_rebuild_deploy_uses_a_new_immutable_local_tag() { : > "$LOG" - run_cli build - assert_call 'docker build -t agent-os:dev .' "build must use the local demo image tag" - pass "build uses the deterministic local image tag" + AGENT_OS_TEST_IMAGE_ID=sha256:stale run_cli build + AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_cli build + AGENT_OS_TEST_IMAGE_ID=sha256:rebuilt run_cli deploy + + assert_call 'docker build -t agent-os:dev .' "build must retain the local demo image tag" + assert_call 'docker tag agent-os:dev agent-os:local-rebuilt' \ + "build must assign the rebuilt image a unique local tag" + assert_call 'kubectl --context orbstack -n agent-os-demo set image statefulset/agent-os-firstmate agent-os-init=agent-os:local-rebuilt firstmate=agent-os:local-rebuilt' \ + "deploy must replace the stale mutable tag with the rebuilt local tag" + pass "rebuild deploy selects the rebuilt local image instead of a stale mutable tag" +} + +test_explicit_image_override_is_used_without_retagging() { + : > "$LOG" + AGENT_OS_IMAGE=example.test/agent-os:custom run_cli build + AGENT_OS_IMAGE=example.test/agent-os:custom run_cli deploy + + assert_call 'docker build -t example.test/agent-os:custom .' \ + "build must preserve an explicit image override" + assert_call 'kubectl --context orbstack -n agent-os-demo set image statefulset/agent-os-firstmate agent-os-init=example.test/agent-os:custom firstmate=example.test/agent-os:custom' \ + "deploy must preserve an explicit image override" + if grep -F 'docker tag example.test/agent-os:custom' "$LOG" >/dev/null; then + fail "explicit image overrides must not be retagged" + fi + pass "explicit image override remains intact" } test_destroy_requires_exact_confirmation() { @@ -88,6 +121,7 @@ test_non_orbstack_context_is_fail_closed() { test_status_pins_context_and_namespace test_deploy_starts_local_kubernetes_and_applies_kustomize -test_build_uses_local_image_name +test_rebuild_deploy_uses_a_new_immutable_local_tag +test_explicit_image_override_is_used_without_retagging test_destroy_requires_exact_confirmation test_non_orbstack_context_is_fail_closed From 0c6588511904e0a6312b529b1b1ccc8ed9c390a2 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 11:07:05 +0200 Subject: [PATCH 19/56] Record GitHub issue to PR fleet proof --- docs/acceptance.md | 4 +- docs/evidence/2026-07-13-github-issue-pr.md | 112 ++++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 docs/evidence/2026-07-13-github-issue-pr.md diff --git a/docs/acceptance.md b/docs/acceptance.md index b69ba6a5f..328b4f363 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -14,10 +14,10 @@ Update it from real runs; planned behavior and model narration never count as pr | Same-Pod general-purpose crewmate | Firstmate-supervised Pi scout, real Kubernetes test, in-cluster context check, and report in `docs/evidence/2026-07-13-same-pod-firstmate.md` | Proven locally | | Separate persistent crewmate Pod with explicit authority | Terra-low task, explicit resources, read-only Pi Secret, no ServiceAccount token, and retained PVC in `docs/evidence/2026-07-13-separate-pod-recovery.md` | Proven locally | | Parent-only supervision and human attach | Same-Pod and separate-Pod tasks accepted only Firstmate briefs and steers; a connected Herdr client observed the live workspaces | Proven locally | -| Real issue to tested review-ready PR | Existing Firstmate delivery machinery | Not yet proven through the Kubernetes fleet | +| Real issue to tested review-ready PR | Issue [#2](https://github.com/akua-dev/agent-os/issues/2) became Terra-low task `fix-local-rebuild-image-r2`, tested commit `761223c`, and PR [#3](https://github.com/akua-dev/agent-os/pull/3), then merged into review branch PR #1; see `docs/evidence/2026-07-13-github-issue-pr.md` | Proven locally | | Parent and child restart without unique-work loss | Primary session recovery plus separate-Pod replacement with unique artifact, report, tools, and exact Pi session resumed | Proven locally | | Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | -| Optional GitHub Issue/Project client | Cortex contract only | Deferred until terminal-native path passes | +| Optional GitHub Issue/Project client | Repository Issues enabled only when the native intake proof needed it; Firstmate read Issue #2 through `gh-axi` and delivered PR #3 with no custom intake service | Issue client proven locally; Project client remains optional and unproven | | Repeatable end-to-end eval and recordable demo | Same-Pod and separate-Pod run records now capture model, time, cost estimate, interventions, resource IDs, and failures | Local agent lifecycle captured; Akua/GitHub/product path incomplete | | Public installable image and release | Multi-architecture workflow run `29188585384` succeeded; local image only because pull-request builds do not publish | Publication not yet proven | diff --git a/docs/evidence/2026-07-13-github-issue-pr.md b/docs/evidence/2026-07-13-github-issue-pr.md new file mode 100644 index 000000000..f9ba605a2 --- /dev/null +++ b/docs/evidence/2026-07-13-github-issue-pr.md @@ -0,0 +1,112 @@ +# GitHub Issue to Kubernetes crewmate PR verification + +Date: 2026-07-13 +Kubernetes context: `orbstack` +Namespace: `agent-os-demo` +Issue: [#2 Make local rebuilds select the new Agent OS image](https://github.com/akua-dev/agent-os/issues/2) +Delivery PR: [#3 fix: select rebuilt local demo image](https://github.com/akua-dev/agent-os/pull/3) +Final review surface: [#1 feat: run Firstmate as a Kubernetes agent OS](https://github.com/akua-dev/agent-os/pull/1) + +## Claim + +A GitHub Issue can act as optional Agent OS intake without becoming the runtime +or communication layer. A Kubernetes-resident Firstmate can read that issue, +turn it into a precise brief, dispatch and supervise a general-purpose +crewmate, and return a tested review-ready PR. Ordinary Git, GitHub, Firstmate +files, treehouse worktrees, and Herdr terminals remain the operating substrate. + +## Topology and authority + +- Primary Firstmate: Pi in dedicated Herdr workspace `w6`, persistent + `FM_HOME=/home/agent`. +- Crewmate task: `fix-local-rebuild-image-r2`, Herdr pane `w2:p9`. +- Model policy: `openai-codex/gpt-5.6-terra`, low thinking, recorded in the + task metadata. +- Worktree: + `/home/agent/.treehouse/agent-os-eaf3be/1/agent-os`. +- Delivery mode: `direct-PR`; no custom task service, workflow engine, or + inter-agent protocol. +- GitHub authority came from the selected credential already persisted in the + Firstmate home. No credential value was copied into the brief or evidence. + +The evaluator initially discovered that the public Agent OS repository had +Issues disabled. It enabled only that native repository surface, then created +Issue #2 from the observed stale-image failure. No GitHub Project or additional +intake service was required. + +## Parent-only supervision + +The evaluator prompted only the primary Firstmate. Firstmate read Issue #2 with +`gh-axi`, registered the existing Agent OS clone, wrote the child brief, and +spawned the crewmate through: + +```text +spawned fix-local-rebuild-image-r2 harness=pi kind=ship mode=direct-PR +model=openai-codex/gpt-5.6-terra effort=low +window=default:w2:p9 +``` + +Firstmate handled the Pi trust prompt through `fm-send.sh`, polled the child's +status and pane, armed `fm-pr-check.sh`, and inspected the delivered diff. The +evaluator made read-only Herdr observations and never prompted or steered the +child directly. The primary pane was moved into its own workspace early in the +run so task-tab cleanup could not remove the supervisor. + +## RED and GREEN evidence + +The child first extended `tests/agent-os-local.test.sh`. Against the old helper, +the focused test failed for the missing immutable tag operation: + +```text +not ok - build must assign the rebuilt image a unique local tag +(missing exact call: docker tag agent-os:dev agent-os:local-rebuilt) +RED_EXIT=1 +``` + +Commit `761223c533de5cba5a96681cf769ec12164a267e` then: + +- tags the default local image with its Docker image ID; +- updates both StatefulSet containers to that content-specific tag on deploy; +- preserves an explicit `AGENT_OS_IMAGE` without retagging it; +- documents the behavior; and +- adds hermetic stale-tag and override tests. + +The child passed `tests/agent-os-local.test.sh` and shell syntax checks. Its +attempt to run `tests/agent-os-kubernetes.test.sh` correctly exposed an +environment boundary: an in-cluster Firstmate Pod has a ServiceAccount token, +so it cannot represent the test's token-free host case. + +The evaluator closed that evidence gap from an isolated host worktree at the +exact child commit. All `tests/agent-os-*.test.sh` passed, including the +host-only refusal case. Focused ShellCheck, `bash -n`, and `git diff --check` +also passed. This independent result was recorded as a PR #3 comment before +integration. + +## GitHub result and timings + +- Issue #2 created: `2026-07-13T08:55:04Z`. +- PR #3 created: `2026-07-13T09:01:43Z`. +- Issue-to-review-ready-PR: 6 minutes 39 seconds. +- PR #3 merged into `feat/orbstack-demo`: `2026-07-13T09:05:28Z`. +- Issue-to-verified-integration: 10 minutes 24 seconds. +- Merge commit on the review branch: + `7679bda383d044aaec6d6fae7231bb7c0fb80576`. +- Files changed by the child: `AGENTS.md`, `bin/agent-os-local.sh`, + `docs/kubernetes.md`, and `tests/agent-os-local.test.sh`. + +Pi displayed approximate subscription cost estimates of `$0.966` for the child +and `$0.815` for the primary by the end of their panes. These are UI estimates, +not provider invoices. + +Three evaluator interventions occurred: move the primary into a dedicated +workspace, run the host-only verification, and merge the verified stacked PR +into the final review branch. No evaluator intervention changed the child's +implementation. + +## Scope + +This proves native GitHub Issue intake, parent-only same-Pod delegation, real +RED-to-GREEN implementation, a tested review-ready PR, and integration into the +single Agent OS review branch. It does not prove GitHub Projects, an +Akua-managed intelligence cluster, Akua worker lifecycle, or product-cluster +access boundaries. From d31cfa906e59324add5a4dfba12bc4e13429ecf9 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 11:10:04 +0200 Subject: [PATCH 20/56] Record fleet task reconciliation --- docs/evidence/2026-07-13-github-issue-pr.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/evidence/2026-07-13-github-issue-pr.md b/docs/evidence/2026-07-13-github-issue-pr.md index f9ba605a2..efe7bd4f3 100644 --- a/docs/evidence/2026-07-13-github-issue-pr.md +++ b/docs/evidence/2026-07-13-github-issue-pr.md @@ -95,13 +95,18 @@ integration. `docs/kubernetes.md`, and `tests/agent-os-local.test.sh`. Pi displayed approximate subscription cost estimates of `$0.966` for the child -and `$0.815` for the primary by the end of their panes. These are UI estimates, +and `$0.917` for the primary after final reconciliation. These are UI estimates, not provider invoices. -Three evaluator interventions occurred: move the primary into a dedicated -workspace, run the host-only verification, and merge the verified stacked PR -into the final review branch. No evaluator intervention changed the child's -implementation. +Four evaluator interventions occurred: move the primary into a dedicated +workspace, run the host-only verification, merge the verified stacked PR into +the final review branch, and ask Firstmate to reconcile and tear down the +completed child after that external merge. No evaluator intervention changed +the child's implementation or communicated with it directly. + +Firstmate then ran its ordinary teardown path, marked the task done in the +backlog, removed the child pane and task metadata, and left the retained +separate-Pod recovery fixture, primary workspace, and committed evidence intact. ## Scope From ff57e0b234a73d02c106c05a5d2ca63d574f3ad5 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 12:01:51 +0200 Subject: [PATCH 21/56] docs: keep Agent OS work tracking private --- docs/acceptance.md | 4 ++-- docs/evidence/2026-07-13-github-issue-pr.md | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/acceptance.md b/docs/acceptance.md index 328b4f363..55ea1956e 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -14,10 +14,10 @@ Update it from real runs; planned behavior and model narration never count as pr | Same-Pod general-purpose crewmate | Firstmate-supervised Pi scout, real Kubernetes test, in-cluster context check, and report in `docs/evidence/2026-07-13-same-pod-firstmate.md` | Proven locally | | Separate persistent crewmate Pod with explicit authority | Terra-low task, explicit resources, read-only Pi Secret, no ServiceAccount token, and retained PVC in `docs/evidence/2026-07-13-separate-pod-recovery.md` | Proven locally | | Parent-only supervision and human attach | Same-Pod and separate-Pod tasks accepted only Firstmate briefs and steers; a connected Herdr client observed the live workspaces | Proven locally | -| Real issue to tested review-ready PR | Issue [#2](https://github.com/akua-dev/agent-os/issues/2) became Terra-low task `fix-local-rebuild-image-r2`, tested commit `761223c`, and PR [#3](https://github.com/akua-dev/agent-os/pull/3), then merged into review branch PR #1; see `docs/evidence/2026-07-13-github-issue-pr.md` | Proven locally | +| Real issue to tested review-ready PR | A private Cortex tracker issue became Terra-low task `fix-local-rebuild-image-r2`, tested commit `761223c`, and public PR [#3](https://github.com/akua-dev/agent-os/pull/3), then merged into review branch PR #1; see `docs/evidence/2026-07-13-github-issue-pr.md` | Proven locally | | Parent and child restart without unique-work loss | Primary session recovery plus separate-Pod replacement with unique artifact, report, tools, and exact Pi session resumed | Proven locally | | Separate production read-only and scoped write identity | Cortex contract only | Not yet implemented or Red-approved | -| Optional GitHub Issue/Project client | Repository Issues enabled only when the native intake proof needed it; Firstmate read Issue #2 through `gh-axi` and delivered PR #3 with no custom intake service | Issue client proven locally; Project client remains optional and unproven | +| Optional GitHub Issue/Project client | Firstmate read a private Cortex GitHub issue through `gh-axi` and delivered public PR #3 with no custom intake service; public Agent OS Issues are disabled because work tracking is private | Issue client proven locally; Project client remains optional and unproven | | Repeatable end-to-end eval and recordable demo | Same-Pod and separate-Pod run records now capture model, time, cost estimate, interventions, resource IDs, and failures | Local agent lifecycle captured; Akua/GitHub/product path incomplete | | Public installable image and release | Multi-architecture workflow run `29188585384` succeeded; local image only because pull-request builds do not publish | Publication not yet proven | diff --git a/docs/evidence/2026-07-13-github-issue-pr.md b/docs/evidence/2026-07-13-github-issue-pr.md index efe7bd4f3..b32b28858 100644 --- a/docs/evidence/2026-07-13-github-issue-pr.md +++ b/docs/evidence/2026-07-13-github-issue-pr.md @@ -3,7 +3,7 @@ Date: 2026-07-13 Kubernetes context: `orbstack` Namespace: `agent-os-demo` -Issue: [#2 Make local rebuilds select the new Agent OS image](https://github.com/akua-dev/agent-os/issues/2) +Issue: private Cortex tracker item; public implementation PR below Delivery PR: [#3 fix: select rebuilt local demo image](https://github.com/akua-dev/agent-os/pull/3) Final review surface: [#1 feat: run Firstmate as a Kubernetes agent OS](https://github.com/akua-dev/agent-os/pull/1) @@ -30,15 +30,16 @@ files, treehouse worktrees, and Herdr terminals remain the operating substrate. Firstmate home. No credential value was copied into the brief or evidence. The evaluator initially discovered that the public Agent OS repository had -Issues disabled. It enabled only that native repository surface, then created -Issue #2 from the observed stale-image failure. No GitHub Project or additional -intake service was required. +Issues disabled. It enabled only that native repository surface and created an +implementation issue from the observed stale-image failure. The tracker item +was later transferred into private Cortex and public Agent OS Issues were +disabled again. No GitHub Project or additional intake service was required. ## Parent-only supervision -The evaluator prompted only the primary Firstmate. Firstmate read Issue #2 with -`gh-axi`, registered the existing Agent OS clone, wrote the child brief, and -spawned the crewmate through: +The evaluator prompted only the primary Firstmate. Firstmate read the issue +with `gh-axi`, registered the existing Agent OS clone, wrote the child brief, +and spawned the crewmate through: ```text spawned fix-local-rebuild-image-r2 harness=pi kind=ship mode=direct-PR @@ -84,7 +85,7 @@ integration. ## GitHub result and timings -- Issue #2 created: `2026-07-13T08:55:04Z`. +- Private issue created: `2026-07-13T08:55:04Z`. - PR #3 created: `2026-07-13T09:01:43Z`. - Issue-to-review-ready-PR: 6 minutes 39 seconds. - PR #3 merged into `feat/orbstack-demo`: `2026-07-13T09:05:28Z`. From 83e69abdd220dd8ccdbc18361430068c96e5072f Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 13 Jul 2026 12:15:43 +0200 Subject: [PATCH 22/56] no-mistakes(review): Captain, generalize Codex auth-boundary guidance --- .agents/skills/harness-adapters/SKILL.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.agents/skills/harness-adapters/SKILL.md b/.agents/skills/harness-adapters/SKILL.md index 2cecc9882..f6ac66b32 100644 --- a/.agents/skills/harness-adapters/SKILL.md +++ b/.agents/skills/harness-adapters/SKILL.md @@ -150,12 +150,11 @@ Directory trust dialog on first run per repo root: "Do you trust the contents of Accept with Enter. The decision persists for the repo, so later worktrees of the same project skip it. -Resume after exit with `codex resume `. +Resume after exit with `env -u CODEX_HOME codex resume `. The session id is printed on quit. -Every Firstmate-launched Codex crewmate, scout, and secondmate must preserve `HOME=/Users/robin` and explicitly unset ambient `CODEX_HOME` at the Codex process boundary. -Apply the same canonical auth boundary to Codex launch, resume, and recovery work, and never request an Orca login. -Pi Quota Router credentials are separate and Pi-only. +Every Firstmate-launched Codex crewmate, scout, and secondmate must preserve the invoking user's existing `HOME` and explicitly unset ambient `CODEX_HOME` at the Codex process boundary. +The same boundary applies to Codex launch, resume, and recovery work. **Primary-session guard fact (verified 2026-07-08, codex-cli 0.142.1).** The firstmate PRIMARY's own `.codex/hooks.json` registers a Stop hook that pipes Codex's Stop payload to `bin/fm-turnend-guard.sh`. From 2364817c2cb196e9b2c6b892a499f4aa42cc181f Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Mon, 13 Jul 2026 03:19:34 -0700 Subject: [PATCH 23/56] fix: guard secondmate primary sessions from blind turn ends (#505) * fix: guard secondmate own-home turn ends Remove the .fm-secondmate-home early-exit in fm-turnend-guard.sh so the 'no turn ends blind' backstop fires in a secondmate's own primary session, matching the cd-guard's scope: the own home is guarded, child crew/scout worktrees stay exempt via the retained git-dir/git-common-dir test. This was pure scoping from the guard's primary-only origin and guarded against no secondmate-specific hazard. Add secondmate regression tests (blind-turn block, idle-by-default, stop_hook_active loop guard, deferred-death recovery loop, child-worktree exemption) and record the autonomous background-notify re-invoke measurement (Claude Code 2.1.207, 11s) in docs/turnend-guard.md. * no-mistakes(document): Correct secondmate guard documentation, captain * fix: force-include marked secondmate homes in turn-end guard The prior remove-only form (just deleting the .fm-secondmate-home check) left the DEFAULT secondmate topology unguarded: a treehouse-leased home is a linked git worktree (git-dir != git-common-dir), which the retained git-dir exemption still skipped, so its own primary session could still end a turn blind. Invert the marker: a genuinely-marked home is force-included as a guarded primary (treehouse-leased linked OR git-cloned plain), and the git-dir exemption applies only to UNMARKED child worktrees. Marker validation (regular non-symlink file, non-empty id-token content) blocks a stray or empty marker from spoofing inclusion. Add real linked-worktree regression tests: a treehouse-leased LINKED secondmate home is guarded, a stray/empty marker stays exempt, and the unmarked child worktree stays exempt - the topology the plain git-init fixtures masked. Predicates, in-flight gate, and loop guard untouched. * fix: force ASCII collation in secondmate marker validation Add a function-scoped local LC_ALL=C in fm_root_is_secondmate_home so the [A-Za-z0-9._-] id allowlist matches under C collation, not the ambient locale - a locale-crafted non-ASCII marker id can no longer slip through the range match and spoof force-inclusion of a linked child worktree. Add a regression test proving a non-ASCII marker id is rejected and the linked worktree stays exempt. * no-mistakes(test): fix backend baseline gate-refusal dependency * no-mistakes(document): Correct secondmate turn-end guard documentation --- AGENTS.md | 2 +- bin/fm-cd-pretool-check.sh | 19 ++- bin/fm-turnend-guard.sh | 74 +++++++--- docs/architecture.md | 2 +- docs/cd-guard.md | 14 +- docs/supervision-protocols/opencode.md | 2 +- docs/turnend-guard.md | 42 +++++- tests/fm-backend.test.sh | 2 +- tests/fm-turnend-guard.test.sh | 189 ++++++++++++++++++++++++- 9 files changed, 296 insertions(+), 50 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 70181739a..705c30768 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -613,7 +613,7 @@ If a crewmate sent to work firstmate-on-itself branches or commits in the primar Only a named non-default branch checked out in the primary alarms: detached HEAD (the legitimate resting state of crewmate worktrees and secondmate homes) and the default branch never do. The same assertion runs at session start as the bootstrap `TANGLE:` line (handled via `bootstrap-diagnostics`), and two upstream guards prevent the tangle: `fm-spawn`'s isolated-worktree assertion and the ship brief's opening isolation check (section 11). -On every verified primary harness, "no turn ends blind" has a structural backstop beyond the pull-based banner: `bin/fm-turnend-guard.sh` blocks the turn end (or forces one bounded follow-up on passive harnesses) when tasks are in flight without a live identity-matched watcher lock and fresh beacon, fires only in the actual primary checkout, and stays silent when supervision is healthy. +On every verified primary harness, "no turn ends blind" has a structural backstop beyond the pull-based banner: `bin/fm-turnend-guard.sh` blocks the turn end (or forces one bounded follow-up on passive harnesses) when tasks are in flight without a live identity-matched watcher lock and fresh beacon, guards both the main primary and a secondmate's own primary session, and stays silent when supervision is healthy. `docs/turnend-guard.md` owns the per-harness hook mechanisms, empirical validation, scoping details, and documented fail-open tradeoffs. Watcher liveness is harness-aware. Do not assume one primary harness can use another harness's foreground or background shape. diff --git a/bin/fm-cd-pretool-check.sh b/bin/fm-cd-pretool-check.sh index 2867c7c9e..a57ba9d2a 100755 --- a/bin/fm-cd-pretool-check.sh +++ b/bin/fm-cd-pretool-check.sh @@ -123,16 +123,15 @@ esac SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" 2>/dev/null && pwd -P) || exit 0 FM_ROOT=${FM_ROOT_OVERRIDE:-$(CDPATH='' cd -- "$SCRIPT_DIR/.." 2>/dev/null && pwd -P)} || exit 0 -# Scope to the ACTUAL primary firstmate checkout. This reuses the turn-end -# guard's primary detection (docs/turnend-guard.md): a plain, non-worktree -# checkout has git-dir equal to git-common-dir, while a crewmate/scout task -# worktree - the shape bin/fm-spawn.sh always hands out - is a linked git -# worktree where the two differ. Unlike the turn-end guard, the cd-guard does -# NOT exclude secondmate homes (the .fm-secondmate-home marker): a secondmate's -# OWN primary session is a primary and must be guarded too; only its child -# crew/scout worktrees are exempt, and they are exempt here by the same -# linked-worktree test. Any failure to confirm the primary is inert (exit 0), -# never a block, so a broken environment never denies a shell command. +# Scope to a plain, non-worktree firstmate checkout, where git-dir equals +# git-common-dir. A crewmate/scout task worktree - the shape bin/fm-spawn.sh +# always hands out - is a linked git worktree where the two differ. This guard +# does not inspect .fm-secondmate-home, so it applies in a git-cloned secondmate +# home but remains inert when the secondmate home is itself a treehouse-leased +# linked worktree. docs/cd-guard.md owns this scope; docs/turnend-guard.md owns +# the turn-end guard's separate marker-aware scope. Any failure to confirm the +# checkout is inert (exit 0), never a block, so a broken environment never +# denies a shell command. [ -f "$FM_ROOT/AGENTS.md" ] || exit 0 [ -d "$FM_ROOT/bin" ] || exit 0 command -v git >/dev/null 2>&1 || exit 0 diff --git a/bin/fm-turnend-guard.sh b/bin/fm-turnend-guard.sh index 3fcadf6bd..eef947f85 100755 --- a/bin/fm-turnend-guard.sh +++ b/bin/fm-turnend-guard.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash -# Primary turn-end guard for the firstmate PRIMARY session only. +# Turn-end guard for any firstmate PRIMARY session: the main home OR a +# secondmate's own home. A secondmate runs its own primary firstmate session and +# is guarded exactly like the main primary; only child crew/scout worktrees are +# exempt (see the scoping block below and docs/turnend-guard.md). # # fm-guard.sh (bin/fm-guard.sh) is pull-based: it only warns when some other # supervision script happens to run. A primary session that ends a turn without @@ -14,11 +17,14 @@ # and fail-open tradeoffs. # # Ships with TRACKED harness hook files at the repo root, so this file is -# checked out into every worktree of this repo: the primary checkout, any -# crewmate/scout task worktree spawned to work on firstmate itself (the -# recursive "firstmate improving itself" case), and every secondmate home -# (treehouse-leased or git-cloned). It must therefore scope itself to the -# PRIMARY at runtime and stay a silent, fast no-op everywhere else. +# checked out into every worktree of this repo: the primary checkout, every +# secondmate home (treehouse-leased or git-cloned), and any crewmate/scout task +# worktree spawned to work on firstmate itself (the recursive "firstmate +# improving itself" case). A secondmate home runs its OWN primary firstmate +# session, so it must be guarded like the main primary; only child crew/scout +# worktrees are exempt. It must therefore scope itself at runtime to a real +# primary checkout - the main home or a genuinely marked secondmate home - and +# stay a silent, fast no-op inside child task worktrees. # # Loop-guard: never block twice in the same turn. Claude Code and codex Stop # payloads carry stop_hook_active=true when the CURRENT stop attempt was itself @@ -55,19 +61,49 @@ command -v jq >/dev/null 2>&1 || exit 0 STOP_HOOK_ACTIVE=$(printf '%s' "$PAYLOAD" | jq -r '.stop_hook_active // false' 2>/dev/null) || exit 0 [ "$STOP_HOOK_ACTIVE" = "true" ] && exit 0 -# --- scope precisely to the PRIMARY checkout -------------------------------- -# Excludes secondmate homes (the .fm-secondmate-home marker is written at seed -# time regardless of whether the home was treehouse-leased or git-cloned; see -# bin/fm-home-seed.sh) and ordinary crewmate/scout task worktrees of -# firstmate-on-itself (bin/fm-spawn.sh only ever hands those out as genuine -# linked `git worktree`s - it aborts the spawn otherwise - so a plain, -# non-worktree checkout is never one of those). A linked worktree's git-dir -# lives under the main repo's .git/worktrees/ and differs from the common -# (shared) git-dir; only the main, non-worktree checkout has the two equal. -[ -f "$FM_ROOT/.fm-secondmate-home" ] && exit 0 -GIT_DIR=$(git -C "$FM_ROOT" rev-parse --git-dir 2>/dev/null) || exit 0 -GIT_COMMON_DIR=$(git -C "$FM_ROOT" rev-parse --git-common-dir 2>/dev/null) || exit 0 -[ "$GIT_DIR" = "$GIT_COMMON_DIR" ] || exit 0 +# Return 0 when $1 (a firstmate root) carries a GENUINE secondmate-home marker. +# bin/fm-home-seed.sh writes .fm-secondmate-home at a seeded secondmate home's +# root (gitignored, so it never propagates into a child worktree); its content is +# the secondmate id. Validate the marker's form so a stray/empty/symlink file +# cannot spoof inclusion and an unmarked child is never guarded by accident: it +# must be a regular (non-symlink) file whose first line, with all whitespace +# removed, is a non-empty id token (letters, digits, dot, underscore, dash only). +# The allowlist is matched under forced C (ASCII) collation - `local LC_ALL=C`, +# restored on return - so a locale-crafted non-ASCII id cannot slip through the +# range match and spoof force-inclusion. This is a deliberately lightweight +# guard-local presence check, distinct from fm-ff-lib.sh's validate_secondmate_home +# (which matches an EXPECTED id and does path-safety); the guard does not source +# that heavier library. +fm_root_is_secondmate_home() { + local marker="$1/.fm-secondmate-home" id LC_ALL=C + [ -L "$marker" ] && return 1 + [ -f "$marker" ] || return 1 + IFS= read -r id < "$marker" 2>/dev/null || return 1 + id=${id//[[:space:]]/} + [ -n "$id" ] || return 1 + case "$id" in + *[!A-Za-z0-9._-]*) return 1 ;; + esac + return 0 +} + +# --- scope precisely to a PRIMARY checkout ---------------------------------- +# A genuinely-marked secondmate home runs its OWN primary firstmate session, so +# force-INCLUDE it as a guarded primary whether treehouse leased it as a linked +# worktree (git-dir != git-common-dir) or it is a git-cloned plain checkout. This +# mirrors the cd-guard's intent that a secondmate's own session is a guarded +# primary. Only an UNMARKED checkout (or one with an invalid marker) falls +# through to the linked-worktree exemption: firstmate hands out crewmate/scout +# task worktrees as genuine linked `git worktree`s (bin/fm-spawn.sh aborts +# otherwise), whose git-dir lives under the parent repo's .git/worktrees/ +# and differs from the common (shared) git-dir, while a main, non-worktree +# checkout has the two equal. Child worktrees never carry the gitignored marker, +# so this exempts them while guarding every real secondmate home. +if ! fm_root_is_secondmate_home "$FM_ROOT"; then + GIT_DIR=$(git -C "$FM_ROOT" rev-parse --git-dir 2>/dev/null) || exit 0 + GIT_COMMON_DIR=$(git -C "$FM_ROOT" rev-parse --git-common-dir 2>/dev/null) || exit 0 + [ "$GIT_DIR" = "$GIT_COMMON_DIR" ] || exit 0 +fi [ -f "$FM_ROOT/AGENTS.md" ] || exit 0 [ -d "$FM_ROOT/bin" ] || exit 0 [ -d "$STATE" ] || exit 0 diff --git a/docs/architecture.md b/docs/architecture.md index ab23db09c..cf746f58b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,7 +43,7 @@ A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if The drain script calls that guard after emptying the queue, which avoids repeating the queued-wakes warning for records it just consumed while still warning on stale watcher liveness. It leads with prominent bordered banners for the tangle and no-watcher cases so they cannot be skimmed past. On every verified primary harness, tracked hook integration gives the primary session a push-based backstop: when work is in flight and no identity-matched watcher lock with a fresh beacon is live, direct Stop hooks block and passive turn-end hooks force one bounded follow-up. -The guard is scoped out of secondmate homes and crewmate/scout worktrees, is loop-safe per harness, and is documented in [turnend-guard.md](turnend-guard.md). +The guard covers the main primary and genuinely marked secondmate homes, exempts child crewmate/scout worktrees, is loop-safe per harness, and is documented in [turnend-guard.md](turnend-guard.md). A presence-gated sub-supervisor (`bin/fm-supervise-daemon.sh`) extends this for walk-away supervision: the `/afk` skill starts it through the tracked foreground helper `bin/fm-afk-start.sh`, after which the watcher reverts to daemon-managed one-shot mode and the daemon self-handles routine wakes in bash. The watcher and daemon share `bin/fm-classify-lib.sh` for captain-relevant status verbs, declared-external-wait vocabulary, and status-scan primitives. diff --git a/docs/cd-guard.md b/docs/cd-guard.md index 7485b3c71..e324bfeb4 100644 --- a/docs/cd-guard.md +++ b/docs/cd-guard.md @@ -19,19 +19,19 @@ This guard is not a general sandbox. It classifies shell command positions only; it never evaluates, expands, sources, or runs any byte of the submitted command. Its threat model is agent mistakes, the same as the watcher-arm seatbelt: an accidental bare `cd projects/foo`, not a deliberately obfuscated bypass. -## Scope: the real primary checkout only +## Scope: plain firstmate checkouts only -The guard fires only in the actual primary firstmate checkout. +The guard fires only in a plain firstmate checkout where git-dir equals git-common-dir. It is a silent no-op (exit 0, no output) everywhere else, so it never interferes with a crewmate or scout that legitimately works inside its own project or firstmate task worktree. -`bin/fm-cd-pretool-check.sh` reuses the turn-end guard's primary detection (`docs/turnend-guard.md`). +`bin/fm-cd-pretool-check.sh` owns its checkout detection; the turn-end guard's marker-aware scope is a separate contract (`docs/turnend-guard.md`). A plain, non-worktree checkout has `git rev-parse --git-dir` equal to `git rev-parse --git-common-dir`. A crewmate or scout task worktree - the shape `bin/fm-spawn.sh` always hands out - is a linked git worktree where the two differ, so the guard is inert there. The checkout must also carry `AGENTS.md` and `bin/`, and any failure to confirm the primary is treated as inert, never as a block. -The one deliberate difference from the turn-end guard: the cd-guard does **not** exclude secondmate homes. -A secondmate is a firstmate in its own home, so its own primary session is a primary and its firstmate-owned commands must stay in its home too; the guard applies there. -Only a secondmate's child crew and scout worktrees are exempt, and they are exempt automatically by the same linked-worktree test. +The cd-guard does not inspect `.fm-secondmate-home`. +It therefore applies in a git-cloned secondmate home where git-dir equals git-common-dir, but remains inert in a treehouse-leased secondmate home that is itself a linked worktree. +Secondmate child crew and scout worktrees are likewise inert under the linked-worktree test. ## Block vs allow @@ -125,7 +125,7 @@ Every shell variable reference in the Grok hook command carries an inline defaul `tests/fm-cd-pretool-check.test.sh` owns the acceptance matrix. Every block and allow case runs through Codex-shaped stdin, Claude-shaped stdin, Grok-shaped stdin, OpenCode-shaped CLI, and Pi-shaped CLI entry forms. -The suite also proves the end-to-end cwd-leak regression (a firstmate-owned backlog write leaking into a project clone, then denied at the exact command), the primary-checkout scoping (fires in a secondmate home, inert in a crewmate/scout linked worktree, inert outside a firstmate checkout, inert outside a git repo), the fail-open transport behavior, the prefilter fast path, the policy CLI output contract, and the per-harness wiring. +The suite also proves the end-to-end cwd-leak regression (a firstmate-owned backlog write leaking into a project clone, then denied at the exact command), the checkout scoping (fires in a git-cloned secondmate fixture, inert in a crewmate/scout linked worktree, inert outside a firstmate checkout, inert outside a git repo), the fail-open transport behavior, the prefilter fast path, the policy CLI output contract, and the per-harness wiring. Run: diff --git a/docs/supervision-protocols/opencode.md b/docs/supervision-protocols/opencode.md index 885702944..56cafc4bb 100644 --- a/docs/supervision-protocols/opencode.md +++ b/docs/supervision-protocols/opencode.md @@ -11,4 +11,4 @@ When this session owns supervision and away mode is not active: 7. Do not rely on this plugin in headless `opencode run`; firstmate primary supervision targets persistent OpenCode TUI sessions. OpenCode's persistent TUI plugin runtime is the wake mechanism. -The plugin scopes itself to the primary firstmate checkout and stays silent in crewmate worktrees and secondmate homes. +The plugin applies in the main primary checkout and a secondmate's own home, and stays silent only in child crewmate and scout worktrees. diff --git a/docs/turnend-guard.md b/docs/turnend-guard.md index afc94123a..344417161 100644 --- a/docs/turnend-guard.md +++ b/docs/turnend-guard.md @@ -3,7 +3,8 @@ This is the authoritative contract for the "no turn ends blind" primary guard referenced from AGENTS.md section 8. The shared predicate lives in `bin/fm-turnend-guard.sh`. Harness-specific tracked hook files only adapt each verified harness's real turn-end mechanism to that shared predicate. -Two related but separate PreToolUse seatbelts deny a bad command shape before it runs rather than detecting a blind turn end afterward: the watcher-arm seatbelt (`bin/fm-arm-pretool-check.sh`, `docs/arm-pretool-check.md`) and the cd-guard (`bin/fm-cd-pretool-check.sh`, `docs/cd-guard.md`), which reuses this guard's linked-worktree exemption but deliberately remains active in secondmate homes. +Two related but separate PreToolUse seatbelts deny a bad command shape before it runs rather than detecting a blind turn end afterward: the watcher-arm seatbelt (`bin/fm-arm-pretool-check.sh`, `docs/arm-pretool-check.md`) and the cd-guard (`bin/fm-cd-pretool-check.sh`, `docs/cd-guard.md`). +Each seatbelt's own document defines its scope; they do not share the turn-end guard's marker-aware primary detection. ## Gap Closed @@ -16,9 +17,11 @@ When tasks are in flight and there is no live identity-matched watcher with a fr ## Shared Predicate -The guard first scopes itself to the real primary checkout. -It is inert in secondmate homes because `.fm-secondmate-home` exists there. -It is inert in crewmate and scout worktrees because firstmate provisions them as linked git worktrees, where `git rev-parse --git-dir` differs from `git rev-parse --git-common-dir`. +The guard first scopes itself to a real primary checkout. +A secondmate home runs its own primary firstmate session, so a genuine `.fm-secondmate-home` marker force-includes it whether treehouse leased it as a linked worktree or it is a git-cloned plain checkout. +The marker must be a regular non-symlink file whose first line, after all whitespace is removed, contains a non-empty identifier made only of letters, digits, dots, underscores, and dashes. +An unmarked checkout, or one with an invalid marker, falls through to the git-dir check. +That check keeps crewmate and scout worktrees inert because firstmate provisions them as linked git worktrees, where `git rev-parse --git-dir` differs from `git rev-parse --git-common-dir`. It also requires `AGENTS.md`, `bin/`, and the effective state directory to exist. For an in-scope primary checkout, it counts in-flight work from `state/*.meta`. @@ -112,8 +115,37 @@ If Grok declines to load project hooks, this primary guard fails open and `fm-gu The hook command was fixed to reference `${GROK_WORKSPACE_ROOT:-}` directly everywhere instead of assigning it to `$root` first, and re-validated against grok 0.2.93 to fire and complete cleanly. See `docs/arm-pretool-check.md`'s "Harness wiring" section for the same Grok expansion requirement; that document's Grok hook shares the same fix. +### 2026-07-12: secondmate-home enablement and the autonomous background-notify wake + +The guard originally early-exited in every secondmate home on the `.fm-secondmate-home` marker. +That was a scoping choice inherited from the guard's primary-only origin, not a defense against any secondmate-specific hazard. +A genuinely marked secondmate home is now force-included as a guarded primary regardless of whether it is a treehouse-leased linked worktree or a git-cloned plain checkout. +Only unmarked child worktrees fall through to the linked-worktree exemption, and marker validation prevents an empty, malformed, or symlink marker from spoofing inclusion. + +"No turn ends blind" for a secondmate is delivered by the same two mechanisms the main primary relies on. +Mechanism B, the turn-end backstop, is this guard; its secondmate-home behavior is covered by hermetic tests in `tests/fm-turnend-guard.test.sh` (`test_hook_blocks_in_secondmate_own_home`, `test_hook_blocks_in_treehouse_leased_secondmate_home`, `test_hook_silent_in_idle_secondmate_home`, `test_hook_secondmate_loop_guard_allows_retry`, `test_hook_secondmate_reinvoke_recovery_loop`, `test_hook_silent_in_secondmate_child_worktree`, and `test_hook_exempts_linked_worktree_with_stray_marker`). +Mechanism A, the autonomous wake, is a harness property: when a background watcher task exits, the harness re-invokes the model, which drains the wake, advances children, and re-arms a fresh watcher. +Mechanism A cannot be a hermetic CI assertion because it requires a live model session, so it is recorded here as a dated first-hand measurement while `test_hook_secondmate_reinvoke_recovery_loop` covers the guard's deterministic half of the same recovery loop. + +Autonomous-re-invoke measurement, run first-hand on Claude Code 2.1.207 (Darwin 25.5.0) on 2026-07-12. +Procedure: launch a detached `run_in_background` Bash task that models a one-shot watcher - it records a launch epoch, runs `sleep 25`, then records a completion epoch just before exit, writing only to the session scratchpad - then end the turn with no further tool calls and no pending question, a genuinely idle session with no human input. +Observed marker timestamps: + +``` +launch_epoch = 1783890980 (14:16:20) turn ends, session goes idle +complete_epoch = 1783891005 (14:16:45) background task exits, 25s idle +reinvoke_epoch = 1783891016 (14:16:56) MODEL RE-INVOKED +-------------------------------------------------------------- +wake latency (task complete -> model re-invoked): 11s, with ZERO human input +``` + +The re-invocation arrived as a `` whose accompanying system notice stated verbatim "No human input has been received since the last genuine user message in this conversation". +So the model was re-invoked solely by the background task's completion while idle, which is Mechanism A - the same background-notify wake the Claude supervision protocol relies on for the main primary. +This matches the harness tool contract that a `run_in_background` task "keeps running across turns and re-invokes you when it exits", and reproduces the 11s latency the task audit measured independently on the same harness version. +No Herdr command was issued and no fleet state was touched; the experiment wrote only to the session scratchpad, which was discarded. + ## Tests -`tests/fm-turnend-guard.test.sh` covers the shared predicate, primary scoping, `FM_HOME` and `FM_STATE_OVERRIDE` precedence, Pi logical-run latch behavior for no-tool and multi-tool runs, fail-open behavior without `jq`, tracked hook registration for all five harnesses, and the Grok adapter's forced-resume loop guard and permission-mode regression. +`tests/fm-turnend-guard.test.sh` covers the shared predicate, primary scoping (including a secondmate's own home being guarded like the main primary while its child worktrees stay exempt), `FM_HOME` and `FM_STATE_OVERRIDE` precedence, Pi logical-run latch behavior for no-tool and multi-tool runs, fail-open behavior without `jq`, tracked hook registration for all five harnesses, and the Grok adapter's forced-resume loop guard and permission-mode regression. The default behavior suite does not invoke live language-model harnesses. `FM_PI_LIVE_E2E=1 tests/fm-pi-primary-live-e2e.test.sh` opts into the isolated interactive Pi regression recorded above. diff --git a/tests/fm-backend.test.sh b/tests/fm-backend.test.sh index 5a9fafcde..cd815f547 100755 --- a/tests/fm-backend.test.sh +++ b/tests/fm-backend.test.sh @@ -110,7 +110,7 @@ BASE_REF=$(resolve_base_ref) \ # tmux-only conformance run the tmux adapter's behavior is what is under test, # and that is unchanged by any later (e.g. non-tmux backend) addition to # fm-backend.sh's own dispatch surface. -OLD_BIN_UNCHANGED_SIBLINGS="fm-guard.sh fm-lock-lib.sh fm-tangle-lib.sh fm-tmux-lib.sh fm-composer-lib.sh fm-marker-lib.sh fm-wake-lib.sh fm-classify-lib.sh fm-ff-lib.sh fm-config-inherit-lib.sh fm-tasks-axi-lib.sh fm-project-mode.sh fm-harness.sh fm-crew-state.sh fm-backend.sh" +OLD_BIN_UNCHANGED_SIBLINGS="fm-gate-refuse-lib.sh fm-guard.sh fm-lock-lib.sh fm-tangle-lib.sh fm-tmux-lib.sh fm-composer-lib.sh fm-marker-lib.sh fm-wake-lib.sh fm-classify-lib.sh fm-ff-lib.sh fm-config-inherit-lib.sh fm-tasks-axi-lib.sh fm-project-mode.sh fm-harness.sh fm-crew-state.sh fm-backend.sh" OLD_BIN_REFACTORED="fm-send.sh fm-peek.sh fm-watch.sh fm-spawn.sh fm-teardown.sh" build_old_bin() { # -> echoes root dir (root/bin/