From 9cbb235e0a5b7ecfd3bc442abdb803fb35ab0517 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:44:46 -0700 Subject: [PATCH] Publish release images and a CLI that knows them Nothing published images for this repo: the Makefile pushed to a personal registry, releases carried no assets, and the manifest command's error message pointed at "images published by the latest release" that did not exist. Every user, including the quickstart, had to build and push ate-env-api and ate-env-guest first. A release workflow now runs on every v* tag (and by hand for an existing tag): ko builds both images for linux/amd64 and linux/arm64, pushes them to ghcr.io///ate-env-api and ate-env-guest tagged with the release and latest, signs them with cosign keyless signing bound to the workflow's identity, then builds the ate-env CLI for linux and darwin on amd64 and arm64 with the two digests and the version baked in, uploads the binaries, an images_.txt and checksums as release assets, and writes an Images section with the digests and the verify command into the release notes. The CLI gains build-time defaults: a release binary's `manifest --api-image` and `manifest template --guest-image` default to the images published with it, `ate-env --version` prints the tag, and a source build keeps the flags required with messages that say why. The worker image stays a required flag, since ateom-gvisor comes from the Substrate repo and must match the Substrate version on the cluster; the message says so. `make build-cli` reproduces a release-style binary locally. A ci workflow runs the Go build, vet and tests, the license-header check, and the Python client tests on pull requests and main; the repo had no checks before. The guest's bash base image is pinned by digest in .ko.yaml so a published guest is reproducible. docs/release.md describes the artifacts, pinning, verification, cutting and republishing a release, and the one-time organization setup for GitHub Container Registry; the README quickstart no longer needs image flags with a release binary. --- .github/workflows/ci.yaml | 49 ++++++++++++ .github/workflows/release.yaml | 133 +++++++++++++++++++++++++++++++++ .ko.yaml | 2 +- Makefile | 15 +++- README.md | 36 +++++++-- cmd/ate-env/main.go | 1 + cmd/ate-env/manifest.go | 21 ++++-- cmd/ate-env/manifest_test.go | 36 +++++++++ cmd/ate-env/version.go | 35 +++++++++ docs/release.md | 87 +++++++++++++++++++++ 10 files changed, 401 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 cmd/ate-env/version.go create mode 100644 docs/release.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..bd7219b --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,49 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Checks for pull requests and main: the Go build, vet and tests, the license +# headers, and the Python client's unit tests. Release publishing lives in +# release.yaml. +name: ci + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + go: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - run: go build ./... + - run: go vet ./... + - run: go test ./... + - run: hack/verify/boilerplate.sh + + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install -e 'clients/python[dev]' + - run: python -m pytest clients/python/tests -q diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..9fe0c56 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,133 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Publishes a release: the ate-env-api and ate-env-guest images to GitHub +# Container Registry, pinned by digest and signed, and the ate-env CLI with +# those digests baked in as its defaults, so a release binary deploys without +# naming any image of this repo. Runs on a v* tag, or by hand for an existing +# tag (to republish). See docs/release.md. +name: release + +on: + push: + tags: ["v*"] + workflow_dispatch: + inputs: + tag: + description: "Existing tag to publish images and assets for (e.g. v0.1.0)" + required: true + type: string + +permissions: + contents: write # release assets and notes + packages: write # ghcr.io + id-token: write # cosign keyless signing + +env: + # ghcr.io///; the images are linked to this repository. + IMAGE_PREFIX: ghcr.io/${{ github.repository }} + KO_DEFAULTPLATFORMS: linux/amd64,linux/arm64 + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Resolve the tag + id: tag + run: echo "tag=${{ inputs.tag || github.ref_name }}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + ref: ${{ steps.tag.outputs.tag }} + fetch-depth: 0 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: ko-build/setup-ko@v0.9 + + - uses: sigstore/cosign-installer@v3 + + - name: Build and push the images + id: images + env: + TAG: ${{ steps.tag.outputs.tag }} + run: | + set -euo pipefail + api=$(KO_DOCKER_REPO="${IMAGE_PREFIX}/ate-env-api" ko build --bare --tags "${TAG},latest" ./cmd/ate-env-api) + guest=$(KO_DOCKER_REPO="${IMAGE_PREFIX}/ate-env-guest" ko build --bare --tags "${TAG},latest" ./cmd/ate-env-guest) + echo "api=${api}" >> "$GITHUB_OUTPUT" + echo "guest=${guest}" >> "$GITHUB_OUTPUT" + echo "ate-env-api: ${api}" + echo "ate-env-guest: ${guest}" + + - name: Sign the images (keyless, bound to this workflow's identity) + run: cosign sign --yes "${{ steps.images.outputs.api }}" "${{ steps.images.outputs.guest }}" + + - name: Build the CLI with the image digests as defaults + env: + TAG: ${{ steps.tag.outputs.tag }} + API_IMAGE: ${{ steps.images.outputs.api }} + GUEST_IMAGE: ${{ steps.images.outputs.guest }} + run: | + set -euo pipefail + mkdir -p dist + ldflags="-s -w -X main.version=${TAG} -X main.defaultAPIImage=${API_IMAGE} -X main.defaultGuestImage=${GUEST_IMAGE}" + for os in linux darwin; do + for arch in amd64 arm64; do + CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -trimpath -ldflags "$ldflags" \ + -o "dist/ate-env_${TAG}_${os}_${arch}" ./cmd/ate-env + done + done + { + echo "ate-env-api=${API_IMAGE}" + echo "ate-env-guest=${GUEST_IMAGE}" + } > "dist/images_${TAG}.txt" + (cd dist && sha256sum ./* > "SHA256SUMS_${TAG}.txt") + ls -l dist + + - name: Publish the release assets and record the digests in the notes + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.tag.outputs.tag }} + API_IMAGE: ${{ steps.images.outputs.api }} + GUEST_IMAGE: ${{ steps.images.outputs.guest }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + if ! gh release view "$TAG" >/dev/null 2>&1; then + gh release create "$TAG" --title "$TAG" --generate-notes + fi + gh release upload "$TAG" dist/* --clobber + existing=$(gh release view "$TAG" --json body --jq .body) + # Replace a previous images block (republish) rather than append twice. + existing=$(printf '%s\n' "$existing" | awk '/^## Images$/{skip=1} skip&&/^## /&&!/^## Images$/{skip=0} !skip') + { + printf '%s\n\n' "$existing" + echo "## Images" + echo + echo "Pinned by digest and signed with cosign (keyless, GitHub Actions OIDC):" + echo + echo "- \`${API_IMAGE}\`" + echo "- \`${GUEST_IMAGE}\`" + echo + echo "The \`ate-env\` binaries attached to this release default \`manifest --api-image\` and \`manifest template --guest-image\` to these digests." + echo + echo '```bash' + echo "cosign verify --certificate-oidc-issuer https://token.actions.githubusercontent.com \\" + echo " --certificate-identity-regexp '^https://github.com/${REPO}/' ${GUEST_IMAGE}" + echo '```' + } > notes.md + gh release edit "$TAG" --notes-file notes.md diff --git a/.ko.yaml b/.ko.yaml index 6d523e0..6e06b48 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -13,4 +13,4 @@ # limitations under the License. baseImageOverrides: - github.com/agent-substrate/env/cmd/ate-env-guest: bash:latest + github.com/agent-substrate/env/cmd/ate-env-guest: bash:5.3@sha256:61962062d969cb46dfc2bad061d36342406fa485f64f246aa7e95693ca07df1f diff --git a/Makefile b/Makefile index 67f348d..92c0ca9 100644 --- a/Makefile +++ b/Makefile @@ -15,11 +15,24 @@ GOOGLE_CLOUD_PROJECT ?= $(shell gcloud config get-value project 2>/dev/null) ATE_ENV_IMAGE_REPO ?= gcr.io/$(GOOGLE_CLOUD_PROJECT) -.PHONY: build install test vet clean images python-protos python-test verify-boilerplate +# Baked into the ate-env CLI by build-cli (release.yaml does the same with the +# published digests): `ate-env --version`, and the defaults of manifest +# --api-image and manifest template --guest-image. +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) +ATE_ENV_API_IMAGE ?= +ATE_ENV_GUEST_IMAGE ?= +CLI_LDFLAGS := -X main.version=$(VERSION) -X main.defaultAPIImage=$(ATE_ENV_API_IMAGE) -X main.defaultGuestImage=$(ATE_ENV_GUEST_IMAGE) + +.PHONY: build build-cli install test vet clean images python-protos python-test verify-boilerplate build: go build ./... +# Build bin/ate-env with the version and, if ATE_ENV_API_IMAGE / +# ATE_ENV_GUEST_IMAGE are set, image defaults baked in. +build-cli: + go build -trimpath -ldflags "$(CLI_LDFLAGS)" -o bin/ate-env ./cmd/ate-env + # Install ate-env and ate-env-api to $GOBIN (or $GOPATH/bin). install: go install ./cmd/... diff --git a/README.md b/README.md index 67978ef..1b6f0a4 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,34 @@ while this project adds the environment-shaped API on top. ## Installation +Download the `ate-env` binary for your platform from the +[latest release](https://github.com/agent-substrate/env/releases/latest). Release +binaries know the digests of the `ate-env-api` and `ate-env-guest` images +published with that release, so the commands below need no image flags for +this repo's images. Building from source works too, but then `--api-image` and +`--guest-image` must be given: + ```bash go install github.com/agent-substrate/env/cmd/ate-env@latest ``` +## Images + +Each release publishes two images to GitHub Container Registry, multi-platform, +signed, and meant to be used by digest: + +| image | runs | +|---|---| +| `ghcr.io/agent-substrate/env/ate-env-api` | the API service, in the cluster | +| `ghcr.io/agent-substrate/env/ate-env-guest` | inside every environment actor | + +The digests are in the release notes and in the `images_.txt` asset; the +release's `ate-env` binary defaults to them. The worker image +(`--worker-image`, `ateom-gvisor`) comes from the +[Substrate repo](https://github.com/agent-substrate/substrate) and must match the +Substrate version on your cluster. See [docs/release.md](docs/release.md) for +verification, pinning and how releases are cut. + ## Quickstart Prerequisites: a cluster with [Agent Substrate](https://github.com/agent-substrate/substrate) @@ -46,10 +70,11 @@ installed and a snapshots bucket. Deploy the namespace, worker pool, and API service: ```bash -export GOOGLE_CLOUD_PROJECT=$(gcloud config get-value project) +# With a release binary the API image defaults to the release's; the worker +# image is ateom-gvisor built from the Substrate repo at your cluster's version. ate-env manifest \ - --api-image gcr.io/$GOOGLE_CLOUD_PROJECT/ate-env-api@sha256:0952ad3fa121597c5ff2943b701f6f0968ba51fdd93b0985d1e831d7cad804a4 \ - --worker-image gcr.io/$GOOGLE_CLOUD_PROJECT/ateom-gvisor-715889664656de67e44382a8d6ab981d@sha256:0e69688125a167ffd62ab084a9ab1a50e3f06e9107b36dcb01c3fb3ac0b23fcb | kubectl apply -f - + --worker-image /ateom-gvisor@sha256: | kubectl apply -f - +# From a source build, add: --api-image ghcr.io/agent-substrate/env/ate-env-api@sha256: # Ensure that the pods are running: kubectl get pods -n ate-env @@ -60,9 +85,10 @@ kubectl get pods -n ate-env Substrate manages ActorTemplates directly in its control plane rather than Kubernetes CRDs. Use `ate-env manifest template` to generate the Substrate ActorTemplate manifest: ```bash +# With a release binary the guest image defaults to the release's. ate-env manifest template \ - --guest-image gcr.io/$GOOGLE_CLOUD_PROJECT/ate-env-guest@sha256:47f18ee80fbdc4aa86ca7bccb78c37add6314ca278b38b88641eb49757921b73 \ - --snapshots-bucket gs://$GOOGLE_CLOUD_PROJECT/ate-env/ | kubectl-ate create actor-template -f - + --snapshots-bucket | kubectl-ate create actor-template -f - +# From a source build, add: --guest-image ghcr.io/agent-substrate/env/ate-env-guest@sha256: ``` Then create and use an environment: diff --git a/cmd/ate-env/main.go b/cmd/ate-env/main.go index a7a6502..2870c4f 100644 --- a/cmd/ate-env/main.go +++ b/cmd/ate-env/main.go @@ -224,6 +224,7 @@ Common environment commands: ate-env shell Run a shell command line in the environment`, SilenceUsage: true, SilenceErrors: true, + Version: version, } root.AddCommand(newManifestCommand()) diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index a83a04a..1921d6e 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -49,10 +49,15 @@ type manifestConfig struct { } func (c *manifestConfig) resolveImages() error { - if c.workerImage == "" || c.apiImage == "" { - return errors.New(`--api-image and --worker-image (or --ateom-image) are required; use the -digest-pinned images published by the latest release (the README -quickstart records them), or build and push your own.`) + if c.workerImage == "" { + return errors.New(`--worker-image (or --ateom-image) is required: the worker image is +ateom-gvisor from the Substrate repo and must match the Substrate version +deployed on your cluster; build it there with ko (see the README).`) + } + if c.apiImage == "" { + return errors.New(`--api-image is required when ate-env is built from source; release +binaries default it to the ate-env-api image published with the release +(see docs/release.md), or build and push your own with make images.`) } return nil } @@ -68,7 +73,9 @@ type templateConfig struct { func (c *templateConfig) resolveImages() error { if c.guestImage == "" { - return errors.New("--guest-image is required; use the digest-pinned ate-env-guest image") + return errors.New(`--guest-image is required when ate-env is built from source; release +binaries default it to the ate-env-guest image published with the release +(see docs/release.md), or build and push your own with make images.`) } if c.snapshotsBucket == "" { return errors.New("--snapshots-bucket is required; use an object-storage bucket (e.g. gs://bucket/prefix/)") @@ -108,7 +115,7 @@ the "template" subcommand: ate-env manifest template`, cmd.Flags().StringVar(&mCfg.template, "template", apiservice.DefaultTemplate, "ActorTemplate name") cmd.Flags().StringVar(&mCfg.workerImage, "worker-image", "", "digest-pinned worker image for the worker pool, e.g. ateom-gvisor built from the Substrate repo") cmd.Flags().StringVar(&mCfg.workerImage, "ateom-image", "", "alias for --worker-image") - cmd.Flags().StringVar(&mCfg.apiImage, "api-image", "", "digest-pinned ate-env-api image for the API service") + cmd.Flags().StringVar(&mCfg.apiImage, "api-image", defaultAPIImage, imageDefaultHelp("digest-pinned ate-env-api image for the API service", defaultAPIImage)) cmd.Flags().Int32Var(&mCfg.apiReplicas, "api-replicas", 1, "number of API service replicas") cmd.Flags().Int32Var(&mCfg.apiPort, "api-port", 7777, "port the ate-env-api service listens on") cmd.Flags().StringVar(&mCfg.workerPool, "workerpool", "", "WorkerPool name (defaults to