From 3f943a867b0ff00c49beec959b6da55bf78e6541 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:56:16 -0700 Subject: [PATCH 1/2] Run unmodified task images: guest injected as an OCI image volume Until now a template's container image had to be the ate-env-guest image itself, so running a task image meant rebaking it with the guest as the entrypoint, once per image. Substrate's image volumes allow the other split: the task image stays unmodified and the guest is mounted into it read-only and started from the mount. Two ways to get such a template: - ate-env manifest template --task-image writes a template whose container is the task image, with the guest image as a volume at /ate and the command re-rooted to /ate/ko-app/ate-env-guest. --workspace passes -workspace to the guest in either mode. - CreateEnvironment gains an image field. ate-env-api takes the request's template (default-template by default) as the base, derives "-<12 hex digits of the digest>" from it on first use and reuses it for later environments on the same image; the response reports it. A derived name already taken by a template that runs something else is a FailedPrecondition, as is a missing base; an unpinned image, a derived name over 63 characters, or a base command that is not an absolute path to the guest is an InvalidArgument. Concurrent creates racing on one derived template both succeed. The derivation lives in internal/apiservice/imagetemplate.go and accepts either kind of base, guest-as-image or already injected, so a second runtime can reuse it. internal/ate gains Get/CreateActorTemplate and the fake control plane stores templates. The CLI gains create --image, the Python client create(image=), and the full-stack e2e suite a create-from-image test gated on ATE_ENV_TASK_IMAGE that checks the guest comes from the volume, the rootfs is the image's, and a second environment reuses the derived template. docs/task-images/README.md is the how-to (both flows, what an image needs, errors, cleanup) and docs/task-images/DESIGN.md the design (derivation rules, naming and idempotency, failure semantics, security, alternatives, future work). Stubs regenerated: Go with buf and protoc-gen-go v1.36.11, Python with grpcio-tools 1.61.3, keeping the committed license headers. --- README.md | 22 +- clients/python/README.md | 16 +- .../ate_env/_gen/ateenv/v1alpha/env_pb2.py | 40 ++-- .../ate_env/_gen/ateenv/v1alpha/env_pb2.pyi | 6 +- clients/python/src/ate_env/client.py | 10 +- clients/python/tests/e2e/test_full_stack.py | 87 ++++++++ clients/python/tests/fakes.py | 7 + clients/python/tests/test_client.py | 24 +++ cmd/ate-env/main.go | 3 + cmd/ate-env/main_test.go | 32 +++ cmd/ate-env/manifest.go | 37 +++- cmd/ate-env/manifest_test.go | 64 ++++++ docs/task-images/DESIGN.md | 195 ++++++++++++++++++ docs/task-images/README.md | 137 ++++++++++++ internal/apiservice/imagetemplate.go | 164 +++++++++++++++ internal/apiservice/imagetemplate_test.go | 172 +++++++++++++++ internal/apiservice/server.go | 53 +++++ internal/apiservice/server_test.go | 128 ++++++++++++ internal/ate/client.go | 36 ++++ .../internaltest/fakecontrol/fakecontrol.go | 69 ++++++- proto/ateenv/v1alpha/env.pb.go | 24 ++- proto/ateenv/v1alpha/env.proto | 10 + 22 files changed, 1306 insertions(+), 30 deletions(-) create mode 100644 docs/task-images/DESIGN.md create mode 100644 docs/task-images/README.md create mode 100644 internal/apiservice/imagetemplate.go create mode 100644 internal/apiservice/imagetemplate_test.go diff --git a/README.md b/README.md index 67978ef..03e5745 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,21 @@ ate-env manifest template \ --snapshots-bucket gs://$GOOGLE_CLOUD_PROJECT/ate-env/ | kubectl-ate create actor-template -f - ``` +That template's container is the guest image itself. To run another image unmodified, name +it with `--task-image`: the guest is then mounted into it as a read-only OCI image volume at +`/ate` and started from there, so the task image is never rebuilt. + +```bash +ate-env manifest template --template py312 \ + --task-image docker.io/library/python@sha256: \ + --guest-image /ate-env-guest@sha256: \ + --snapshots-bucket | kubectl-ate create actor-template -f - +``` + +See [docs/task-images/README.md](docs/task-images/README.md) for the full guide, including +creating environments from an image on demand, and [docs/task-images/DESIGN.md](docs/task-images/DESIGN.md) +for the design. + Then create and use an environment: ```bash @@ -74,6 +89,11 @@ kubectl port-forward -n ate-env svc/ate-env-api 7777:7777 & # Create an environment. ate-env create dev1 +# Or run any digest-pinned image unmodified. ate-env-api derives a template +# from default-template on first use (guest mounted in as an image volume) +# and reuses it for later environments on the same image. +ate-env create py1 --image docker.io/library/python@sha256: + # Execute a shell command inside the environment. ate-env dev1 shell 'echo hello > /note.txt' @@ -152,7 +172,7 @@ Manages the lifecycle of isolated execution environments (defined in [`proto/ate | RPC | Description | | --- | ----------- | -| `CreateEnvironment` | Creates and starts a new environment actor from an ActorTemplate | +| `CreateEnvironment` | Creates and starts a new environment actor from an ActorTemplate, or from a digest-pinned `image` on top of one: the template becomes the base, the image the container, and the guest is mounted in as a read-only image volume | | `GetEnvironment` | Retrieves environment details and status | | `SuspendEnvironment` | Suspends and checkpoints the environment to snapshot storage | | `DeleteEnvironment` | Deletes the environment permanently | diff --git a/clients/python/README.md b/clients/python/README.md index bdaea8f..c4ab9a8 100644 --- a/clients/python/README.md +++ b/clients/python/README.md @@ -141,6 +141,17 @@ env = await client.create("dev1", template_name="my-template", template_atespace="my-atespace") ``` +To run an arbitrary OCI image, pass it pinned by digest. The server derives +an ActorTemplate from the template above, which acts as the base: the image +becomes the container and the `ate-env-guest` is mounted into it as a +read-only image volume, so the image runs unmodified. The derived template +is created on first use and shared by every environment on that image: + +```python +env = await client.create("py1", image="docker.io/library/python@sha256:…") +print((await env.info()).template.name) # default-template-<12 hex of the digest> +``` + To get a handle to an environment that already exists (no RPC is made): ```python @@ -336,4 +347,7 @@ ATE_ENV_API_TARGET=127.0.0.1:17777 .venv/bin/pytest tests/e2e/test_full_stack.py `ATE_ENV_TEMPLATE` optionally overrides the ActorTemplate used for the test environment; `ATE_ENV_READY_TIMEOUT` (default 180s) bounds the wait -for the environment to start serving. +for the environment to start serving. `ATE_ENV_TASK_IMAGE`, a digest-pinned +image that has `sh`, enables the create-from-image test, which runs that +image unmodified with the guest injected and checks the derived template is +reused by a second environment. diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py index 36748dd..03b1143 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.py @@ -28,7 +28,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x61teenv/v1alpha/env.proto\x12\x0e\x61teenv.v1alpha\"*\n\x08Template\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x8a\x01\n\x0b\x45nvironment\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\x31\n\x06status\x18\x04 \x01(\x0e\x32!.ateenv.v1alpha.EnvironmentStatus\"d\n\x18\x43reateEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\"M\n\x19\x43reateEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"5\n\x15GetEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"J\n\x16GetEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"9\n\x19SuspendEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1c\n\x1aSuspendEnvironmentResponse\"8\n\x18\x44\x65leteEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1b\n\x19\x44\x65leteEnvironmentResponse*\xbd\x02\n\x11\x45nvironmentStatus\x12\"\n\x1e\x45NVIRONMENT_STATUS_UNSPECIFIED\x10\x00\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_RESUMING\x10\x01\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_RUNNING\x10\x02\x12!\n\x1d\x45NVIRONMENT_STATUS_SUSPENDING\x10\x03\x12 \n\x1c\x45NVIRONMENT_STATUS_SUSPENDED\x10\x04\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_PAUSING\x10\x05\x12\x1d\n\x19\x45NVIRONMENT_STATUS_PAUSED\x10\x06\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_CRASHED\x10\x07\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_DELETING\x10\x08\x32\xb6\x03\n\x12\x45nvironmentService\x12h\n\x11\x43reateEnvironment\x12(.ateenv.v1alpha.CreateEnvironmentRequest\x1a).ateenv.v1alpha.CreateEnvironmentResponse\x12_\n\x0eGetEnvironment\x12%.ateenv.v1alpha.GetEnvironmentRequest\x1a&.ateenv.v1alpha.GetEnvironmentResponse\x12k\n\x12SuspendEnvironment\x12).ateenv.v1alpha.SuspendEnvironmentRequest\x1a*.ateenv.v1alpha.SuspendEnvironmentResponse\x12h\n\x11\x44\x65leteEnvironment\x12(.ateenv.v1alpha.DeleteEnvironmentRequest\x1a).ateenv.v1alpha.DeleteEnvironmentResponseBCZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alphab\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x61teenv/v1alpha/env.proto\x12\x0e\x61teenv.v1alpha\"*\n\x08Template\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x8a\x01\n\x0b\x45nvironment\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\x31\n\x06status\x18\x04 \x01(\x0e\x32!.ateenv.v1alpha.EnvironmentStatus\"s\n\x18\x43reateEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12*\n\x08template\x18\x03 \x01(\x0b\x32\x18.ateenv.v1alpha.Template\x12\r\n\x05image\x18\x04 \x01(\t\"M\n\x19\x43reateEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"5\n\x15GetEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"J\n\x16GetEnvironmentResponse\x12\x30\n\x0b\x65nvironment\x18\x01 \x01(\x0b\x32\x1b.ateenv.v1alpha.Environment\"9\n\x19SuspendEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1c\n\x1aSuspendEnvironmentResponse\"8\n\x18\x44\x65leteEnvironmentRequest\x12\n\n\x02id\x18\x01 \x01(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\"\x1b\n\x19\x44\x65leteEnvironmentResponse*\xbd\x02\n\x11\x45nvironmentStatus\x12\"\n\x1e\x45NVIRONMENT_STATUS_UNSPECIFIED\x10\x00\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_RESUMING\x10\x01\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_RUNNING\x10\x02\x12!\n\x1d\x45NVIRONMENT_STATUS_SUSPENDING\x10\x03\x12 \n\x1c\x45NVIRONMENT_STATUS_SUSPENDED\x10\x04\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_PAUSING\x10\x05\x12\x1d\n\x19\x45NVIRONMENT_STATUS_PAUSED\x10\x06\x12\x1e\n\x1a\x45NVIRONMENT_STATUS_CRASHED\x10\x07\x12\x1f\n\x1b\x45NVIRONMENT_STATUS_DELETING\x10\x08\x32\xb6\x03\n\x12\x45nvironmentService\x12h\n\x11\x43reateEnvironment\x12(.ateenv.v1alpha.CreateEnvironmentRequest\x1a).ateenv.v1alpha.CreateEnvironmentResponse\x12_\n\x0eGetEnvironment\x12%.ateenv.v1alpha.GetEnvironmentRequest\x1a&.ateenv.v1alpha.GetEnvironmentResponse\x12k\n\x12SuspendEnvironment\x12).ateenv.v1alpha.SuspendEnvironmentRequest\x1a*.ateenv.v1alpha.SuspendEnvironmentResponse\x12h\n\x11\x44\x65leteEnvironment\x12(.ateenv.v1alpha.DeleteEnvironmentRequest\x1a).ateenv.v1alpha.DeleteEnvironmentResponseBCZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alphab\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -36,28 +36,28 @@ if _descriptor._USE_C_DESCRIPTORS == False: _globals['DESCRIPTOR']._options = None _globals['DESCRIPTOR']._serialized_options = b'ZAgithub.com/agent-substrate/env/proto/ateenv/v1alpha;ateenvv1alpha' - _globals['_ENVIRONMENTSTATUS']._serialized_start=718 - _globals['_ENVIRONMENTSTATUS']._serialized_end=1035 + _globals['_ENVIRONMENTSTATUS']._serialized_start=733 + _globals['_ENVIRONMENTSTATUS']._serialized_end=1050 _globals['_TEMPLATE']._serialized_start=44 _globals['_TEMPLATE']._serialized_end=86 _globals['_ENVIRONMENT']._serialized_start=89 _globals['_ENVIRONMENT']._serialized_end=227 _globals['_CREATEENVIRONMENTREQUEST']._serialized_start=229 - _globals['_CREATEENVIRONMENTREQUEST']._serialized_end=329 - _globals['_CREATEENVIRONMENTRESPONSE']._serialized_start=331 - _globals['_CREATEENVIRONMENTRESPONSE']._serialized_end=408 - _globals['_GETENVIRONMENTREQUEST']._serialized_start=410 - _globals['_GETENVIRONMENTREQUEST']._serialized_end=463 - _globals['_GETENVIRONMENTRESPONSE']._serialized_start=465 - _globals['_GETENVIRONMENTRESPONSE']._serialized_end=539 - _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_start=541 - _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_end=598 - _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_start=600 - _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_end=628 - _globals['_DELETEENVIRONMENTREQUEST']._serialized_start=630 - _globals['_DELETEENVIRONMENTREQUEST']._serialized_end=686 - _globals['_DELETEENVIRONMENTRESPONSE']._serialized_start=688 - _globals['_DELETEENVIRONMENTRESPONSE']._serialized_end=715 - _globals['_ENVIRONMENTSERVICE']._serialized_start=1038 - _globals['_ENVIRONMENTSERVICE']._serialized_end=1476 + _globals['_CREATEENVIRONMENTREQUEST']._serialized_end=344 + _globals['_CREATEENVIRONMENTRESPONSE']._serialized_start=346 + _globals['_CREATEENVIRONMENTRESPONSE']._serialized_end=423 + _globals['_GETENVIRONMENTREQUEST']._serialized_start=425 + _globals['_GETENVIRONMENTREQUEST']._serialized_end=478 + _globals['_GETENVIRONMENTRESPONSE']._serialized_start=480 + _globals['_GETENVIRONMENTRESPONSE']._serialized_end=554 + _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_start=556 + _globals['_SUSPENDENVIRONMENTREQUEST']._serialized_end=613 + _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_start=615 + _globals['_SUSPENDENVIRONMENTRESPONSE']._serialized_end=643 + _globals['_DELETEENVIRONMENTREQUEST']._serialized_start=645 + _globals['_DELETEENVIRONMENTREQUEST']._serialized_end=701 + _globals['_DELETEENVIRONMENTRESPONSE']._serialized_start=703 + _globals['_DELETEENVIRONMENTRESPONSE']._serialized_end=730 + _globals['_ENVIRONMENTSERVICE']._serialized_start=1053 + _globals['_ENVIRONMENTSERVICE']._serialized_end=1491 # @@protoc_insertion_point(module_scope) diff --git a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi index 103cc7a..9181b38 100644 --- a/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi +++ b/clients/python/src/ate_env/_gen/ateenv/v1alpha/env_pb2.pyi @@ -61,14 +61,16 @@ class Environment(_message.Message): def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ..., status: _Optional[_Union[EnvironmentStatus, str]] = ...) -> None: ... class CreateEnvironmentRequest(_message.Message): - __slots__ = ("id", "atespace", "template") + __slots__ = ("id", "atespace", "template", "image") ID_FIELD_NUMBER: _ClassVar[int] ATESPACE_FIELD_NUMBER: _ClassVar[int] TEMPLATE_FIELD_NUMBER: _ClassVar[int] + IMAGE_FIELD_NUMBER: _ClassVar[int] id: str atespace: str template: Template - def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ...) -> None: ... + image: str + def __init__(self, id: _Optional[str] = ..., atespace: _Optional[str] = ..., template: _Optional[_Union[Template, _Mapping]] = ..., image: _Optional[str] = ...) -> None: ... class CreateEnvironmentResponse(_message.Message): __slots__ = ("environment",) diff --git a/clients/python/src/ate_env/client.py b/clients/python/src/ate_env/client.py index ee4c6b4..4804447 100644 --- a/clients/python/src/ate_env/client.py +++ b/clients/python/src/ate_env/client.py @@ -88,13 +88,21 @@ async def create( atespace: str = DEFAULT_ATESPACE, template_name: str | None = None, template_atespace: str | None = None, + image: str | None = None, ) -> Env: """Register and start a new environment; returns a handle to it. The server fills defaults for the template (name "default-template" in atespace "ate-env") when none is given. + + With image (a digest-pinned OCI reference, repo@sha256:...), the + environment runs that image unmodified: the server derives an + ActorTemplate from the template, which then acts as the base, with the + image as the container and the ate-env-guest mounted into it as an + image volume. The derived template is created on first use and reused + for later environments on the same image; info() reports its name. """ - req = env_pb2.CreateEnvironmentRequest(id=id, atespace=atespace) + req = env_pb2.CreateEnvironmentRequest(id=id, atespace=atespace, image=image or "") if template_name or template_atespace: req.template.name = template_name or "" req.template.atespace = template_atespace or "" diff --git a/clients/python/tests/e2e/test_full_stack.py b/clients/python/tests/e2e/test_full_stack.py index 7d897d2..cbeb120 100644 --- a/clients/python/tests/e2e/test_full_stack.py +++ b/clients/python/tests/e2e/test_full_stack.py @@ -25,6 +25,11 @@ Unlike test_guest_daemon.py, this exercises the whole path: environment lifecycle against the Substrate control plane, and guest operations proxied by ate-env-api through the atenet router into the environment's actor. + +Set ATE_ENV_TASK_IMAGE to a digest-pinned image that has `sh` (for example +docker.io/library/python@sha256:...) to also run the create-from-image test, +which checks that an unmodified image runs with the guest injected as an +image volume and that a second environment reuses the derived template. """ from __future__ import annotations @@ -36,19 +41,25 @@ import pytest +import grpc + from ate_env import ( Client, EnvError, EnvironmentStatus, + InvalidArgumentError, OutputSource, NotFoundError, ProcessStatus, + RpcError, ) TARGET = os.environ.get("ATE_ENV_API_TARGET") READY_TIMEOUT = float(os.environ.get("ATE_ENV_READY_TIMEOUT", "180")) # Optional ActorTemplate override; the server default is "default-template". TEMPLATE = os.environ.get("ATE_ENV_TEMPLATE") +# Optional digest-pinned task image for the create-from-image test. +TASK_IMAGE = os.environ.get("ATE_ENV_TASK_IMAGE") pytestmark = pytest.mark.skipif( not TARGET, @@ -164,3 +175,79 @@ async def test_full_lifecycle(client): return await asyncio.sleep(2) pytest.fail(f"environment {env_id} still exists after delete") + + +async def test_create_from_image_rejects_unpinned_image(client): + # Validated before anything touches the control plane. + with pytest.raises(InvalidArgumentError): + await client.create(f"pye2e-img-{uuid.uuid4().hex[:8]}", image="python:3.12-slim") + + +async def test_create_from_image_needs_a_base_template(client): + missing = f"pye2e-nobase-{uuid.uuid4().hex[:8]}" + with pytest.raises(RpcError) as excinfo: + await client.create( + f"pye2e-img-{uuid.uuid4().hex[:8]}", + template_name=missing, + image="docker.io/library/busybox@sha256:" + "0" * 64, + ) + assert excinfo.value.code == grpc.StatusCode.FAILED_PRECONDITION + assert missing in str(excinfo.value) + + +@pytest.mark.skipif( + not TASK_IMAGE, + reason="set ATE_ENV_TASK_IMAGE=repo@sha256:... (an image with sh) for the create-from-image test", +) +async def test_create_from_image(client): + base = TEMPLATE or "default-template" + want_template = f"{base}-{TASK_IMAGE.split('@sha256:', 1)[1][:12]}" + envs = [] + try: + env = await client.create( + f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=TEMPLATE, image=TASK_IMAGE + ) + envs.append(env) + + # The derived template is reported at once, before the actor serves. + info = await env.info() + assert info.template is not None + assert info.template.name == want_template + assert info.template.atespace == env.atespace + + await _wait_until_serving(env) + + # The process runs in the task image's rootfs, with the guest coming + # from the image volume rather than from the image itself. + result = await env.shell( + "test -x /ate/ko-app/ate-env-guest && echo guest:volume; " + "test -e /ko-app/ate-env-guest && echo guest:baked; " + "test -r /etc/os-release && echo rootfs:ok" + ) + assert result.exit_code == 0, result + lines = result.stdout.splitlines() + assert "guest:volume" in lines, result + assert "guest:baked" not in lines, "the task image should not carry the guest" + assert "rootfs:ok" in lines, result + + # The workspace is writable and the guest file path works unchanged. + content = os.urandom(64 * 1024 + 1) + path = f"/tmp/pye2e-img-{uuid.uuid4().hex[:8]}.bin" + assert await env.write_file(path, content) == len(content) + assert await env.read_file_bytes(path) == content + + # A second environment on the same image reuses the derived template + # instead of minting another one. + env2 = await client.create( + f"pye2e-img-{uuid.uuid4().hex[:8]}", template_name=TEMPLATE, image=TASK_IMAGE + ) + envs.append(env2) + assert (await env2.info()).template.name == want_template + await _wait_until_serving(env2) + assert (await env2.shell("echo second")).stdout == "second\n" + finally: + for e in envs: + try: + await e.delete() + except EnvError as exc: + pytest.fail(f"cleanup delete of {e.id} failed: {exc}") diff --git a/clients/python/tests/fakes.py b/clients/python/tests/fakes.py index 0a04acd..3e16cd5 100644 --- a/clients/python/tests/fakes.py +++ b/clients/python/tests/fakes.py @@ -61,6 +61,13 @@ async def CreateEnvironment(self, request, context): template.name = request.template.name if request.template.atespace: template.atespace = request.template.atespace + if request.image: + # Mirror ate-env-api: the template becomes the base of one derived + # per image, named after the digest. + if "@sha256:" not in request.image: + await context.abort(grpc.StatusCode.INVALID_ARGUMENT, "image is not pinned by digest") + digest = request.image.split("@sha256:", 1)[1] + template.name = f"{template.name}-{digest[:12]}" environment = env_pb2.Environment( id=request.id, atespace=atespace, diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 15ed523..3879f20 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -49,6 +49,30 @@ async def test_create_omits_template_when_not_given(fake_stack): assert not fakes.environments.last_create_request.HasField("template") +async def test_create_with_image_reports_derived_template(fake_stack): + client, fakes = fake_stack + image = "docker.io/library/python@sha256:" + "0123456789abcdef" * 4 + env = await client.create("dev3", image=image) + assert fakes.environments.last_create_request.image == image + info = await env.info() + assert info.template is not None + assert info.template.name == "default-template-0123456789ab" + + +async def test_create_with_image_and_base_template(fake_stack): + client, fakes = fake_stack + image = "docker.io/library/python@sha256:" + "0123456789abcdef" * 4 + env = await client.create("dev4", template_name="py-base", image=image) + info = await env.info() + assert info.template.name == "py-base-0123456789ab" + + +async def test_create_omits_image_when_not_given(fake_stack): + client, fakes = fake_stack + await client.create("dev5") + assert fakes.environments.last_create_request.image == "" + + async def test_create_duplicate_maps_to_rpc_error_with_code(fake_stack): client, _ = fake_stack await client.create("dev1") diff --git a/cmd/ate-env/main.go b/cmd/ate-env/main.go index a7a6502..1d47184 100644 --- a/cmd/ate-env/main.go +++ b/cmd/ate-env/main.go @@ -121,6 +121,7 @@ func newCreateCommand() *cobra.Command { atespace string createTemplate string createTemplateAtespace string + image string ) cmd := &cobra.Command{ Use: "create ", @@ -138,6 +139,7 @@ func newCreateCommand() *cobra.Command { req := &ateenvv1alpha.CreateEnvironmentRequest{ Id: args[0], Atespace: atespace, + Image: image, } if createTemplate != "" || createTemplateAtespace != "" { tmplAtespace := createTemplateAtespace @@ -157,6 +159,7 @@ func newCreateCommand() *cobra.Command { cmd.Flags().StringVar(&atespace, "atespace", apiservice.DefaultAtespace, "Substrate atespace") cmd.Flags().StringVar(&createTemplate, "template", "", "ActorTemplate name (defaults to server default)") cmd.Flags().StringVar(&createTemplateAtespace, "template-atespace", "", "Substrate atespace of the ActorTemplate (defaults to environment atespace)") + cmd.Flags().StringVar(&image, "image", "", "digest-pinned OCI image to run unmodified (repo@sha256:...); the template becomes the base the guest is taken from") return cmd } diff --git a/cmd/ate-env/main_test.go b/cmd/ate-env/main_test.go index ee995ca..911005a 100644 --- a/cmd/ate-env/main_test.go +++ b/cmd/ate-env/main_test.go @@ -202,3 +202,35 @@ func TestHelpOutput(t *testing.T) { } }) } + +func TestCreateImageFlag(t *testing.T) { + args := []string{"create", "dev1", "--image", "example.com/task@sha256:abc"} + root := newRootCommand(args) + cmd, _, err := root.Find(args[:2]) + if err != nil { + t.Fatalf("root.Find failed: %v", err) + } + if cmd.Flags().Lookup("image") == nil { + t.Fatal("create has no --image flag") + } + if err := cmd.ParseFlags(args[2:]); err != nil { + t.Fatalf("parsing --image: %v", err) + } + if got, _ := cmd.Flags().GetString("image"); got != "example.com/task@sha256:abc" { + t.Errorf("--image = %q", got) + } +} + +func TestManifestTemplateTaskImageFlags(t *testing.T) { + args := []string{"manifest", "template"} + root := newRootCommand(args) + cmd, _, err := root.Find(args) + if err != nil { + t.Fatalf("root.Find failed: %v", err) + } + for _, name := range []string{"task-image", "workspace", "guest-image", "snapshots-bucket"} { + if cmd.Flags().Lookup(name) == nil { + t.Errorf("manifest template has no --%s flag", name) + } + } +} diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index a83a04a..b906013 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -64,6 +64,12 @@ type templateConfig struct { guestImage string guestCommand []string snapshotsBucket string + // taskImage, when set, is the digest-pinned image the actor runs; the + // guest is then mounted into it as an image volume instead of being the + // container image itself. + taskImage string + // workspace, when set, is passed to the guest as -workspace. + workspace string } func (c *templateConfig) resolveImages() error { @@ -73,6 +79,14 @@ func (c *templateConfig) resolveImages() error { if c.snapshotsBucket == "" { return errors.New("--snapshots-bucket is required; use an object-storage bucket (e.g. gs://bucket/prefix/)") } + if c.taskImage != "" { + if _, err := apiservice.ImageDigest(c.taskImage); err != nil { + return fmt.Errorf("--task-image: %w", err) + } + if _, err := apiservice.ImageDigest(c.guestImage); err != nil { + return fmt.Errorf("--guest-image must be digest-pinned to be mounted as an image volume: %w", err) + } + } return nil } @@ -135,7 +149,11 @@ It prints YAML to stdout without touching the cluster.`, if err := tCfg.resolveImages(); err != nil { return err } - return writeActorTemplate(cmd.OutOrStdout(), buildActorTemplate(tCfg)) + tmpl, err := buildTemplate(tCfg) + if err != nil { + return err + } + return writeActorTemplate(cmd.OutOrStdout(), tmpl) }, } @@ -144,6 +162,8 @@ It prints YAML to stdout without touching the cluster.`, cmd.Flags().StringVar(&tCfg.guestImage, "guest-image", "", "digest-pinned ate-env-guest image (repo@sha256:...)") cmd.Flags().StringSliceVar(&tCfg.guestCommand, "guest-command", []string{"/ko-app/ate-env-guest"}, "guest container entrypoint") cmd.Flags().StringVar(&tCfg.snapshotsBucket, "snapshots-bucket", "", "object-storage bucket (with optional prefix) for actor snapshots, e.g. gs://bucket/prefix/") + cmd.Flags().StringVar(&tCfg.taskImage, "task-image", "", "digest-pinned image to run unmodified; the guest is mounted into it as a read-only image volume at "+apiservice.GuestMountPath) + cmd.Flags().StringVar(&tCfg.workspace, "workspace", "", "workspace root inside the actor, passed to the guest as -workspace (default: the guest's default)") return cmd } @@ -305,6 +325,21 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate { } } +// buildTemplate returns the ActorTemplate for cfg: the guest template from +// buildActorTemplate with the workspace flag applied and, when a task image is +// set, that image as the container with the guest mounted into it. +func buildTemplate(cfg templateConfig) (*ateapipb.ActorTemplate, error) { + tmpl := buildActorTemplate(cfg) + if cfg.workspace != "" { + c := tmpl.Containers[0] + c.Command = append(append([]string(nil), c.Command...), "-workspace", cfg.workspace) + } + if cfg.taskImage == "" { + return tmpl, nil + } + return apiservice.DeriveImageTemplate(tmpl, cfg.template, cfg.taskImage) +} + // buildAPIDeployment returns the ate-env-api Deployment, pointed at the // in-cluster Substrate endpoints. func buildAPIDeployment(cfg manifestConfig) *appsv1.Deployment { diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go index 42a5426..cca2e8b 100644 --- a/cmd/ate-env/manifest_test.go +++ b/cmd/ate-env/manifest_test.go @@ -19,6 +19,7 @@ import ( "strings" "testing" + "github.com/agent-substrate/env/internal/apiservice" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" appsv1 "k8s.io/api/apps/v1" @@ -265,3 +266,66 @@ func TestManifestTemplateCommandExecution(t *testing.T) { t.Errorf("expected guest image and storage location in output:\n%s", out) } } + +const testHex64 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestBuildTemplateTaskImage(t *testing.T) { + cfg := testTemplateConfig() + cfg.guestImage = "example.com/guest@sha256:" + testHex64 + cfg.taskImage = "docker.io/library/python@sha256:" + testHex64 + cfg.workspace = "/workspace" + + tmpl, err := buildTemplate(cfg) + if err != nil { + t.Fatalf("buildTemplate: %v", err) + } + if tmpl.GetMetadata().GetName() != "default-template" { + t.Errorf("name = %q, want the configured template name unchanged", tmpl.GetMetadata().GetName()) + } + c := tmpl.GetContainers()[0] + if c.GetImage() != cfg.taskImage { + t.Errorf("container image = %q, want the task image", c.GetImage()) + } + if len(tmpl.GetVolumes()) != 1 || tmpl.GetVolumes()[0].GetImage().GetReference() != cfg.guestImage { + t.Errorf("volumes = %v, want the guest image volume", tmpl.GetVolumes()) + } + if len(c.GetVolumeMounts()) != 1 || c.GetVolumeMounts()[0].GetMountPath() != apiservice.GuestMountPath { + t.Errorf("mounts = %v", c.GetVolumeMounts()) + } + want := []string{"/ate/ko-app/ate-env-guest", "-workspace", "/workspace"} + if got := c.GetCommand(); len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] { + t.Errorf("command = %v, want %v", got, want) + } +} + +func TestBuildTemplateWorkspaceOnly(t *testing.T) { + cfg := testTemplateConfig() + cfg.workspace = "/w" + tmpl, err := buildTemplate(cfg) + if err != nil { + t.Fatalf("buildTemplate: %v", err) + } + c := tmpl.GetContainers()[0] + if c.GetImage() != cfg.guestImage || len(tmpl.GetVolumes()) != 0 { + t.Errorf("without --task-image the guest stays the container image and no volume is added: %v", tmpl) + } + if got := c.GetCommand(); len(got) != 3 || got[0] != "/ko-app/ate-env-guest" || got[2] != "/w" { + t.Errorf("command = %v", got) + } +} + +func TestTemplateConfigResolveImagesTaskImage(t *testing.T) { + cfg := testTemplateConfig() + cfg.taskImage = "python:3.12" + if err := cfg.resolveImages(); err == nil { + t.Error("unpinned --task-image accepted") + } + cfg.taskImage = "docker.io/library/python@sha256:" + testHex64 + if err := cfg.resolveImages(); err == nil { + t.Error("short guest digest accepted together with --task-image") + } + cfg.guestImage = "example.com/guest@sha256:" + testHex64 + if err := cfg.resolveImages(); err != nil { + t.Errorf("pinned images rejected: %v", err) + } +} diff --git a/docs/task-images/DESIGN.md b/docs/task-images/DESIGN.md new file mode 100644 index 0000000..b806acc --- /dev/null +++ b/docs/task-images/DESIGN.md @@ -0,0 +1,195 @@ +# Design: injecting the guest into unmodified task images + +Status: implemented. Companion to [README.md](README.md), which is the how-to. + +## Problem + +Every ate-env operation goes through `ate-env-guest`, a daemon inside the +actor. Until now the only way to run an image as an environment was to make the +guest the container image, which for a task image meant rebuilding it with the +guest layered on top and set as the entrypoint. For one image that is a `ko` +invocation. For an evaluation or RL fleet it is a build and a push per image, +duplicated guest bytes in the registry, a rebuild of every image for every guest +change, and a registry pipeline the harness has to own. + +The two things a container sources, its rootfs and its process, can be supplied +independently. Substrate's ActorTemplate already allows a second OCI image to be +mounted read-only into the actor. This design uses that to inject the guest. + +## Goals + +- Run any digest-pinned image as an environment with no rebuild. +- Keep the guest behavior, API and client code identical in both modes. +- Make the on-demand path idempotent and safe under concurrency, so harnesses + can call it per task without coordinating. +- Keep one place that knows the mount layout, so a second injected runtime can + reuse it. + +Non-goals: validating that an image exists or starts (the actor reports that), +per-create resource overrides, template garbage collection, and multi-container +templates. These are listed under future work. + +## Design + +### Template derivation + +`internal/apiservice/imagetemplate.go` holds `DeriveImageTemplate(base, name, +image)`. Given a base template it returns a new template that: + +- keeps the base's worker selector, snapshot configuration, sandbox + configuration, resources, container env and readiness probe; +- replaces the first container's image with the task image; +- if the base runs the guest as its image, adds a read-only image volume named + `guest` whose reference is the base's image, mounts it at `/ate`, and moves + the command's first element under `/ate` (`/ko-app/ate-env-guest` becomes + `/ate/ko-app/ate-env-guest`); +- if the base already has an image volume, leaves volumes, mounts and command + alone; +- drops server-assigned metadata and status. + +It refuses an unpinned task or guest image, a base without containers, and a +base command that is not an absolute path (a shell wrapper cannot be re-rooted +and would produce a template whose guest is silently missing). + +Both entry points use it: `ate-env manifest template --task-image` derives from +the guest-image template it would otherwise print, and `ate-env-api` derives +from a template fetched from the control plane. + +### Naming and idempotency + +A derived template is named `-`, in the base's atespace. The name is a pure function of the inputs a +caller controls, so: + +- the same image always maps to the same template, and a lookup by name is the + reuse check; +- a human can find the template for an image from its digest; +- 12 hex digits (48 bits) make accidental collisions a non-concern at any + realistic image count, while leaving room under the 63-character resource + name limit for a base name of up to 50 characters. Longer bases are rejected + with a message naming the limit. + +The guest is deliberately not part of the name. Templates are immutable in +Substrate, so a guest change is a new base template with a new name, which +yields new derived names. Encoding the guest digest would only lengthen names +without adding information. + +### Server flow + +`CreateEnvironment` with `image` set: + +1. Validate the digest and compute the derived name; invalid input is + `InvalidArgument` and touches nothing. +2. `GetActorTemplate(derived)`. If it exists and its container image is the + requested image, use it. If it exists with a different image, return + `FailedPrecondition` naming both images: someone else owns that name. +3. Otherwise `GetActorTemplate(base)`; a missing base is `FailedPrecondition` + with the atespace and the command that registers one. +4. Derive and `CreateActorTemplate`. `AlreadyExists` is success: a concurrent + create won the race and built the same template. +5. Create the actor from the derived template and report it in the response. + +The check in step 2 compares the container image only. Volumes and command are +not compared because the only way for them to differ is a template created by +other means under the derived name, which the image comparison already flags +in practice; a stricter comparison would cost a base fetch on every reuse. + +### API shape + +`image` is a field on `CreateEnvironmentRequest` rather than a new RPC, and the +existing `template` field doubles as the base. This keeps one create path, lets +clients that already pass a template start passing an image with one more +argument, and makes "which base" an explicit, per-request choice rather than +server configuration. The derived template is surfaced through the existing +`Environment.template`, so nothing new is needed to observe it. + +### Placement + +Derived templates live in the environment's atespace, next to the base. This is +also where `ate-env-api` already creates actors, so no new permissions are +involved beyond template creation, which the API server's identity needs for +this feature. + +## Failure semantics + +| Situation | Result | +|---|---| +| unpinned task image | `InvalidArgument`, nothing created | +| base missing | `FailedPrecondition`, nothing created | +| base command not re-rootable | `InvalidArgument`, nothing created | +| derived name held by another image | `FailedPrecondition`, nothing created | +| two creates race on a new image | both succeed, one template | +| image cannot be pulled or guest cannot start | create succeeds; actor never becomes ready; golden bake fails and the template reports it | +| control plane unavailable mid-flow | the underlying gRPC status is returned; a half-created template is reused next time because the name is deterministic | + +## Security + +- The task image is run with the same sandbox class and configuration as the + base, so injection changes what runs, not how it is isolated. +- The guest volume is read-only. The task image cannot alter the guest. +- Digest pinning is enforced for both images at every entry point. A tag could + be re-pointed between the golden bake and a later cold boot, which would + invalidate snapshots and make two environments on "the same image" differ. +- Callers of `CreateEnvironment` can now cause template creation. An + `ate-env-api` that is reachable by untrusted callers should already be behind + authentication; this adds template records and golden snapshots to what such + a caller can accumulate. + +## Performance + +The first environment on an image pays the task image pull on the worker it +lands on and a cold boot; the derived template's golden bake runs concurrently +on another worker. Later environments restore from the golden. In the +verification run below the first environment answered a shell command 37 s +after the create call on a small two-worker pool, most of it image pull. + +Costs that scale with the number of distinct images: one template record and +one golden snapshot per image. Neither is reclaimed automatically. + +## Alternatives considered + +- **Bake the guest into each image.** Works today and remains supported; it is + the per-image build cost this design removes. +- **An init container that copies the guest into a shared volume.** Needs a + writable volume and an extra container per actor, and the copy happens on + every cold boot. An image volume is shared by the node's cache and needs no + copy. +- **A server-side image-to-template map.** Equivalent to the derived name, but + state that can drift from the control plane. The name derivation is stateless + and recoverable from the templates themselves. +- **Including the guest digest in the derived name.** Rejected because template + immutability already forces a new base name for a new guest; see Naming. +- **A separate `CreateEnvironmentFromImage` RPC.** More surface for the same + behavior; a field keeps one path. + +## Future work + +- **Second injected runtime.** The derivation is runtime-agnostic apart from + the binary path. A runtime such as OpenSandbox's `execd` or E2B's `envd` is a + second image volume plus a command and readiness probe; the base template + describes which runtime it carries. +- **Per-create overrides** for resources and workspace, so one base can serve + images with different needs. +- **Derived-template garbage collection**, by age or by last use, with the + golden snapshot removed alongside. +- **Image validation at create time**, so a bad reference fails the call + instead of the actor. +- **Template atespace.** `CreateEnvironment` takes `template.atespace` but the + server creates the actor and resolves the template in the environment's + atespace; this predates the change and is unchanged by it. + +## Verification + +- Unit tests cover derivation from both base kinds, naming and its limits, and + every rejection. +- Server tests against the in-process fake control plane cover first create, + reuse, a named base in another atespace, and the failure table. +- `clients/python/tests/e2e/test_full_stack.py` has a create-from-image test + gated on `ATE_ENV_TASK_IMAGE`, which checks that the guest comes from the + volume and not the image, that the rootfs is the task image's, that files + round-trip, and that a second environment reuses the derived template. +- Run once end to end on a Kubernetes cluster running Substrate with a fresh + two-worker ate-env: `ate-env create --image` of an unmodified + `python:3.12-slim` produced the derived template, served `python3 --version` + from the image's own rootfs with the guest present under `/ate`, and deleted + cleanly. diff --git a/docs/task-images/README.md b/docs/task-images/README.md new file mode 100644 index 0000000..6baecb6 --- /dev/null +++ b/docs/task-images/README.md @@ -0,0 +1,137 @@ +# Running task images on ate-env + +Any OCI image can run as an ate-env environment without being rebuilt. The +`ate-env-guest` daemon that serves exec and file operations is mounted into the +image's container as a read-only image volume and started from there, so the +image's filesystem, interpreters and tools are exactly what was published. + +This page is the how-to. The reasoning behind it is in [DESIGN.md](DESIGN.md). + +## How it works + +An ActorTemplate normally names one container image, and for ate-env that image +has been the guest itself. A *task-image template* splits the two things a +container sources: + +``` +rootfs = the task image, unmodified, pinned by digest +process = /ate/ko-app/ate-env-guest, from a second image mounted read-only at /ate +``` + +Substrate materializes both at actor start. Everything else about the template +(worker selector, snapshot settings, sandbox class, resources, readiness probe) +is the same as for a guest-image template, and the guest behaves identically: +the environment's `shell`, process and file APIs, MCP, and suspend and resume +all work unchanged. + +## Prerequisites + +- A Substrate control plane whose ActorTemplate API supports image volumes + (`volumes[].image`). Every supported release does. +- Images pinned by digest (`repo@sha256:...`), both the task image and the + guest. Substrate requires this because a changed image invalidates snapshots. +- Worker nodes that can pull the task image's registry. +- A base template registered with `ate-env manifest template` (the quickstart + in the root README does this). Its guest image is the one injected. + +What the task image needs: + +- A `sh` for `Env.shell()` and for the `shell` MCP tool. Images without one + (distroless) still work for `start_process` with absolute binaries and for + file transfer. +- Port 80 free inside the sandbox: the guest listens there and the readiness + probe hits `/readyz` on it. +- Nothing else. The guest is a static binary and runs as the image's default + user; a writable workspace is wherever `--workspace` points (the guest's + default is `/`). + +## Option A: register a task-image template up front + +Good for a fixed set of images, or when you want the template to carry its own +name, resources or workspace. + +```bash +ate-env manifest template --template py312 \ + --task-image docker.io/library/python@sha256: \ + --guest-image /ate-env-guest@sha256: \ + --workspace /workspace \ + --snapshots-bucket | kubectl-ate create actor-template -f - + +ate-env create job1 --template py312 +``` + +`--task-image` makes the task image the container and the guest an image +volume; `--workspace` is passed to the guest as `-workspace` in either mode. The +printed template is plain YAML you can edit before registering it (resources, +worker selector, sandbox class). + +## Option B: create from an image on demand + +Good when the set of images is open-ended or chosen by a caller, as in RL and +evaluation harnesses. + +```bash +ate-env create py1 --image docker.io/library/python@sha256: +``` + +```python +env = await client.create("py1", image="docker.io/library/python@sha256:") +(await env.info()).template.name # "default-template-<12 hex of the digest>" +``` + +```go +client.Create(ctx, &ateenvv1alpha.CreateEnvironmentRequest{ + Id: "py1", Image: "docker.io/library/python@sha256:", +}) +``` + +What happens on the server: + +1. The request's template (default `default-template`) is the **base**. It must + exist in the environment's atespace. +2. `ate-env-api` looks for a template named `-` next to the base. If it exists and runs that image, it is reused. +3. Otherwise the server derives it from the base and creates it: same worker + selector, snapshot and sandbox settings, env and readiness; the task image as + the container; the base's guest image as a volume at `/ate`; the command + re-rooted to `/ate/ko-app/ate-env-guest`. +4. The environment is created from the derived template, which the response and + `GetEnvironment` report. + +The first environment on an image pays the image pull and a cold boot while the +template's golden snapshot bakes in the background; later ones restore from the +golden. Concurrent first creates on the same image are safe: whichever wins +creates the template, the others reuse it. + +Templates created this way are visible like any other: + +```bash +kubectl-ate get actor-template --atespace ate-env +``` + +A fresh template shows `Failed` in that listing for a few seconds while its +golden bakes, then `Ready`. Environments can be created in the meantime. + +## Errors + +| Error | Meaning | +|---|---| +| `InvalidArgument: image ... is not pinned by digest` | Pass `repo@sha256:<64 hex>`; tags are rejected before anything is created | +| `FailedPrecondition: base actor template ... not found` | Register the base in the environment's atespace first (`ate-env manifest template`) | +| `FailedPrecondition: actor template ... exists but runs ...` | The derived name is taken by a template with a different image; delete it or use another base name | +| `InvalidArgument: ... cannot re-root command ...` | The base's command is not an absolute path to the guest (a shell wrapper); fix the base template | +| Environment never serves, derived template stays `Failed` | The task image could not be pulled or does not start the guest (port 80 taken, no exec permission); check the worker pod logs | + +## Cleanup and limits + +- Derived templates are not deleted when environments are. They are small + control-plane records plus one golden snapshot in object storage each; remove + them with `kubectl-ate delete actor-template` when an image is retired. +- Bumping the guest means a new base template (templates are immutable), and a + new base name yields new derived names. Existing derived templates keep the + guest they were built with. +- Resources, sandbox class and worker selector come from the base; a task image + that needs more is served by registering its own template (Option A) or a + second base. +- The task image's existence is not checked at create time. A bad reference is + reported by the actor failing to start, not by `CreateEnvironment`. diff --git a/internal/apiservice/imagetemplate.go b/internal/apiservice/imagetemplate.go new file mode 100644 index 0000000..706b8de --- /dev/null +++ b/internal/apiservice/imagetemplate.go @@ -0,0 +1,164 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiservice + +import ( + "errors" + "fmt" + "path" + "regexp" + "strings" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" +) + +// GuestMountPath is where the ate-env-guest image is mounted inside an actor +// whose container image is a task image rather than the guest itself. The +// guest binary is then at GuestMountPath + "/ko-app/ate-env-guest". +const GuestMountPath = "/ate" + +// GuestVolumeName is the name of the image volume carrying the guest. +const GuestVolumeName = "guest" + +// defaultGuestBinary is the guest's path inside its own image (ko layout). +const defaultGuestBinary = "/ko-app/ate-env-guest" + +// imageTemplateDigestLen is how many hex digits of the digest go into a +// derived template's name: enough to make collisions a non-concern and short +// enough to keep the name a valid k8s short name next to any base name. +const imageTemplateDigestLen = 12 + +var digestRef = regexp.MustCompile(`@sha256:([0-9a-f]{64})$`) + +// shortName is the shape substrate accepts for a resource name (k8s short +// name): lowercase alphanumerics and '-', at most 63 characters. +var shortName = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +const maxShortNameLen = 63 + +// ImageDigest returns the hex digest of a digest-pinned image reference, or an +// error when the reference is not pinned. Substrate requires pinned images in +// ActorTemplates because changing the image invalidates snapshots. +func ImageDigest(image string) (string, error) { + m := digestRef.FindStringSubmatch(image) + if m == nil { + return "", fmt.Errorf("image %q is not pinned by digest (want repo@sha256:<64 hex>)", image) + } + return m[1], nil +} + +// ImageTemplateName is the name of the ActorTemplate derived from base for +// image: "-". +func ImageTemplateName(base, image string) (string, error) { + digest, err := ImageDigest(image) + if err != nil { + return "", err + } + name := base + "-" + digest[:imageTemplateDigestLen] + if len(name) > maxShortNameLen { + return "", fmt.Errorf("derived template name %q is %d characters, the limit is %d; use a shorter base template name", + name, len(name), maxShortNameLen) + } + if !shortName.MatchString(name) { + return "", fmt.Errorf("derived template name %q is not a valid resource name (lowercase alphanumerics and '-')", name) + } + return name, nil +} + +// DeriveImageTemplate returns a copy of base, named name, whose container runs +// image unmodified with the ate-env-guest mounted into it as a read-only image +// volume at GuestMountPath. +// +// base can be either kind of template: one whose container image is the +// guest itself (the shape `ate-env manifest template` writes), in which case +// that image becomes the guest volume and the command is re-rooted under the +// mount; or one that already mounts the guest as an image volume, in which +// case only the container image changes. Everything else (worker selector, +// snapshots, sandbox config, resources, env, readiness) carries over. +func DeriveImageTemplate(base *ateapipb.ActorTemplate, name, image string) (*ateapipb.ActorTemplate, error) { + if base == nil { + return nil, errors.New("base template is required") + } + if name == "" { + return nil, errors.New("derived template name is required") + } + if _, err := ImageDigest(image); err != nil { + return nil, err + } + if len(base.GetContainers()) == 0 { + return nil, fmt.Errorf("base template %q has no containers", base.GetMetadata().GetName()) + } + + tmpl := proto.Clone(base).(*ateapipb.ActorTemplate) + tmpl.Metadata = &ateapipb.ResourceMetadata{ + Atespace: base.GetMetadata().GetAtespace(), + Name: name, + } + tmpl.Status = nil + + c := tmpl.Containers[0] + if guestVolume(tmpl) == nil { + // The base runs the guest as its container image: that image becomes + // the guest volume, and the command moves under the mount. + guestImage := c.GetImage() + if _, err := ImageDigest(guestImage); err != nil { + return nil, fmt.Errorf("base template %q guest image: %w", base.GetMetadata().GetName(), err) + } + tmpl.Volumes = append(tmpl.Volumes, &ateapipb.Volume{ + Name: GuestVolumeName, + Image: &ateapipb.ImageVolumeSource{Reference: guestImage}, + }) + c.VolumeMounts = append(c.VolumeMounts, &ateapipb.VolumeMount{ + Name: GuestVolumeName, + MountPath: GuestMountPath, + }) + command, err := rerootCommand(c.GetCommand()) + if err != nil { + return nil, fmt.Errorf("base template %q: %w", base.GetMetadata().GetName(), err) + } + c.Command = command + } + c.Image = image + return tmpl, nil +} + +// guestVolume returns the template's image volume, or nil when it has none. +func guestVolume(tmpl *ateapipb.ActorTemplate) *ateapipb.Volume { + for _, v := range tmpl.GetVolumes() { + if v.GetImage() != nil { + return v + } + } + return nil +} + +// rerootCommand moves a guest command that pointed into the guest image's +// root under GuestMountPath. An empty command means the guest's default path. +// A command that does not start with an absolute path cannot be re-rooted +// (a shell wrapper, for instance), so it is an error rather than a template +// whose guest is silently not found. +func rerootCommand(command []string) ([]string, error) { + if len(command) == 0 { + return []string{path.Join(GuestMountPath, defaultGuestBinary)}, nil + } + if !strings.HasPrefix(command[0], "/") { + return nil, fmt.Errorf("cannot re-root command %q under %s: it must start with an absolute path to the guest binary", + strings.Join(command, " "), GuestMountPath) + } + out := append([]string(nil), command...) + out[0] = path.Join(GuestMountPath, out[0]) + return out, nil +} diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go new file mode 100644 index 0000000..9190eb4 --- /dev/null +++ b/internal/apiservice/imagetemplate_test.go @@ -0,0 +1,172 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiservice + +import ( + "strings" + "testing" + + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/proto" +) + +const ( + testGuestImage = "example.com/ate-env-guest@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + testTaskImage = "docker.io/library/python@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +) + +func modeABase() *ateapipb.ActorTemplate { + return &ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ate-env", Name: "default-template", Uid: "u1", Version: 3}, + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"workload": "default-env"}}, + Containers: []*ateapipb.Container{{ + Name: "guest", + Image: testGuestImage, + Command: []string{"/ko-app/ate-env-guest", "-workspace", "/workspace"}, + Env: []*ateapipb.EnvVar{{Name: "PORT", Value: "80"}}, + Readyz: &ateapipb.ContainerReadyz{HttpGet: &ateapipb.HTTPGetAction{Path: "/readyz", Port: 80}}, + }}, + SnapshotsConfig: &ateapipb.SnapshotsConfig{StorageLocation: "gs://b/p/"}, + SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, + Status: &ateapipb.ActorTemplateStatus{}, + } +} + +func TestImageDigestAndTemplateName(t *testing.T) { + name, err := ImageTemplateName("default-template", testTaskImage) + if err != nil { + t.Fatal(err) + } + if name != "default-template-0123456789ab" { + t.Errorf("name = %q", name) + } + for _, bad := range []string{"python:3.12", "python@sha256:abc", "python@sha512:" + strings.Repeat("a", 128), ""} { + if _, err := ImageDigest(bad); err == nil { + t.Errorf("ImageDigest(%q) accepted an unpinned reference", bad) + } + } + // The derived name must stay a valid resource name. + if _, err := ImageTemplateName(strings.Repeat("b", 51), testTaskImage); err == nil { + t.Error("a 64-character derived name was accepted") + } + if name, err := ImageTemplateName(strings.Repeat("b", 50), testTaskImage); err != nil || len(name) != 63 { + t.Errorf("a 63-character derived name should be accepted: %q, %v", name, err) + } + if _, err := ImageTemplateName("Default", testTaskImage); err == nil { + t.Error("an uppercase base name was accepted") + } +} + +func TestDeriveImageTemplateFromGuestImageBase(t *testing.T) { + base := modeABase() + before := proto.Clone(base).(*ateapipb.ActorTemplate) + + got, err := DeriveImageTemplate(base, "default-template-0123456789ab", testTaskImage) + if err != nil { + t.Fatal(err) + } + if !proto.Equal(base, before) { + t.Error("DeriveImageTemplate mutated the base template") + } + + md := got.GetMetadata() + if md.GetName() != "default-template-0123456789ab" || md.GetAtespace() != "ate-env" { + t.Errorf("metadata = %v", md) + } + if md.GetUid() != "" || md.GetVersion() != 0 || got.GetStatus() != nil { + t.Errorf("server-assigned fields were copied: uid=%q version=%d status=%v", md.GetUid(), md.GetVersion(), got.GetStatus()) + } + + c := got.GetContainers()[0] + if c.GetImage() != testTaskImage { + t.Errorf("container image = %q, want the task image", c.GetImage()) + } + wantCmd := []string{"/ate/ko-app/ate-env-guest", "-workspace", "/workspace"} + if strings.Join(c.GetCommand(), " ") != strings.Join(wantCmd, " ") { + t.Errorf("command = %v, want %v", c.GetCommand(), wantCmd) + } + if len(got.GetVolumes()) != 1 || got.GetVolumes()[0].GetName() != GuestVolumeName || + got.GetVolumes()[0].GetImage().GetReference() != testGuestImage { + t.Errorf("volumes = %v, want one image volume %q with the guest image", got.GetVolumes(), GuestVolumeName) + } + if len(c.GetVolumeMounts()) != 1 || c.GetVolumeMounts()[0].GetName() != GuestVolumeName || + c.GetVolumeMounts()[0].GetMountPath() != GuestMountPath { + t.Errorf("volume mounts = %v, want %s at %s", c.GetVolumeMounts(), GuestVolumeName, GuestMountPath) + } + // Everything else carries over. + if !proto.Equal(got.GetWorkerSelector(), base.GetWorkerSelector()) || + !proto.Equal(got.GetSnapshotsConfig(), base.GetSnapshotsConfig()) || + !proto.Equal(got.GetSandboxConfig(), base.GetSandboxConfig()) || + !proto.Equal(c.GetReadyz(), base.GetContainers()[0].GetReadyz()) || + len(c.GetEnv()) != 1 { + t.Error("worker selector, snapshots, sandbox config, readyz or env did not carry over") + } +} + +func TestDeriveImageTemplateFromInjectedBase(t *testing.T) { + // A base that already mounts the guest: only the container image changes. + base, err := DeriveImageTemplate(modeABase(), "py-base", "example.com/base@sha256:"+strings.Repeat("b", 64)) + if err != nil { + t.Fatal(err) + } + got, err := DeriveImageTemplate(base, "py-base-0123456789ab", testTaskImage) + if err != nil { + t.Fatal(err) + } + c := got.GetContainers()[0] + if c.GetImage() != testTaskImage { + t.Errorf("image = %q", c.GetImage()) + } + if c.GetCommand()[0] != "/ate/ko-app/ate-env-guest" { + t.Errorf("command re-rooted twice: %v", c.GetCommand()) + } + if len(got.GetVolumes()) != 1 || len(c.GetVolumeMounts()) != 1 { + t.Errorf("guest volume duplicated: volumes=%d mounts=%d", len(got.GetVolumes()), len(c.GetVolumeMounts())) + } +} + +func TestDeriveImageTemplateDefaultsCommand(t *testing.T) { + base := modeABase() + base.Containers[0].Command = nil + got, err := DeriveImageTemplate(base, "x", testTaskImage) + if err != nil { + t.Fatal(err) + } + if cmd := got.GetContainers()[0].GetCommand(); len(cmd) != 1 || cmd[0] != "/ate/ko-app/ate-env-guest" { + t.Errorf("command = %v", cmd) + } +} + +func TestDeriveImageTemplateRejects(t *testing.T) { + if _, err := DeriveImageTemplate(modeABase(), "x", "python:3.12"); err == nil { + t.Error("unpinned task image accepted") + } + unpinnedGuest := modeABase() + unpinnedGuest.Containers[0].Image = "example.com/ate-env-guest:latest" + if _, err := DeriveImageTemplate(unpinnedGuest, "x", testTaskImage); err == nil { + t.Error("unpinned guest image accepted as an image volume") + } + if _, err := DeriveImageTemplate(&ateapipb.ActorTemplate{Metadata: &ateapipb.ResourceMetadata{Name: "empty"}}, "x", testTaskImage); err == nil { + t.Error("base without containers accepted") + } + if _, err := DeriveImageTemplate(modeABase(), "", testTaskImage); err == nil { + t.Error("empty name accepted") + } + wrapped := modeABase() + wrapped.Containers[0].Command = []string{"sh", "-c", "exec /ko-app/ate-env-guest"} + if _, err := DeriveImageTemplate(wrapped, "x", testTaskImage); err == nil || !strings.Contains(err.Error(), "re-root") { + t.Errorf("a shell-wrapped base command must be rejected, got %v", err) + } +} diff --git a/internal/apiservice/server.go b/internal/apiservice/server.go index f32a1f1..9f293e6 100644 --- a/internal/apiservice/server.go +++ b/internal/apiservice/server.go @@ -96,6 +96,14 @@ func (s *Server) CreateEnvironment(ctx context.Context, req *ateenvv1alpha.Creat atespace = DefaultAtespace } + if image := req.GetImage(); image != "" { + name, err := s.ensureImageTemplate(ctx, atespace, templateName, image) + if err != nil { + return nil, err + } + templateName = name + } + opts := ate.CreateOptions{ ID: req.GetId(), Template: templateName, @@ -118,6 +126,51 @@ func (s *Server) CreateEnvironment(ctx context.Context, req *ateenvv1alpha.Creat }, nil } +// ensureImageTemplate returns the name of the ActorTemplate that runs image +// on top of base, creating it from base on first use. The derived template +// lives in the environment's atespace, next to base. +func (s *Server) ensureImageTemplate(ctx context.Context, atespace, base, image string) (string, error) { + name, err := ImageTemplateName(base, image) + if err != nil { + return "", status.Error(codes.InvalidArgument, err.Error()) + } + existing, err := s.client.GetActorTemplate(ctx, atespace, name) + switch { + case err == nil: + if got := firstContainerImage(existing); got != image { + return "", status.Errorf(codes.FailedPrecondition, + "actor template %q exists but runs %q, not %q; delete it or use another base template", name, got, image) + } + return name, nil + case !errors.Is(err, ate.ErrNotFound): + return "", toGRPCError(err) + } + baseTmpl, err := s.client.GetActorTemplate(ctx, atespace, base) + if err != nil { + if errors.Is(err, ate.ErrNotFound) { + return "", status.Errorf(codes.FailedPrecondition, + "base actor template %q not found in atespace %q; register it first (ate-env manifest template)", base, atespace) + } + return "", toGRPCError(err) + } + derived, err := DeriveImageTemplate(baseTmpl, name, image) + if err != nil { + return "", status.Error(codes.InvalidArgument, err.Error()) + } + // A concurrent create may have won the race; it built the same template. + if _, err := s.client.CreateActorTemplate(ctx, derived); err != nil && !errors.Is(err, ate.ErrAlreadyExists) { + return "", toGRPCError(err) + } + return name, nil +} + +func firstContainerImage(t *ateapipb.ActorTemplate) string { + if len(t.GetContainers()) == 0 { + return "" + } + return t.GetContainers()[0].GetImage() +} + // GetEnvironment retrieves the status and configuration of an existing environment. func (s *Server) GetEnvironment(ctx context.Context, req *ateenvv1alpha.GetEnvironmentRequest) (*ateenvv1alpha.GetEnvironmentResponse, error) { if req.GetId() == "" { diff --git a/internal/apiservice/server_test.go b/internal/apiservice/server_test.go index 56578a9..15904b9 100644 --- a/internal/apiservice/server_test.go +++ b/internal/apiservice/server_test.go @@ -453,3 +453,131 @@ func TestActorStatusToEnvStatus(t *testing.T) { } } } + +const ( + testGuestImage = "example.com/ate-env-guest@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + testTaskImage = "docker.io/library/python@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +) + +// seedGuestTemplate registers the template `ate-env manifest template` +// writes: the guest is the container image. +func seedGuestTemplate(control *fakecontrol.Server, atespace, name string) { + control.AddTemplate(&ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: name}, + WorkerSelector: &ateapipb.Selector{MatchLabels: map[string]string{"workload": "default-env"}}, + Containers: []*ateapipb.Container{{ + Name: "guest", + Image: testGuestImage, + Command: []string{"/ko-app/ate-env-guest"}, + Env: []*ateapipb.EnvVar{{Name: "PORT", Value: "80"}}, + Readyz: &ateapipb.ContainerReadyz{HttpGet: &ateapipb.HTTPGetAction{Path: "/readyz", Port: 80}}, + }}, + SnapshotsConfig: &ateapipb.SnapshotsConfig{StorageLocation: "gs://bucket/ate-env/"}, + SandboxConfig: &ateapipb.SandboxConfig{SandboxClass: ateapipb.SandboxClass_SANDBOX_CLASS_GVISOR, ConfigName: "gvisor-default"}, + }) +} + +func TestCreateFromImageDerivesTemplate(t *testing.T) { + envClient, control := newTestEnv(t) + seedGuestTemplate(control, "ate-env", "default-template") + ctx := context.Background() + + resp, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "img1", Image: testTaskImage}) + if err != nil { + t.Fatal(err) + } + const want = "default-template-0123456789ab" + if got := resp.GetEnvironment().GetTemplate().GetName(); got != want { + t.Errorf("response template = %q, want %q", got, want) + } + + tmpl := control.Template("ate-env", want) + if tmpl == nil { + t.Fatalf("derived template %q was not created", want) + } + c := tmpl.GetContainers()[0] + if c.GetImage() != testTaskImage { + t.Errorf("derived container image = %q", c.GetImage()) + } + if len(tmpl.GetVolumes()) != 1 || tmpl.GetVolumes()[0].GetImage().GetReference() != testGuestImage { + t.Errorf("derived volumes = %v, want the guest image volume", tmpl.GetVolumes()) + } + if len(c.GetVolumeMounts()) != 1 || c.GetVolumeMounts()[0].GetMountPath() != apiservice.GuestMountPath { + t.Errorf("derived mounts = %v", c.GetVolumeMounts()) + } + if len(c.GetCommand()) != 1 || c.GetCommand()[0] != "/ate/ko-app/ate-env-guest" { + t.Errorf("derived command = %v", c.GetCommand()) + } + if tmpl.GetSnapshotsConfig().GetStorageLocation() != "gs://bucket/ate-env/" || tmpl.GetWorkerSelector() == nil { + t.Error("snapshots config or worker selector did not carry over") + } + + // The actor references the derived template. + got, err := envClient.GetEnvironment(ctx, &ateenvv1alpha.GetEnvironmentRequest{Id: "img1"}) + if err != nil { + t.Fatal(err) + } + if got.GetEnvironment().GetTemplate().GetName() != want { + t.Errorf("actor template = %q, want %q", got.GetEnvironment().GetTemplate().GetName(), want) + } + + // A second environment on the same image reuses the template. + if _, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "img2", Image: testTaskImage}); err != nil { + t.Fatal(err) + } + if n := control.TemplateCount(); n != 2 { + t.Errorf("template count = %d, want 2 (base + one derived)", n) + } +} + +func TestCreateFromImageUsesNamedBaseAndAtespace(t *testing.T) { + envClient, control := newTestEnv(t) + seedGuestTemplate(control, "team-a", "py-base") + + resp, err := envClient.CreateEnvironment(context.Background(), &ateenvv1alpha.CreateEnvironmentRequest{ + Id: "img1", + Atespace: "team-a", + Template: &ateenvv1alpha.Template{Name: "py-base"}, + Image: testTaskImage, + }) + if err != nil { + t.Fatal(err) + } + if got := resp.GetEnvironment().GetTemplate().GetName(); got != "py-base-0123456789ab" { + t.Errorf("template = %q", got) + } + if control.Template("team-a", "py-base-0123456789ab") == nil { + t.Error("derived template not created in the environment's atespace") + } +} + +func TestCreateFromImageRejects(t *testing.T) { + envClient, control := newTestEnv(t) + seedGuestTemplate(control, "ate-env", "default-template") + ctx := context.Background() + + cases := []struct { + name string + req *ateenvv1alpha.CreateEnvironmentRequest + code codes.Code + }{ + {"unpinned image", &ateenvv1alpha.CreateEnvironmentRequest{Id: "a", Image: "python:3.12"}, codes.InvalidArgument}, + {"missing base", &ateenvv1alpha.CreateEnvironmentRequest{Id: "b", Template: &ateenvv1alpha.Template{Name: "nope"}, Image: testTaskImage}, codes.FailedPrecondition}, + } + for _, tc := range cases { + _, err := envClient.CreateEnvironment(ctx, tc.req) + if status.Code(err) != tc.code { + t.Errorf("%s: code = %v (%v), want %v", tc.name, status.Code(err), err, tc.code) + } + } + + // A derived name already taken by a template running something else. + control.AddTemplate(&ateapipb.ActorTemplate{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "ate-env", Name: "default-template-0123456789ab"}, + Containers: []*ateapipb.Container{{Name: "guest", Image: "example.com/other@sha256:" + testGuestImage[len(testGuestImage)-64:]}}, + }) + _, err := envClient.CreateEnvironment(ctx, &ateenvv1alpha.CreateEnvironmentRequest{Id: "c", Image: testTaskImage}) + if status.Code(err) != codes.FailedPrecondition { + t.Errorf("conflicting derived template: code = %v (%v), want FailedPrecondition", status.Code(err), err) + } +} diff --git a/internal/ate/client.go b/internal/ate/client.go index a3bf79c..5109d84 100644 --- a/internal/ate/client.go +++ b/internal/ate/client.go @@ -68,6 +68,9 @@ const DefaultAtespace = "ate-env" // ErrNotFound is returned when an env, file, or directory does not exist. var ErrNotFound = errors.New("not found") +// ErrAlreadyExists is returned when a resource with the same name exists. +var ErrAlreadyExists = errors.New("already exists") + // Options configures a Client. type Options struct { // ControlAddr is the ateapi gRPC endpoint, e.g. "localhost:8080" @@ -309,6 +312,39 @@ func (c *Client) Delete(ctx context.Context, atespace, id string) error { // ref returns the ObjectRef identifying the actor backing actor id in atespace. // If atespace is empty, DefaultAtespace is used. +// GetActorTemplate reads an ActorTemplate. A missing template is ErrNotFound. +func (c *Client) GetActorTemplate(ctx context.Context, atespace, name string) (*ateapipb.ActorTemplate, error) { + if name == "" { + return nil, errors.New("ate: template name is required") + } + tmpl, err := c.control.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: c.ref(atespace, name)}) + if err != nil { + return nil, fmt.Errorf("ate: getting template %q: %w", name, wrapGRPCError(err)) + } + return tmpl, nil +} + +// CreateActorTemplate creates an ActorTemplate in its metadata's atespace. A +// name collision is ErrAlreadyExists, so callers racing to create the same +// derived template can treat it as success. +func (c *Client) CreateActorTemplate(ctx context.Context, tmpl *ateapipb.ActorTemplate) (*ateapipb.ActorTemplate, error) { + name := tmpl.GetMetadata().GetName() + if name == "" { + return nil, errors.New("ate: template metadata.name is required") + } + if tmpl.GetMetadata().GetAtespace() == "" { + tmpl.Metadata.Atespace = DefaultAtespace + } + created, err := c.control.CreateActorTemplate(ctx, &ateapipb.CreateActorTemplateRequest{ActorTemplate: tmpl}) + if err != nil { + if status.Code(err) == codes.AlreadyExists { + return nil, fmt.Errorf("ate: creating template %q: %w: %s", name, ErrAlreadyExists, status.Convert(err).Message()) + } + return nil, fmt.Errorf("ate: creating template %q: %w", name, wrapGRPCError(err)) + } + return created, nil +} + func (c *Client) ref(atespace, id string) *ateapipb.ObjectRef { if atespace == "" { atespace = DefaultAtespace diff --git a/internal/internaltest/fakecontrol/fakecontrol.go b/internal/internaltest/fakecontrol/fakecontrol.go index c382822..8baac22 100644 --- a/internal/internaltest/fakecontrol/fakecontrol.go +++ b/internal/internaltest/fakecontrol/fakecontrol.go @@ -47,6 +47,7 @@ type Server struct { mu sync.Mutex actors map[string]*ateapipb.Actor atespaces map[string]*ateapipb.Atespace + templates map[string]*ateapipb.ActorTemplate } // New returns an empty fake control server. @@ -54,9 +55,76 @@ func New() *Server { return &Server{ actors: make(map[string]*ateapipb.Actor), atespaces: make(map[string]*ateapipb.Atespace), + templates: make(map[string]*ateapipb.ActorTemplate), } } +// AddTemplate seeds an ActorTemplate, as an operator would have registered it. +func (s *Server) AddTemplate(tmpl *ateapipb.ActorTemplate) { + s.mu.Lock() + defer s.mu.Unlock() + s.templates[key(tmpl.GetMetadata().GetAtespace(), tmpl.GetMetadata().GetName())] = proto.Clone(tmpl).(*ateapipb.ActorTemplate) +} + +// Template returns a copy of a stored ActorTemplate, or nil. +func (s *Server) Template(atespace, name string) *ateapipb.ActorTemplate { + s.mu.Lock() + defer s.mu.Unlock() + t, ok := s.templates[key(atespace, name)] + if !ok { + return nil + } + return proto.Clone(t).(*ateapipb.ActorTemplate) +} + +// TemplateCount is how many ActorTemplates are stored. +func (s *Server) TemplateCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.templates) +} + +func (s *Server) GetActorTemplate(ctx context.Context, req *ateapipb.GetActorTemplateRequest) (*ateapipb.ActorTemplate, error) { + s.mu.Lock() + defer s.mu.Unlock() + ref := req.GetActorTemplate() + t, ok := s.templates[key(ref.GetAtespace(), ref.GetName())] + if !ok { + return nil, status.Errorf(codes.NotFound, "actor template %q not found", ref.GetName()) + } + return proto.Clone(t).(*ateapipb.ActorTemplate), nil +} + +func (s *Server) CreateActorTemplate(ctx context.Context, req *ateapipb.CreateActorTemplateRequest) (*ateapipb.ActorTemplate, error) { + s.mu.Lock() + defer s.mu.Unlock() + t := req.GetActorTemplate() + md := t.GetMetadata() + if md.GetName() == "" { + return nil, status.Error(codes.InvalidArgument, "metadata.name is required") + } + k := key(md.GetAtespace(), md.GetName()) + if _, exists := s.templates[k]; exists { + return nil, status.Errorf(codes.AlreadyExists, "actor template %q already exists", md.GetName()) + } + // Mirror the server's volume_mounts validation so a bad derivation fails here too. + declared := map[string]bool{} + for _, v := range t.GetVolumes() { + declared[v.GetName()] = true + } + for _, c := range t.GetContainers() { + for _, m := range c.GetVolumeMounts() { + if !declared[m.GetName()] { + return nil, status.Errorf(codes.InvalidArgument, "container %q mounts undeclared volume %q", c.GetName(), m.GetName()) + } + } + } + stored := proto.Clone(t).(*ateapipb.ActorTemplate) + stored.Metadata.Uid = "tmpl-" + md.GetName() + stored.Metadata.Version = 1 + s.templates[k] = stored + return proto.Clone(stored).(*ateapipb.ActorTemplate), nil +} // Serve starts the fake on a random localhost port and returns its // address and a shutdown function. Like the real ateapi, it serves TLS @@ -189,7 +257,6 @@ func (s *Server) SuspendActor(ctx context.Context, req *ateapipb.SuspendActorReq return &ateapipb.SuspendActorResponse{Actor: clone(a)}, nil } - // suspend checkpoints a. The caller holds s.mu. func (s *Server) suspend(a *ateapipb.Actor) { a.Status = &ateapipb.ActorStatus{ diff --git a/proto/ateenv/v1alpha/env.pb.go b/proto/ateenv/v1alpha/env.pb.go index a242a1d..4007dc0 100644 --- a/proto/ateenv/v1alpha/env.pb.go +++ b/proto/ateenv/v1alpha/env.pb.go @@ -252,7 +252,17 @@ type CreateEnvironmentRequest struct { // Substrate atespace for the environment (defaults to "ate-env"). Atespace string `protobuf:"bytes,2,opt,name=atespace,proto3" json:"atespace,omitempty"` // ActorTemplate configuration to instantiate (defaults to name "default-template" in atespace "ate-env"). - Template *Template `protobuf:"bytes,3,opt,name=template,proto3" json:"template,omitempty"` + Template *Template `protobuf:"bytes,3,opt,name=template,proto3" json:"template,omitempty"` + // OCI image to run, pinned by digest (repo@sha256:...). Optional. When set, + // the environment is created from an ActorTemplate derived from `template`, + // which acts as the base: its worker selector, snapshot, sandbox and + // resource settings are kept, its container image is replaced by this + // image, and the ate-env-guest from the base is mounted into it as a + // read-only image volume, so the task image runs unmodified. The derived + // template is named "-" in the + // base's atespace; it is created on first use and reused afterwards, and + // the response reports it. + Image string `protobuf:"bytes,4,opt,name=image,proto3" json:"image,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -308,6 +318,13 @@ func (x *CreateEnvironmentRequest) GetTemplate() *Template { return nil } +func (x *CreateEnvironmentRequest) GetImage() string { + if x != nil { + return x.Image + } + return "" +} + // Response returned after creating an environment. type CreateEnvironmentResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -651,11 +668,12 @@ const file_proto_ateenv_v1alpha_env_proto_rawDesc = "" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x124\n" + "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\x129\n" + - "\x06status\x18\x04 \x01(\x0e2!.ateenv.v1alpha.EnvironmentStatusR\x06status\"|\n" + + "\x06status\x18\x04 \x01(\x0e2!.ateenv.v1alpha.EnvironmentStatusR\x06status\"\x92\x01\n" + "\x18CreateEnvironmentRequest\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1a\n" + "\batespace\x18\x02 \x01(\tR\batespace\x124\n" + - "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\"Z\n" + + "\btemplate\x18\x03 \x01(\v2\x18.ateenv.v1alpha.TemplateR\btemplate\x12\x14\n" + + "\x05image\x18\x04 \x01(\tR\x05image\"Z\n" + "\x19CreateEnvironmentResponse\x12=\n" + "\venvironment\x18\x01 \x01(\v2\x1b.ateenv.v1alpha.EnvironmentR\venvironment\"C\n" + "\x15GetEnvironmentRequest\x12\x0e\n" + diff --git a/proto/ateenv/v1alpha/env.proto b/proto/ateenv/v1alpha/env.proto index 64d77ef..b2ce9b0 100644 --- a/proto/ateenv/v1alpha/env.proto +++ b/proto/ateenv/v1alpha/env.proto @@ -105,6 +105,16 @@ message CreateEnvironmentRequest { string atespace = 2; // ActorTemplate configuration to instantiate (defaults to name "default-template" in atespace "ate-env"). Template template = 3; + // OCI image to run, pinned by digest (repo@sha256:...). Optional. When set, + // the environment is created from an ActorTemplate derived from `template`, + // which acts as the base: its worker selector, snapshot, sandbox and + // resource settings are kept, its container image is replaced by this + // image, and the ate-env-guest from the base is mounted into it as a + // read-only image volume, so the task image runs unmodified. The derived + // template is named "-" in the + // base's atespace; it is created on first use and reused afterwards, and + // the response reports it. + string image = 4; } // Response returned after creating an environment. From a76974d99e348c4d9345b14f825fe22911080155 Mon Sep 17 00:00:00 2001 From: Tomer Glottman <83163454+tomergee@users.noreply.github.com> Date: Wed, 30 Sep 2026 21:46:46 -0700 Subject: [PATCH 2/2] Inject a second runtime as a layer: guest sidecars and template layers A task-image template carries the guest as a read-only image volume; an agent framework's own in-sandbox daemon, such as OpenSandbox's execd, is one more volume plus a process that has to start next to the guest. The guest is the container's only process, so it is the one that starts it. ate-env-guest gains -sidecar (repeatable: an absolute binary path and its arguments, run without a shell, output logged with the binary's name as prefix, restarted with a backoff that doubles from one second to thirty and resets after a minute of stable running) and -sidecar-readyz (repeatable: URLs that must answer 2xx once before /readyz reports ready, so Substrate's wakeup probe admits traffic only when every runtime is serving). Bad -sidecar values fail at start rather than loop. ate-env manifest template gains --layer name=image@sha256:...=/mount, --sidecar and --sidecar-readyz, which become image volumes, mounts and guest flags on the template. Template derivation recognizes the guest volume by name rather than as "an image volume", so a base with layers still gets the guest added or kept correctly and environments created on demand with --image inherit the second runtime. Only a command that starts with the guest binary is re-rooted under the mount. docs/task-images/RUNTIMES.md describes the model, the flags, who can reach the second runtime from where (processes inside the environment now; outside callers still only reach port 80 through the router, which is the next piece), suspend and restart behavior, and the limits. examples/runtime-layers is a runnable demo with a static busybox as the stand-in runtime: httpd on 44772 as the sidecar under both a python and a debian task image, the second one created from an image on the layered base. The first attempt with the dynamically linked busybox failed on the debian image with a glibc version error, which is why the docs make the self-contained-layer rule and its failure signature explicit. --- README.md | 3 +- cmd/ate-env-guest/main.go | 27 +++ cmd/ate-env-guest/sidecar.go | 191 ++++++++++++++++++++++ cmd/ate-env-guest/sidecar_test.go | 142 ++++++++++++++++ cmd/ate-env/manifest.go | 97 ++++++++++- cmd/ate-env/manifest_test.go | 58 +++++++ docs/task-images/DESIGN.md | 36 +++- docs/task-images/README.md | 7 + docs/task-images/RUNTIMES.md | 125 ++++++++++++++ examples/runtime-layers/README.md | 60 +++++++ examples/runtime-layers/demo.sh | 107 ++++++++++++ internal/apiservice/imagetemplate.go | 24 ++- internal/apiservice/imagetemplate_test.go | 49 +++++- 13 files changed, 905 insertions(+), 21 deletions(-) create mode 100644 cmd/ate-env-guest/sidecar.go create mode 100644 cmd/ate-env-guest/sidecar_test.go create mode 100644 docs/task-images/RUNTIMES.md create mode 100644 examples/runtime-layers/README.md create mode 100755 examples/runtime-layers/demo.sh diff --git a/README.md b/README.md index 03e5745..937a817 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,8 @@ ate-env manifest template --template py312 \ ``` See [docs/task-images/README.md](docs/task-images/README.md) for the full guide, including -creating environments from an image on demand, and [docs/task-images/DESIGN.md](docs/task-images/DESIGN.md) +creating environments from an image on demand, [docs/task-images/RUNTIMES.md](docs/task-images/RUNTIMES.md) +for injecting a second runtime as a layer, and [docs/task-images/DESIGN.md](docs/task-images/DESIGN.md) for the design. Then create and use an environment: diff --git a/cmd/ate-env-guest/main.go b/cmd/ate-env-guest/main.go index d6f952f..ac50196 100644 --- a/cmd/ate-env-guest/main.go +++ b/cmd/ate-env-guest/main.go @@ -15,6 +15,8 @@ // Command ate-env-guest is the daemon that runs inside a Substrate actor // and exposes command execution and filesystem access over gRPC // (ProcessService and FileSystemService) with HTTP readiness probe support. +// It can also start and supervise extra runtimes next to itself (-sidecar), +// folding their readiness into /readyz (-sidecar-readyz). package main import ( @@ -27,6 +29,7 @@ import ( "net/http" "os" "os/signal" + "strings" "syscall" "github.com/agent-substrate/env/guest" @@ -36,8 +39,23 @@ func main() { listen := flag.String("listen", ":80", "address to serve the guest API on") logDir := flag.String("log-dir", "", "directory for process logs (defaults to /var/log/ate-jobs or temporary dir)") workspace := flag.String("workspace", "/", "workspace root directory") + var sidecars, sidecarReadyz multiFlag + flag.Var(&sidecars, "sidecar", "extra runtime to start next to the guest and keep running: an absolute binary path followed by its arguments, whitespace-separated (repeatable)") + flag.Var(&sidecarReadyz, "sidecar-readyz", "URL that must answer 2xx before /readyz reports ready, e.g. http://127.0.0.1:44772/ready (repeatable)") flag.Parse() + // Reject bad sidecar values before listening, so a misconfigured template + // fails at actor start instead of looping on a restarting sidecar. + var sidecarArgv [][]string + for _, s := range sidecars { + argv, err := parseSidecar(s) + if err != nil { + log.Fatalf("invalid -sidecar: %v", err) + } + sidecarArgv = append(sidecarArgv, argv) + } + gate := newReadyGate(sidecarReadyz) + addr := *listen if addr == "" { addr = ":80" @@ -59,6 +77,10 @@ func main() { mux := http.NewServeMux() mux.HandleFunc("GET /readyz", func(w http.ResponseWriter, r *http.Request) { + if ok, why := gate.Ready(r.Context()); !ok { + http.Error(w, why, http.StatusServiceUnavailable) + return + } io.WriteString(w, "ok\n") }) mux.Handle("/", grpcServer) @@ -90,6 +112,11 @@ func main() { _ = srv.Shutdown(context.Background()) }() + for _, argv := range sidecarArgv { + log.Printf("starting sidecar %s", strings.Join(argv, " ")) + go runSidecar(ctx, argv, log.Printf) + } + log.Printf("ate-env-guest listening on %s (gRPC services=%s, workspace=%s, logdir=%s)", lis.Addr(), guest.FormatEnabledServices(cfg), cfg.Workspace, cfg.LogDir) if err := srv.Serve(lis); err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/cmd/ate-env-guest/sidecar.go b/cmd/ate-env-guest/sidecar.go new file mode 100644 index 0000000..f7d9d0e --- /dev/null +++ b/cmd/ate-env-guest/sidecar.go @@ -0,0 +1,191 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +// Sidecars are extra runtimes started next to the guest, from the same +// container: typically a second data-plane daemon that was mounted into the +// actor as an image volume (see docs/task-images/RUNTIMES.md). The guest is +// the container's only process by contract, so it is the one that starts +// them, restarts them if they exit, and folds their readiness into /readyz. + +// multiFlag collects the values of a repeatable flag. +type multiFlag []string + +func (m *multiFlag) String() string { return strings.Join(*m, "; ") } + +func (m *multiFlag) Set(v string) error { + if strings.TrimSpace(v) == "" { + return errors.New("value must not be empty") + } + *m = append(*m, v) + return nil +} + +// parseSidecar turns a -sidecar value into argv. Values are split on +// whitespace and no quoting is interpreted: sidecars run without a shell, so +// a runtime that needs one ships a wrapper script in its layer. The binary +// must be an absolute path, which is what a mounted layer provides and what +// keeps the guest independent of the task image's PATH. +func parseSidecar(v string) ([]string, error) { + argv := strings.Fields(v) + if len(argv) == 0 { + return nil, errors.New("sidecar command is empty") + } + if !filepath.IsAbs(argv[0]) { + return nil, fmt.Errorf("sidecar %q: the binary must be an absolute path (sidecars run without a shell or PATH lookup)", argv[0]) + } + return argv, nil +} + +const ( + sidecarMinBackoff = time.Second + sidecarMaxBackoff = 30 * time.Second + // sidecarStableAfter is how long a sidecar must have run for its next + // restart to start from the minimum backoff again. + sidecarStableAfter = time.Minute +) + +// runSidecar runs argv until ctx is done, restarting it whenever it exits. +// The backoff doubles from one second to thirty while the sidecar keeps +// failing quickly and resets once it has run for a while. Output goes to +// logf, one line at a time, prefixed with the binary's name. +func runSidecar(ctx context.Context, argv []string, logf func(string, ...any)) { + name := filepath.Base(argv[0]) + backoff := sidecarMinBackoff + for { + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = os.Environ() + out := &lineLogger{prefix: name, logf: logf} + cmd.Stdout = out + cmd.Stderr = out + start := time.Now() + err := cmd.Run() + out.flush() + if ctx.Err() != nil { + return + } + ran := time.Since(start) + logf("sidecar %s exited after %s: %v; restarting in %s", name, ran.Round(time.Millisecond), err, backoff) + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + if ran >= sidecarStableAfter { + backoff = sidecarMinBackoff + } else { + backoff = min(backoff*2, sidecarMaxBackoff) + } + } +} + +// lineLogger writes complete lines to logf with a prefix; a partial trailing +// line is kept until the next write or flush. +type lineLogger struct { + prefix string + logf func(string, ...any) + mu sync.Mutex + buf bytes.Buffer +} + +func (l *lineLogger) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + l.buf.Write(p) + for { + line, err := l.buf.ReadString('\n') + if err != nil { + // No newline yet: put the partial line back. + l.buf.Reset() + l.buf.WriteString(line) + break + } + l.logf("[%s] %s", l.prefix, strings.TrimRight(line, "\r\n")) + } + return len(p), nil +} + +func (l *lineLogger) flush() { + l.mu.Lock() + defer l.mu.Unlock() + if l.buf.Len() > 0 { + l.logf("[%s] %s", l.prefix, l.buf.String()) + l.buf.Reset() + } +} + +// readyGate makes /readyz wait for the sidecars' own readiness endpoints. +// Each URL is polled until it answers 2xx once; after that it is not asked +// again. Readiness is a start-up gate, so Substrate's wakeup probe admits +// traffic only when every runtime in the actor is serving, not a liveness +// check on the sidecars. +type readyGate struct { + urls []string + client *http.Client + + mu sync.Mutex + seen map[string]bool +} + +func newReadyGate(urls []string) *readyGate { + return &readyGate{ + urls: urls, + client: &http.Client{Timeout: 2 * time.Second}, + seen: make(map[string]bool, len(urls)), + } +} + +// Ready reports whether every URL has answered 2xx, and if not, which one is +// still pending and why. +func (g *readyGate) Ready(ctx context.Context) (bool, string) { + for _, u := range g.urls { + g.mu.Lock() + ok := g.seen[u] + g.mu.Unlock() + if ok { + continue + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return false, fmt.Sprintf("sidecar readiness %s: %v", u, err) + } + resp, err := g.client.Do(req) + if err != nil { + return false, fmt.Sprintf("sidecar readiness %s: %v", u, err) + } + resp.Body.Close() + if resp.StatusCode/100 != 2 { + return false, fmt.Sprintf("sidecar readiness %s: HTTP %d", u, resp.StatusCode) + } + g.mu.Lock() + g.seen[u] = true + g.mu.Unlock() + } + return true, "" +} diff --git a/cmd/ate-env-guest/sidecar_test.go b/cmd/ate-env-guest/sidecar_test.go new file mode 100644 index 0000000..3480d76 --- /dev/null +++ b/cmd/ate-env-guest/sidecar_test.go @@ -0,0 +1,142 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os/exec" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestParseSidecar(t *testing.T) { + argv, err := parseSidecar(" /opt/osb/execd --port 44772 ") + if err != nil || strings.Join(argv, " ") != "/opt/osb/execd --port 44772" { + t.Errorf("parseSidecar = %v, %v", argv, err) + } + for _, bad := range []string{"", " ", "execd --port 1", "sh -c /opt/osb/execd"} { + if _, err := parseSidecar(bad); err == nil { + t.Errorf("parseSidecar(%q) accepted a non-absolute or empty command", bad) + } + } + var m multiFlag + if err := m.Set(""); err == nil { + t.Error("empty flag value accepted") + } + _ = m.Set("/a") + _ = m.Set("/b x") + if len(m) != 2 || m.String() != "/a; /b x" { + t.Errorf("multiFlag = %v", m) + } +} + +func TestReadyGateWaitsOnceForEachURL(t *testing.T) { + var state atomic.Int32 // 0: 503, 1: 200 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if state.Load() == 0 { + http.Error(w, "booting", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + closed := httptest.NewServer(http.NotFoundHandler()) + closedURL := closed.URL + closed.Close() + + g := newReadyGate([]string{srv.URL + "/ready"}) + ctx := context.Background() + if ok, why := g.Ready(ctx); ok || !strings.Contains(why, "HTTP 503") { + t.Errorf("not ready expected, got ok=%v why=%q", ok, why) + } + state.Store(1) + if ok, why := g.Ready(ctx); !ok { + t.Errorf("ready expected, got %q", why) + } + // Once seen, the URL is not re-checked: readiness is a start-up gate. + state.Store(0) + if ok, _ := g.Ready(ctx); !ok { + t.Error("readiness flipped back after the sidecar had answered once") + } + + unreachable := newReadyGate([]string{closedURL + "/ready"}) + if ok, why := unreachable.Ready(ctx); ok || why == "" { + t.Errorf("an unreachable sidecar must report not ready with a reason, got ok=%v why=%q", ok, why) + } + if ok, _ := newReadyGate(nil).Ready(ctx); !ok { + t.Error("no sidecars means ready") + } +} + +func TestRunSidecarRestartsAndStopsWithContext(t *testing.T) { + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("no sh on this host") + } + var mu sync.Mutex + var lines []string + logf := func(format string, args ...any) { + mu.Lock() + defer mu.Unlock() + lines = append(lines, fmt.Sprintf(format, args...)) + } + + // A sidecar that prints and exits non-zero is restarted with backoff. + ctx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond) + defer cancel() + done := make(chan struct{}) + go func() { + runSidecar(ctx, []string{sh, "-c", "echo hello from sidecar; exit 3"}, logf) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("runSidecar did not return after its context ended") + } + mu.Lock() + joined := strings.Join(lines, "\n") + mu.Unlock() + if !strings.Contains(joined, "[sh] hello from sidecar") { + t.Errorf("sidecar output was not logged with its prefix:\n%s", joined) + } + if n := strings.Count(joined, "restarting in"); n < 2 { + t.Errorf("expected at least two restarts within the window (1s then 2s backoff), got %d:\n%s", n, joined) + } + if !strings.Contains(joined, "restarting in 1s") || !strings.Contains(joined, "restarting in 2s") { + t.Errorf("backoff did not double:\n%s", joined) + } + + // A long-running sidecar is killed promptly when the guest shuts down. + ctx2, cancel2 := context.WithCancel(context.Background()) + done2 := make(chan struct{}) + go func() { + runSidecar(ctx2, []string{sh, "-c", "sleep 60"}, logf) + close(done2) + }() + time.Sleep(200 * time.Millisecond) + cancel2() + select { + case <-done2: + case <-time.After(3 * time.Second): + t.Fatal("sidecar was not stopped when the context was cancelled") + } +} diff --git a/cmd/ate-env/manifest.go b/cmd/ate-env/manifest.go index b906013..08f64ea 100644 --- a/cmd/ate-env/manifest.go +++ b/cmd/ate-env/manifest.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "io" + "strings" "github.com/agent-substrate/env/internal/apiservice" atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" @@ -70,6 +71,58 @@ type templateConfig struct { taskImage string // workspace, when set, is passed to the guest as -workspace. workspace string + // layers are extra runtimes mounted as read-only image volumes, each + // "name=image@sha256:...=/mount/path". + layers []string + // sidecars are command lines the guest starts and supervises next to + // itself (absolute binary path first), one per -sidecar flag. + sidecars []string + // sidecarReadyz are URLs the guest's /readyz waits for. + sidecarReadyz []string +} + +// runtimeLayer is a parsed --layer value. +type runtimeLayer struct { + name, image, mountPath string +} + +// parseLayer parses "name=image@sha256:...=/mount/path". The image reference +// carries no '=' and the mount path is absolute, so splitting on the first two +// '=' is unambiguous. +func parseLayer(v string) (runtimeLayer, error) { + parts := strings.SplitN(v, "=", 3) + if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" { + return runtimeLayer{}, fmt.Errorf("--layer %q: want name=image@sha256:...=/mount/path", v) + } + l := runtimeLayer{name: parts[0], image: parts[1], mountPath: parts[2]} + if l.name == apiservice.GuestVolumeName { + return runtimeLayer{}, fmt.Errorf("--layer %q: the name %q is reserved for the guest", v, apiservice.GuestVolumeName) + } + if _, err := apiservice.ImageDigest(l.image); err != nil { + return runtimeLayer{}, fmt.Errorf("--layer %q: %w", l.name, err) + } + if !strings.HasPrefix(l.mountPath, "/") || l.mountPath == apiservice.GuestMountPath { + return runtimeLayer{}, fmt.Errorf("--layer %q: mount path must be absolute and not %s", l.name, apiservice.GuestMountPath) + } + return l, nil +} + +// parseLayers parses and cross-checks every --layer value. +func parseLayers(values []string) ([]runtimeLayer, error) { + var layers []runtimeLayer + names, mounts := map[string]bool{}, map[string]bool{} + for _, v := range values { + l, err := parseLayer(v) + if err != nil { + return nil, err + } + if names[l.name] || mounts[l.mountPath] { + return nil, fmt.Errorf("--layer %q: duplicate name or mount path", v) + } + names[l.name], mounts[l.mountPath] = true, true + layers = append(layers, l) + } + return layers, nil } func (c *templateConfig) resolveImages() error { @@ -87,6 +140,20 @@ func (c *templateConfig) resolveImages() error { return fmt.Errorf("--guest-image must be digest-pinned to be mounted as an image volume: %w", err) } } + if _, err := parseLayers(c.layers); err != nil { + return err + } + for _, s := range c.sidecars { + argv := strings.Fields(s) + if len(argv) == 0 || !strings.HasPrefix(argv[0], "/") { + return fmt.Errorf("--sidecar %q: want an absolute binary path followed by its arguments", s) + } + } + for _, u := range c.sidecarReadyz { + if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") { + return fmt.Errorf("--sidecar-readyz %q: want an http(s) URL", u) + } + } return nil } @@ -164,6 +231,9 @@ It prints YAML to stdout without touching the cluster.`, cmd.Flags().StringVar(&tCfg.snapshotsBucket, "snapshots-bucket", "", "object-storage bucket (with optional prefix) for actor snapshots, e.g. gs://bucket/prefix/") cmd.Flags().StringVar(&tCfg.taskImage, "task-image", "", "digest-pinned image to run unmodified; the guest is mounted into it as a read-only image volume at "+apiservice.GuestMountPath) cmd.Flags().StringVar(&tCfg.workspace, "workspace", "", "workspace root inside the actor, passed to the guest as -workspace (default: the guest's default)") + cmd.Flags().StringArrayVar(&tCfg.layers, "layer", nil, "extra runtime mounted as a read-only image volume, as name=image@sha256:...=/mount/path (repeatable)") + cmd.Flags().StringArrayVar(&tCfg.sidecars, "sidecar", nil, "command the guest starts and keeps running next to itself, an absolute binary path plus arguments, e.g. '/opt/opensandbox/execd --port 44772' (repeatable)") + cmd.Flags().StringArrayVar(&tCfg.sidecarReadyz, "sidecar-readyz", nil, "URL the actor's readiness waits for, e.g. http://127.0.0.1:44772/ready (repeatable)") return cmd } @@ -330,10 +400,33 @@ func buildActorTemplate(cfg templateConfig) *ateapipb.ActorTemplate { // set, that image as the container with the guest mounted into it. func buildTemplate(cfg templateConfig) (*ateapipb.ActorTemplate, error) { tmpl := buildActorTemplate(cfg) + c := tmpl.Containers[0] + command := append([]string(nil), c.Command...) if cfg.workspace != "" { - c := tmpl.Containers[0] - c.Command = append(append([]string(nil), c.Command...), "-workspace", cfg.workspace) + command = append(command, "-workspace", cfg.workspace) + } + // Extra runtimes: each layer is an image volume, each sidecar a flag the + // guest acts on. They are added before any task-image derivation so the + // derived template inherits them, and so do environments created from + // it with an image of their own. + layers, err := parseLayers(cfg.layers) + if err != nil { + return nil, err + } + for _, l := range layers { + tmpl.Volumes = append(tmpl.Volumes, &ateapipb.Volume{ + Name: l.name, + Image: &ateapipb.ImageVolumeSource{Reference: l.image}, + }) + c.VolumeMounts = append(c.VolumeMounts, &ateapipb.VolumeMount{Name: l.name, MountPath: l.mountPath}) + } + for _, s := range cfg.sidecars { + command = append(command, "-sidecar", s) + } + for _, u := range cfg.sidecarReadyz { + command = append(command, "-sidecar-readyz", u) } + c.Command = command if cfg.taskImage == "" { return tmpl, nil } diff --git a/cmd/ate-env/manifest_test.go b/cmd/ate-env/manifest_test.go index cca2e8b..b0f69f0 100644 --- a/cmd/ate-env/manifest_test.go +++ b/cmd/ate-env/manifest_test.go @@ -329,3 +329,61 @@ func TestTemplateConfigResolveImagesTaskImage(t *testing.T) { t.Errorf("pinned images rejected: %v", err) } } + +func TestBuildTemplateWithRuntimeLayer(t *testing.T) { + cfg := testTemplateConfig() + cfg.guestImage = "example.com/guest@sha256:" + testHex64 + cfg.taskImage = "docker.io/library/python@sha256:" + testHex64 + cfg.layers = []string{"execd=example.com/execd@sha256:" + testHex64 + "=/opt/opensandbox"} + cfg.sidecars = []string{"/opt/opensandbox/execd --port 44772"} + cfg.sidecarReadyz = []string{"http://127.0.0.1:44772/ready"} + + tmpl, err := buildTemplate(cfg) + if err != nil { + t.Fatalf("buildTemplate: %v", err) + } + vols := tmpl.GetVolumes() + if len(vols) != 2 || vols[0].GetName() != "execd" || vols[0].GetImage().GetReference() != "example.com/execd@sha256:"+testHex64 || vols[1].GetName() != apiservice.GuestVolumeName { + t.Errorf("volumes = %v, want the execd layer then the guest volume", vols) + } + c := tmpl.GetContainers()[0] + if len(c.GetVolumeMounts()) != 2 || c.GetVolumeMounts()[0].GetMountPath() != "/opt/opensandbox" || c.GetVolumeMounts()[1].GetMountPath() != apiservice.GuestMountPath { + t.Errorf("mounts = %v", c.GetVolumeMounts()) + } + want := "/ate/ko-app/ate-env-guest -sidecar /opt/opensandbox/execd --port 44772 -sidecar-readyz http://127.0.0.1:44772/ready" + if got := strings.Join(c.GetCommand(), " "); got != want { + t.Errorf("command = %q, want %q", got, want) + } + if c.GetImage() != cfg.taskImage { + t.Errorf("image = %q", c.GetImage()) + } +} + +func TestParseLayerRejects(t *testing.T) { + cases := map[string]string{ + "missing parts": "execd=example.com/execd@sha256:" + testHex64, + "unpinned image": "execd=example.com/execd:latest=/opt/execd", + "relative mount": "execd=example.com/execd@sha256:" + testHex64 + "=opt/execd", + "guest mount path": "execd=example.com/execd@sha256:" + testHex64 + "=/ate", + "reserved name": "guest=example.com/execd@sha256:" + testHex64 + "=/opt/execd", + } + for name, v := range cases { + if _, err := parseLayer(v); err == nil { + t.Errorf("%s: %q accepted", name, v) + } + } + dup := "a=example.com/x@sha256:" + testHex64 + "=/opt/a" + if _, err := parseLayers([]string{dup, dup}); err == nil { + t.Error("duplicate layer accepted") + } + cfg := testTemplateConfig() + cfg.sidecars = []string{"execd --port 1"} + if err := cfg.resolveImages(); err == nil { + t.Error("relative sidecar binary accepted") + } + cfg = testTemplateConfig() + cfg.sidecarReadyz = []string{"127.0.0.1:44772/ready"} + if err := cfg.resolveImages(); err == nil { + t.Error("sidecar-readyz without a scheme accepted") + } +} diff --git a/docs/task-images/DESIGN.md b/docs/task-images/DESIGN.md index b806acc..24165e4 100644 --- a/docs/task-images/DESIGN.md +++ b/docs/task-images/DESIGN.md @@ -110,6 +110,28 @@ also where `ate-env-api` already creates actors, so no new permissions are involved beyond template creation, which the API server's identity needs for this feature. +### Runtime layers + +A second runtime is expressed with the same primitives rather than a new +template concept: an extra image volume for its binary, and two guest flags. +`-sidecar` makes the guest start and supervise a process (restart with +backoff, output into the guest log), and `-sidecar-readyz` makes the guest's +`/readyz`, which Substrate uses as the wakeup probe, wait for the runtime's own +health endpoint once. The guest is the right supervisor because it is already +the container's only process and the readiness authority. + +Derivation treats layers as part of what carries over. The guest volume is +recognized by its name, not by being an image volume, so a base with a layer +still gets the guest added or kept correctly, and `create --image` on a layered +base yields environments with the second runtime present. Layers are +documented in [RUNTIMES.md](RUNTIMES.md); the verified example uses busybox as +a stand-in runtime. + +What this does not solve is reaching a second runtime from outside the actor. +The router forwards to port 80 unless a CONNECT authority names another port, +and `ate-env-api` proxies only the guest. Exposing a runtime's port is the next +piece, and it belongs with the OpenSandbox backend that needs it. + ## Failure semantics | Situation | Result | @@ -164,10 +186,10 @@ one golden snapshot per image. Neither is reclaimed automatically. ## Future work -- **Second injected runtime.** The derivation is runtime-agnostic apart from - the binary path. A runtime such as OpenSandbox's `execd` or E2B's `envd` is a - second image volume plus a command and readiness probe; the base template - describes which runtime it carries. +- **Reaching a second runtime from outside.** Layers and sidecars put a + runtime such as OpenSandbox's `execd` inside the actor and gate readiness on + it; an endpoint lookup on `ate-env-api` that names the runtime's port + through the router is what an external SDK still needs. - **Per-create overrides** for resources and workspace, so one base can serve images with different needs. - **Derived-template garbage collection**, by age or by last use, with the @@ -180,8 +202,10 @@ one golden snapshot per image. Neither is reclaimed automatically. ## Verification -- Unit tests cover derivation from both base kinds, naming and its limits, and - every rejection. +- Unit tests cover derivation from both base kinds, naming and its limits, + every rejection, and that runtime layers and sidecar flags survive + derivation. The guest's sidecar supervisor and readiness gate have their own + tests (restart with backoff, stop on shutdown, gate once per URL). - Server tests against the in-process fake control plane cover first create, reuse, a named base in another atespace, and the failure table. - `clients/python/tests/e2e/test_full_stack.py` has a create-from-image test diff --git a/docs/task-images/README.md b/docs/task-images/README.md index 6baecb6..3af73ac 100644 --- a/docs/task-images/README.md +++ b/docs/task-images/README.md @@ -112,6 +112,13 @@ kubectl-ate get actor-template --atespace ate-env A fresh template shows `Failed` in that listing for a few seconds while its golden bakes, then `Ready`. Environments can be created in the meantime. +## A second runtime + +The same mechanism carries more runtimes. `--layer` mounts another image, +`--sidecar` has the guest start a process from it, and `--sidecar-readyz` +folds its readiness into the actor's. See [RUNTIMES.md](RUNTIMES.md) and the +runnable example in [`examples/runtime-layers`](../../examples/runtime-layers). + ## Errors | Error | Meaning | diff --git a/docs/task-images/RUNTIMES.md b/docs/task-images/RUNTIMES.md new file mode 100644 index 0000000..25ab84e --- /dev/null +++ b/docs/task-images/RUNTIMES.md @@ -0,0 +1,125 @@ +# Injecting a second runtime as a layer + +[README.md](README.md) explains how the ate-env guest is injected into an +unmodified task image. The same mechanism carries any number of further +runtimes: a second daemon that an agent framework expects inside its sandbox, +such as OpenSandbox's `execd`, is one more read-only image volume plus a process +the guest starts next to itself. This page explains the model, the flags, and +what reaches the second runtime from where. + +## Model + +A runtime layer is three things: + +| part | where it is expressed | what it does | +|---|---|---| +| the binary | an image volume on the ActorTemplate (`--layer name=image@sha256:...=/mount`) | mounts the runtime's image read-only at a path of your choice, next to the guest at `/ate` | +| the process | a guest flag (`--sidecar '/mount/binary args...'`) | the guest starts it after it is listening, logs its output with a prefix, and restarts it with backoff if it exits | +| its readiness | a guest flag (`--sidecar-readyz http://127.0.0.1:PORT/health`) | the guest's `/readyz`, which is the actor's wakeup probe, answers 503 until every such URL has answered 2xx once | + +The guest is the container's only process by contract, which is why it is the +one that supervises the others. Sidecars run without a shell or `PATH` lookup, +so the binary is an absolute path into its mount, and the task image needs +nothing for it. They inherit the container's user, environment and filesystem. + +Template derivation keeps all of it. A base with layers yields derived +templates with the same layers and the same guest flags, so environments +created on demand with `image=` inherit the second runtime without any caller +knowing it exists. + +## Registering a layered template + +```bash +ate-env manifest template --template osb-base \ + --task-image docker.io/library/python@sha256: \ + --guest-image /ate-env-guest@sha256: \ + --layer execd=/opensandbox/execd@sha256:=/opt/opensandbox \ + --sidecar '/opt/opensandbox/execd' \ + --sidecar-readyz http://127.0.0.1:44772/ready \ + --snapshots-bucket | kubectl-ate create actor-template -f - +``` + +The resulting template has two image volumes (`execd` at `/opt/opensandbox`, +`guest` at `/ate`), the task image as the container, and the command + +``` +/ate/ko-app/ate-env-guest -sidecar "/opt/opensandbox/execd" -sidecar-readyz http://127.0.0.1:44772/ready +``` + +Everything else is as in a plain task-image template. Environments are created +the usual way, from this template by name or from any other image with `--image` +while naming it as the base: + +```bash +ate-env create py1 --template osb-base +ate-env create node1 --template osb-base --image docker.io/library/node@sha256: +``` + +Both run the guest and `execd`; the second one from a derived template named +`osb-base-<12 hex of the node digest>`. + +### OpenSandbox `execd` specifics + +`execd` is a static binary that OpenSandbox injects into sandboxes without +modifying the base image, which is exactly the shape this page describes. Its +image carries the binary at `/execd`; mounting the image at `/opt/opensandbox` +puts it at `/opt/opensandbox/execd`, OpenSandbox's own default install path. It +serves HTTP on port 44772 by default, with `/ping` and `/ready` open and every +other endpoint behind the `X-EXECD-ACCESS-TOKEN` header when a token is +configured. Check the OpenSandbox documentation for the current flags and +environment variables for the port, token and workspace; pass them in the +`--sidecar` value, or wrap them in a script shipped in the layer if quoting is +needed. Its optional Jupyter code-interpreter mode starts a Jupyter server +inside the sandbox; for fleets of small environments use the plain command and +file endpoints. + +A verified stand-in for the mechanics, which needs nothing from any registry +but Docker Hub, is in [`examples/runtime-layers`](../../examples/runtime-layers): +the static `busybox:1.37-musl` mounted as the layer and its `httpd` as the +sidecar, run under two different task images. + +## Who can reach the second runtime + +- **Processes inside the environment** reach it at `127.0.0.1:` right + away. An agent framework whose tool calls run inside the sandbox is served. +- **The guest's own APIs** are unaffected: exec, files and MCP keep going + through `ate-env-api` on port 80. +- **Clients outside the environment** currently reach only port 80. The atenet + router forwards to the actor's port 80 unless the request is a CONNECT whose + authority names another port, and `ate-env-api` proxies only the guest. An + SDK that speaks HTTP to the second runtime therefore needs one of two things, + neither of which is in this branch: an endpoint lookup on `ate-env-api` that + returns a router address naming the runtime's port, or a reverse proxy. This + is the next step for the OpenSandbox integration, whose lifecycle server + resolves sandbox endpoints exactly this way. + +## Suspend, resume and restarts + +A sidecar is part of the actor, so a suspend checkpoints its memory with the +guest's and a resume brings it back without a restart, open sockets included. If +a runtime does not survive that (it polls a clock it can no longer trust, say) +and exits, the guest restarts it: after one second, doubling up to thirty while +it keeps failing, and from one second again once it has run for a minute. +Readiness is gated once, at start; a sidecar that dies later is restarted but +does not take the actor out of service. + +## Limits + +- **A layer must be self-contained.** The task image's libraries are whatever + that image ships, so a runtime binary that is dynamically linked works on + some images and not on others. Ship static binaries (the guest and `execd` + are) or put the runtime's libraries in the layer and point it at them. The + failure is easy to recognize: the guest log shows the sidecar exiting at + once with a loader error such as ``version `GLIBC_2.38' not found`` and + restarting with growing backoff, readiness never passes, and the actor + fails after the wakeup probe timeout. +- One container per actor: sidecars share the task image's user, environment + variables and filesystem, and have no resource limits of their own. +- `--sidecar` values are split on whitespace with no quoting. Arguments that + need quotes or shell features go in a wrapper script inside the layer. +- Port 80 belongs to the guest. A runtime that insists on it needs a wrapper + that moves it elsewhere. +- Sidecar output goes to the guest's log, line by line, prefixed with the + binary's name. There is no separate log stream yet. +- Layers are read-only. A runtime that writes next to its binary needs a + writable working directory passed as an argument. diff --git a/examples/runtime-layers/README.md b/examples/runtime-layers/README.md new file mode 100644 index 0000000..7687433 --- /dev/null +++ b/examples/runtime-layers/README.md @@ -0,0 +1,60 @@ +# Example: a second runtime as a layer + +Runs an unmodified task image with two injected runtimes: the ate-env guest at +`/ate`, and a static busybox at `/opt/web` standing in for a real runtime +daemon such as OpenSandbox's `execd`. The guest starts `busybox httpd` on port 44772 as a +sidecar and holds the actor's readiness until it answers. The mechanics are the +ones [docs/task-images/RUNTIMES.md](../../docs/task-images/RUNTIMES.md) +describes; only the binary differs. + +`demo.sh` registers the template, creates an environment, fetches a file from +the second runtime from inside the environment, optionally creates another +environment from a different image on the same base to show the layer is +inherited, and deletes what it created. + +```bash +kubectl -n ate-env port-forward svc/ate-env-api 7777:7777 & + +GUEST_IMAGE=/ate-env-guest@sha256: \ +SNAPSHOTS_BUCKET= \ +TASK_IMAGE=docker.io/library/python@sha256: \ +LAYER_IMAGE=docker.io/library/busybox@sha256: \ +OTHER_IMAGE=docker.io/library/node@sha256: \ +examples/runtime-layers/demo.sh +``` + +Digests for public images come from `crane digest --platform linux/amd64 +python:3.12-slim` or `docker manifest inspect`. The guest image must be built +from a revision of this repo that has the `-sidecar` flags. + +Use the `busybox:1.37-musl` tag, which is statically linked. The default +`busybox:1.37` is built against glibc and only runs on task images whose glibc +is at least as new as its own: it works inside `python:3.12-slim` and fails +inside `debian:bookworm-slim` with ``GLIBC_2.38' not found``, which is the +textbook case of why a runtime layer has to be self-contained. + +What the template looks like, as `ate-env manifest template` prints it: + +```yaml +containers: +- command: + - /ate/ko-app/ate-env-guest + - -sidecar + - /opt/web/bin/busybox httpd -f -p 44772 -h / + - -sidecar-readyz + - http://127.0.0.1:44772/etc/hostname + image: docker.io/library/python@sha256: + name: guest + readyz: {httpGet: {path: /readyz, port: 80}} + volumeMounts: + - {name: web, mountPath: /opt/web} + - {name: guest, mountPath: /ate} +volumes: +- {name: web, image: {reference: docker.io/library/busybox@sha256:}} +- {name: guest, image: {reference: /ate-env-guest@sha256:}} +``` + +To swap in a real runtime, change the three values: the layer image, the +sidecar command, and the readiness URL. For `execd` that is its image mounted +at `/opt/opensandbox`, `/opt/opensandbox/execd` with its flags, and +`http://127.0.0.1:44772/ready`. diff --git a/examples/runtime-layers/demo.sh b/examples/runtime-layers/demo.sh new file mode 100755 index 0000000..a080475 --- /dev/null +++ b/examples/runtime-layers/demo.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A second runtime injected as a layer, end to end. +# +# busybox stands in for a real runtime daemon (OpenSandbox execd, say): its +# image is mounted read-only at /opt/web, the guest starts `busybox httpd` on +# port 44772 as a sidecar and waits for it before reporting the actor ready. +# The task image is unmodified. The script registers the template, creates an +# environment from it, asks the second runtime for a file from inside the +# environment, creates a second environment from a different image on the +# same base to show the layer is inherited, and deletes both. +# +# Required: +# GUEST_IMAGE digest-pinned ate-env-guest image (built from this repo, +# at or after the -sidecar flags) +# SNAPSHOTS_BUCKET object-storage URL for actor snapshots +# TASK_IMAGE digest-pinned task image, e.g. docker.io/library/python@sha256:... +# LAYER_IMAGE digest-pinned STATIC busybox image (the 1.37-musl tag), +# e.g. docker.io/library/busybox@sha256:... +# Optional: +# OTHER_IMAGE a second digest-pinned image for the inheritance check +# TEMPLATE base template name (default layered-demo) +# ATESPACE atespace (default ate-env) +# SUBSTRATE_ENV_API ate-env-api address (default 127.0.0.1:7777; port-forward it) +# ATE_ENV, KUBECTL_ATE binaries (default: on PATH) +set -euo pipefail + +: "${GUEST_IMAGE:?set GUEST_IMAGE to the digest-pinned ate-env-guest image}" +: "${SNAPSHOTS_BUCKET:?set SNAPSHOTS_BUCKET to the snapshot object-storage URL}" +: "${TASK_IMAGE:?set TASK_IMAGE to a digest-pinned task image}" +: "${LAYER_IMAGE:?set LAYER_IMAGE to a digest-pinned busybox image}" +TEMPLATE=${TEMPLATE:-layered-demo} +ATESPACE=${ATESPACE:-ate-env} +export SUBSTRATE_ENV_API=${SUBSTRATE_ENV_API:-127.0.0.1:7777} +ATE_ENV=${ATE_ENV:-ate-env} +KUBECTL_ATE=${KUBECTL_ATE:-kubectl-ate} + +PORT=44772 +SIDECAR="/opt/web/bin/busybox httpd -f -p ${PORT} -h /" +READYZ="http://127.0.0.1:${PORT}/etc/hostname" + +step() { printf '\n== %s\n' "$*"; } + +step "register ${ATESPACE}/${TEMPLATE}: ${TASK_IMAGE} + guest + busybox layer" +"${ATE_ENV}" manifest template --template "${TEMPLATE}" --atespace "${ATESPACE}" \ + --task-image "${TASK_IMAGE}" --guest-image "${GUEST_IMAGE}" \ + --snapshots-bucket "${SNAPSHOTS_BUCKET}" \ + --layer "web=${LAYER_IMAGE}=/opt/web" \ + --sidecar "${SIDECAR}" \ + --sidecar-readyz "${READYZ}" | "${KUBECTL_ATE}" create actor-template -f - >/dev/null +"${KUBECTL_ATE}" get actor-template --atespace "${ATESPACE}" "${TEMPLATE}" + +ids=() +cleanup() { + for id in "${ids[@]:-}"; do + [ -n "${id}" ] && "${ATE_ENV}" delete "${id}" --atespace "${ATESPACE}" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +# wait_serving : the guest answers only once the sidecar's readiness URL +# has answered, so the first successful shell means both runtimes are up. +wait_serving() { + local id=$1 i + for i in $(seq 1 60); do + if "${ATE_ENV}" "${id}" shell 'true' >/dev/null 2>&1; then return 0; fi + sleep 3 + done + echo "environment ${id} did not start serving" >&2 + return 1 +} + +ID="layered-$(date +%s)" +ids+=("${ID}") +step "create ${ID} from ${TEMPLATE}" +t0=$(date +%s) +"${ATE_ENV}" create "${ID}" --atespace "${ATESPACE}" --template "${TEMPLATE}" +wait_serving "${ID}" +echo "serving after $(( $(date +%s) - t0 ))s" + +step "ask the second runtime from inside the environment" +"${ATE_ENV}" "${ID}" shell "/opt/web/bin/busybox wget -qO- http://127.0.0.1:${PORT}/etc/os-release | head -2; echo guest: \$(ls /ate/ko-app); echo layer: \$(ls /opt/web/bin | head -1)" + +if [ -n "${OTHER_IMAGE:-}" ]; then + ID2="layered2-$(date +%s)" + ids+=("${ID2}") + step "create ${ID2} from ${OTHER_IMAGE} with ${TEMPLATE} as the base (layer inherited)" + "${ATE_ENV}" create "${ID2}" --atespace "${ATESPACE}" --template "${TEMPLATE}" --image "${OTHER_IMAGE}" + "${KUBECTL_ATE}" get actor-template --atespace "${ATESPACE}" | grep "${TEMPLATE}-" || true + wait_serving "${ID2}" + "${ATE_ENV}" "${ID2}" shell "/opt/web/bin/busybox wget -qO- http://127.0.0.1:${PORT}/etc/os-release | head -1" +fi + +step "done; deleting" diff --git a/internal/apiservice/imagetemplate.go b/internal/apiservice/imagetemplate.go index 706b8de..f986ed4 100644 --- a/internal/apiservice/imagetemplate.go +++ b/internal/apiservice/imagetemplate.go @@ -36,6 +36,10 @@ const GuestVolumeName = "guest" // defaultGuestBinary is the guest's path inside its own image (ko layout). const defaultGuestBinary = "/ko-app/ate-env-guest" +// guestBinaryName is what a re-rootable command must start with: only the +// guest's own path moves under the mount, never a shell or another binary. +const guestBinaryName = "ate-env-guest" + // imageTemplateDigestLen is how many hex digits of the digest go into a // derived template's name: enough to make collisions a non-concern and short // enough to keep the name a valid k8s short name next to any base name. @@ -85,9 +89,11 @@ func ImageTemplateName(base, image string) (string, error) { // base can be either kind of template: one whose container image is the // guest itself (the shape `ate-env manifest template` writes), in which case // that image becomes the guest volume and the command is re-rooted under the -// mount; or one that already mounts the guest as an image volume, in which -// case only the container image changes. Everything else (worker selector, -// snapshots, sandbox config, resources, env, readiness) carries over. +// mount; or one that already mounts the guest as an image volume (named +// GuestVolumeName), in which case only the container image changes. +// Everything else carries over: worker selector, snapshots, sandbox config, +// resources, env, readiness, and any other volumes, such as extra runtime +// layers, together with the command-line flags that start them. func DeriveImageTemplate(base *ateapipb.ActorTemplate, name, image string) (*ateapipb.ActorTemplate, error) { if base == nil { return nil, errors.New("base template is required") @@ -135,10 +141,12 @@ func DeriveImageTemplate(base *ateapipb.ActorTemplate, name, image string) (*ate return tmpl, nil } -// guestVolume returns the template's image volume, or nil when it has none. +// guestVolume returns the template's guest image volume, or nil when the +// guest is the container image. Other image volumes (extra runtime layers) +// do not count: they can be present in either kind of base. func guestVolume(tmpl *ateapipb.ActorTemplate) *ateapipb.Volume { for _, v := range tmpl.GetVolumes() { - if v.GetImage() != nil { + if v.GetName() == GuestVolumeName && v.GetImage() != nil { return v } } @@ -154,9 +162,9 @@ func rerootCommand(command []string) ([]string, error) { if len(command) == 0 { return []string{path.Join(GuestMountPath, defaultGuestBinary)}, nil } - if !strings.HasPrefix(command[0], "/") { - return nil, fmt.Errorf("cannot re-root command %q under %s: it must start with an absolute path to the guest binary", - strings.Join(command, " "), GuestMountPath) + if !strings.HasPrefix(command[0], "/") || path.Base(command[0]) != guestBinaryName { + return nil, fmt.Errorf("cannot re-root command %q under %s: it must start with the absolute path of the guest binary (.../%s)", + strings.Join(command, " "), GuestMountPath, guestBinaryName) } out := append([]string(nil), command...) out[0] = path.Join(GuestMountPath, out[0]) diff --git a/internal/apiservice/imagetemplate_test.go b/internal/apiservice/imagetemplate_test.go index 9190eb4..06621cc 100644 --- a/internal/apiservice/imagetemplate_test.go +++ b/internal/apiservice/imagetemplate_test.go @@ -164,9 +164,50 @@ func TestDeriveImageTemplateRejects(t *testing.T) { if _, err := DeriveImageTemplate(modeABase(), "", testTaskImage); err == nil { t.Error("empty name accepted") } - wrapped := modeABase() - wrapped.Containers[0].Command = []string{"sh", "-c", "exec /ko-app/ate-env-guest"} - if _, err := DeriveImageTemplate(wrapped, "x", testTaskImage); err == nil || !strings.Contains(err.Error(), "re-root") { - t.Errorf("a shell-wrapped base command must be rejected, got %v", err) + for _, cmd := range [][]string{ + {"sh", "-c", "exec /ko-app/ate-env-guest"}, + {"/bin/sh", "-c", "exec /ko-app/ate-env-guest"}, + {"/opt/other/daemon"}, + } { + wrapped := modeABase() + wrapped.Containers[0].Command = cmd + if _, err := DeriveImageTemplate(wrapped, "x", testTaskImage); err == nil || !strings.Contains(err.Error(), "re-root") { + t.Errorf("command %v must be rejected (only the guest binary is re-rooted), got %v", cmd, err) + } + } +} + +func TestDeriveImageTemplateKeepsRuntimeLayers(t *testing.T) { + // A guest-image base that also carries a second runtime as an image + // volume and starts it as a guest sidecar: the layer and the flags + // survive derivation, and the guest volume is still added. + layerImage := "example.com/execd@sha256:" + strings.Repeat("c", 64) + base := modeABase() + base.Volumes = []*ateapipb.Volume{{Name: "execd", Image: &ateapipb.ImageVolumeSource{Reference: layerImage}}} + base.Containers[0].VolumeMounts = []*ateapipb.VolumeMount{{Name: "execd", MountPath: "/opt/opensandbox"}} + base.Containers[0].Command = []string{"/ko-app/ate-env-guest", "-sidecar", "/opt/opensandbox/execd --port 44772", "-sidecar-readyz", "http://127.0.0.1:44772/ready"} + + got, err := DeriveImageTemplate(base, "x", testTaskImage) + if err != nil { + t.Fatal(err) + } + if len(got.GetVolumes()) != 2 || got.GetVolumes()[0].GetName() != "execd" || got.GetVolumes()[1].GetName() != GuestVolumeName { + t.Errorf("volumes = %v, want the layer followed by the guest volume", got.GetVolumes()) + } + c := got.GetContainers()[0] + if len(c.GetVolumeMounts()) != 2 || c.GetVolumeMounts()[1].GetMountPath() != GuestMountPath { + t.Errorf("mounts = %v", c.GetVolumeMounts()) + } + want := "/ate/ko-app/ate-env-guest -sidecar /opt/opensandbox/execd --port 44772 -sidecar-readyz http://127.0.0.1:44772/ready" + if got := strings.Join(c.GetCommand(), " "); got != want { + t.Errorf("command = %q, want %q", got, want) + } + // Deriving again from the result (an already-injected base) changes only the image. + again, err := DeriveImageTemplate(got, "y", "example.com/other@sha256:"+strings.Repeat("d", 64)) + if err != nil { + t.Fatal(err) + } + if len(again.GetVolumes()) != 2 || strings.Join(again.GetContainers()[0].GetCommand(), " ") != want { + t.Errorf("second derivation altered layers or command: %v", again) } }