ateom: export telemetry through an atelet unix-socket OTLP relay

cmd/atelet/main.go

@@ -43,6 +43,7 @@ import (
 	"github.com/agent-substrate/substrate/internal/ateompath"
 	"github.com/agent-substrate/substrate/internal/credbundle"
 	"github.com/agent-substrate/substrate/internal/imagecache"
+	"github.com/agent-substrate/substrate/internal/otlprelay"
 	"github.com/agent-substrate/substrate/internal/proto/ateletpb"
 	"github.com/agent-substrate/substrate/internal/proto/ateompb"
 	"github.com/agent-substrate/substrate/internal/resources"
@@ -95,6 +96,8 @@ var (
 	showVersion  = pflag.Bool("version", false, "Print version and exit.")
 	logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.")
 
+	otlpRelaySocket = pflag.String("otlp-relay-socket", ateompath.AteletOTLPSocketPath(), "Unix socket to serve the OTLP relay on, which forwards the node's ateom telemetry to OTEL_EXPORTER_OTLP_ENDPOINT so worker pods need no network path to the collector. Empty disables the relay.")
+
 	drainDelay   = pflag.Duration("drain-delay", 0, "How long to keep accepting new RPCs after SIGTERM before starting the gRPC drain.")
 	drainTimeout = pflag.Duration("drain-timeout", 5*time.Minute, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.")
 )
@@ -150,6 +153,27 @@ func main() {
 		EnableHealthz: true,
 	})
 
+	// The OTLP relay lets the ateom pods on this node export telemetry over a
+	// unix socket instead of their own network (see internal/otlprelay). Started
+	// early: an ateom that finds no socket at startup falls back to exporting
+	// directly for its whole life, so the socket should exist before any worker
+	// pod on this node boots.
+	if relay, err := otlprelay.NewServer(ctx, *otlpRelaySocket); err != nil {
+		slog.ErrorContext(ctx, "Failed to create the OTLP relay; ateoms will export directly", slog.Any("err", err))
+	} else if relay != nil {
+		// Deferred rather than tied to the drain: the relay carries other
+		// processes' telemetry, so it should outlive atelet's own RPC serving
+		// and stay up while the ateoms it serves are themselves shutting down.
+		defer relay.Stop()
+		go func() {
+			if err := relay.Serve(ctx); err != nil {
+				// Not fatal: atelet's actual job does not depend on the relay,
+				// and the ateoms fall back to exporting directly.
+				slog.ErrorContext(ctx, "OTLP relay stopped", slog.Any("err", err))
+			}
+		}()
+	}
+
 	ateomDialer := &AteomDialer{
 		conns: lru.New(256),
 	}

cmd/ateom-gvisor/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/agent-substrate/substrate/internal/atunnel"
 	"github.com/agent-substrate/substrate/internal/contextlogging"
 	"github.com/agent-substrate/substrate/internal/imagecache"
+	"github.com/agent-substrate/substrate/internal/otlprelay"
 	"github.com/agent-substrate/substrate/internal/proto/ateompb"
 	"github.com/agent-substrate/substrate/internal/readyz"
 	"github.com/agent-substrate/substrate/internal/resources"
@@ -66,6 +67,9 @@ var (
 	showVersion  = pflag.Bool("version", false, "Print version and exit.")
 	logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.")
 
+	otlpRelaySocket = pflag.String("otlp-relay-socket", ateompath.AteletOTLPSocketPath(),
+		"Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.")
+
 	reapLock sync.RWMutex
 )
 
@@ -102,16 +106,39 @@ func do(ctx context.Context) error {
 	slog.InfoContext(ctx, "ateom booting")
 
 	const serviceName = "ateom-gvisor"
+	// Export through atelet's node-local relay when it is there, so telemetry
+	// never touches the worker pod's network. A nil conn means it is not, and
+	// both providers fall back to dialing the collector directly.
+	//
+	// A relay that cannot be dialed is logged rather than fatal, matching both
+	// ends of the same decision: Dial already treats an absent socket as a
+	// fallback rather than an error, and atelet logs and keeps going when it
+	// cannot serve the relay at all. What is lost here is the node-local export
+	// path, not the ateom's ability to run actors, and failing the worker pod
+	// over its telemetry route would turn a misconfigured flag into an outage.
+	relayConn, err := otlprelay.Dial(ctx, *otlpRelaySocket)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to connect to the OTLP relay; exporting telemetry directly over the pod network",
+			slog.String("socket", *otlpRelaySocket), slog.Any("err", err))
+	}
+	if relayConn != nil {
+		defer relayConn.Close()
+	}
+
 	tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
-		ServiceName: serviceName,
-		Sampling:    serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
+		ServiceName:  serviceName,
+		Sampling:     serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
+		ExporterConn: relayConn,
+		// So the spans say which path they took, including when relayConn is nil
+		// because the dial above failed and this ateom is exporting directly.
+		RelayCapable: true,
 	})
 	if err != nil {
 		serverboot.Fatal(ctx, "Failed to initialize tracing", err)
 	}
 	defer serverboot.ShutdownProvider("TracerProvider", tp.Shutdown)
 
-	mp, err := serverboot.InitMetricsPushOnly(ctx, serviceName)
+	mp, err := serverboot.InitMetricsPushOnlyVia(ctx, serviceName, relayConn)
 	if err != nil {
 		serverboot.Fatal(ctx, "Failed to initialize metrics", err)
 	}

cmd/ateom-microvm/main.go

@@ -42,6 +42,7 @@ import (
 	"github.com/agent-substrate/substrate/internal/ateomnet"
 	"github.com/agent-substrate/substrate/internal/ateompath"
 	"github.com/agent-substrate/substrate/internal/atunnel"
+	"github.com/agent-substrate/substrate/internal/otlprelay"
 	"github.com/agent-substrate/substrate/internal/proto/ateompb"
 	"github.com/agent-substrate/substrate/internal/serverboot"
 	"github.com/agent-substrate/substrate/internal/version"
@@ -60,6 +61,9 @@ var (
 	showVersion  = flag.Bool("version", false, "Print version and exit.")
 	logLevelFlag = flag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.")
 
+	otlpRelaySocket = flag.String("otlp-relay-socket", ateompath.AteletOTLPSocketPath(),
+		"Unix socket of atelet's OTLP relay to export telemetry through, keeping it off the pod network. Empty, or absent at startup, exports directly to OTEL_EXPORTER_OTLP_ENDPOINT instead.")
+
 	atunnelListenAddress       = flag.String("atunnel-listen-address", "0.0.0.0:443", "Address for actor ingress HTTPS")
 	workerCredentialBundle     = flag.String("atunnel-credential-bundle", "/run/podidentity.podcert.ate.dev/credential-bundle.pem", "Worker Pod credential bundle used by atunnel for inbound serving and outbound mTLS")
 	podIdentityTrustBundle     = flag.String("atunnel-trust-bundle", "/run/podidentity.podcert.ate.dev/trust-bundle.pem", "Pod identity trust bundle used for router clients and the node-local atelet")
@@ -101,16 +105,39 @@ func do(ctx context.Context) error {
 	slog.InfoContext(ctx, "ateom-microvm booting", slog.String("version", version.String()))
 
 	const serviceName = "ateom-microvm"
+	// Export through atelet's node-local relay when it is there, so telemetry
+	// never touches the worker pod's network. A nil conn means it is not, and
+	// both providers fall back to dialing the collector directly.
+	//
+	// A relay that cannot be dialed is logged rather than fatal, matching both
+	// ends of the same decision: Dial already treats an absent socket as a
+	// fallback rather than an error, and atelet logs and keeps going when it
+	// cannot serve the relay at all. What is lost here is the node-local export
+	// path, not the ateom's ability to run actors, and failing the worker pod
+	// over its telemetry route would turn a misconfigured flag into an outage.
+	relayConn, err := otlprelay.Dial(ctx, *otlpRelaySocket)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to connect to the OTLP relay; exporting telemetry directly over the pod network",
+			slog.String("socket", *otlpRelaySocket), slog.Any("err", err))
+	}
+	if relayConn != nil {
+		defer relayConn.Close()
+	}
+
 	tp, err := serverboot.InitTracing(ctx, serverboot.TracingOptions{
-		ServiceName: serviceName,
-		Sampling:    serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
+		ServiceName:  serviceName,
+		Sampling:     serverboot.ResolveTraceSampling(ctx, serverboot.ParentRatioSampling(serverboot.ControlPlaneTraceRatio)),
+		ExporterConn: relayConn,
+		// So the spans say which path they took, including when relayConn is nil
+		// because the dial above failed and this ateom is exporting directly.
+		RelayCapable: true,
 	})
 	if err != nil {
 		serverboot.Fatal(ctx, "Failed to initialize tracing", err)
 	}
 	defer serverboot.ShutdownProvider("TracerProvider", tp.Shutdown)
 
-	mp, err := serverboot.InitMetricsPushOnly(ctx, serviceName)
+	mp, err := serverboot.InitMetricsPushOnlyVia(ctx, serviceName, relayConn)
 	if err != nil {
 		serverboot.Fatal(ctx, "Failed to initialize metrics", err)
 	}

docs/observability.md

@@ -198,7 +198,9 @@ To visualize traces locally:
    ```
    The kind overlay pins `ateapi` to `parentbased_always_on`, so API calls show up even without `--trace`; the flag additionally prints the trace ID and forces sampling on every hop.
 
-4. **Search and Inspect**: Copy the printed Trace ID from the CLI output and paste it into the Jaeger search box (top right), or select `ateapi` or `atelet` under the **Service** dropdown and click **Find Traces** to inspect detailed call stacks, DB transactions, state updates, and worker pod handoffs.
+4. **Search and Inspect**: Copy the printed Trace ID from the CLI output and paste it into the Jaeger search box (top right), or select `ateapi`, `atelet`, or `ateom-gvisor` under the **Service** dropdown and click **Find Traces** to inspect detailed call stacks, DB transactions, state updates, and worker pod handoffs.
+
+> ateom carries no manual spans — its only instrumentation is the `otelgrpc` interceptor on the gRPC surface `atelet` calls. So it produces a span for an actor lifecycle operation (`suspend`, `resume`) and nothing at all for a read like `kubectl ate get actor`. Its sampler is parent based, so a lifecycle command is traced end to end into ateom whenever `ateapi` roots a sampled trace, which the kind overlay makes unconditional; the per-component ratio never enters into it. `ateom-gvisor` appearing here at all is also the sign its [OTLP relay](#the-ateom-otlp-relay) is working, since that is the only path its telemetry has out of the worker pod.
 
 > **Developer Guide:** For detailed instructions on configuring OpenTelemetry tracer providers, middleware, and exporters in your servers or clients, please refer to the [Tracing Best Practices](dev/best-practices/tracing.md) guide.
 
@@ -221,6 +223,24 @@ Telemetry is emitted the same way everywhere; only the backend differs between a
 >
 > ateom workers don't read the ConfigMap at all — `ate-controller` copies the value into each worker pod at creation. A new endpoint reaches them only once the controller itself restarts, and that restart then rolls every WorkerPool Deployment, replacing the running workers along with the actors on them.
 
+### The ateom OTLP relay
+
+ateom is the one component that does not talk to the collector directly. It exports over a unix socket at `/var/lib/ateom-gvisor/atelet-otlp.sock`, which `atelet` serves and forwards to the collector on the node's network ([`internal/otlprelay`](../internal/otlprelay)):
+
+```
+ateom ──OTLP/gRPC over unix socket──► atelet relay ──OTLP/gRPC──► collector
+```
+
+The socket sits in the `BasePath` hostPath already mounted into both, so nothing new is mounted. `atelet` is a DaemonSet, so every ateom on a node shares one relay, and the many per-pod collector connections collapse into one per node. Four things motivate it: the worker pod runs untrusted agent code and will not need egress to the collector once direct fallback is phased out; the connection count drops; ateom's own telemetry stays clear of the transparent egress redirect it installs for the actor; and `atelet` outlives the worker pod, so spans still queued at teardown are not lost with it.
+
+The relay is best-effort. If the socket is absent when ateom starts — `atelet` not up yet, `--otlp-relay-socket=""`, or no collector configured for the relay to forward to — ateom logs it and exports directly to `OTEL_EXPORTER_OTLP_ENDPOINT` as before. That fallback is decided once at startup, not per export, and is stamped on telemetry as the `substrate.otlp.relay` resource attribute (`relay` vs `direct`).
+
+> **Note on Network Egress Lockdown:** Complete network policy lockdown of worker pod egress to the collector is planned as a Phase 2 milestone once the relay path is fully proven and direct fallback is deprecated. While the fallback path remains active, worker pods retain network egress to the collector and `ate-controller` continues to inject `OTEL_EXPORTER_OTLP_ENDPOINT`.
+
+For verified ateom sources, the relay forwards each request verbatim rather than decoding and re-exporting, which is what keeps every ateom its own service in Jaeger/GCP Trace instead of being absorbed into `atelet`'s. `ate-controller` injects `k8s.pod.name`, `k8s.namespace.name`, `k8s.pod.uid`, and `service.instance.id` directly into `OTEL_RESOURCE_ATTRIBUTES` via the Kubernetes Downward API; because the relay preserves resources verbatim, Kubernetes attributes remain intact even though the TCP connection to the collector originates from `atelet` rather than the worker pod IP (bypassing reliance on collector-side IP-based `k8sattributes` enrichment).
+
+Verbatim forwarding is restricted to known ateom sources and refuses anything else with `PermissionDenied`. Actor telemetry is what that excludes: actors share a hostname (`runsc`) and an interior IP, so their series merge unless identity is injected from outside the actor ([#761](https://github.com/agent-substrate/substrate/issues/761)) — a rewrite, which will be implemented as an explicit rewriting path alongside this forwarder.
+
 ---
 
 ## 5. Dashboards

go.mod

@@ -47,6 +47,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.43.0
 	go.opentelemetry.io/otel/sdk/metric v1.43.0
 	go.opentelemetry.io/otel/trace v1.43.0
+	go.opentelemetry.io/proto/otlp v1.10.0
 	golang.org/x/oauth2 v0.36.0
 	golang.org/x/sync v0.21.0
 	golang.org/x/sys v0.46.0
@@ -168,7 +169,6 @@ require (
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect

internal/ateompath/ateompath.go

@@ -53,6 +53,28 @@ func GVisorReleaseDir(sha256 string) string {
 	return filepath.Join(StaticFilesDir, "gvisor-"+sha256)
 }
 
+// AteletOTLPSocketPath is the node-scoped unix socket atelet serves the OTLP
+// relay on (see internal/otlprelay). It is node-scoped rather than per-pod
+// because every ateom on the node pushes into the same relay: atelet is a
+// DaemonSet, so one socket collapses N per-pod collector connections into one
+// per-node connection.
+//
+// It sits directly under BasePath, which is the host directory already mounted
+// at the same path into atelet and into every ateom pod, so no new volume is
+// needed for ateom to reach it. Note that BasePath is mounted writable
+// (workerpool_apply.go) and shared with CredentialBrokerSocket and the image
+// cache, so a worker pod can unlink or replace this socket. Confining
+// atelet-owned sockets to a subdirectory mounted read-only would be an
+// improvement, but it is a property of the whole BasePath mount rather than of
+// this socket — a read-only subdir needs its own volume and mount, and the pod
+// keeps CAP_SYS_ADMIN. Tracked separately rather than solved here.
+func AteletOTLPSocketPath() string {
+	return filepath.Join(
+		BasePath,
+		"atelet-otlp.sock",
+	)
+}
+
 func AteomPath(podUID string) string {
 	return filepath.Join(
 		BasePath,

internal/ateompath/ateompath_test.go

@@ -47,6 +47,28 @@ func TestAteomSocketPathLimits(t *testing.T) {
 	}
 }
 
+func TestAteletOTLPSocketPath(t *testing.T) {
+	sockPath := AteletOTLPSocketPath()
+
+	// Unix domain socket path limit is 107 bytes (108 with NUL terminator)
+	const maxUnixSocketLen = 107
+	if len(sockPath) > maxUnixSocketLen {
+		t.Errorf("socket path length %d exceeds max allowed length %d: %q", len(sockPath), maxUnixSocketLen, sockPath)
+	}
+
+	// It must sit under BasePath: that is the host directory already mounted at
+	// the same path into atelet and into every ateom pod, which is the whole
+	// reason the relay needs no new volume.
+	if !strings.HasPrefix(sockPath, BasePath+"/") {
+		t.Errorf("AteletOTLPSocketPath() = %q, want it under %q so ateom and atelet see the same file", sockPath, BasePath)
+	}
+
+	// Node-scoped, so it must not collide with any per-pod ateom socket.
+	if other := AteomSocketPath("123e4567-e89b-12d3-a456-426614174000"); sockPath == other {
+		t.Errorf("AteletOTLPSocketPath() collides with AteomSocketPath: %q", sockPath)
+	}
+}
+
 func TestAteomPathUniqueness(t *testing.T) {
 	uid1 := "123e4567-e89b-12d3-a456-426614174000"
 	uid2 := "987f6543-e21b-32d1-b654-246614174111"

internal/otlprelay/client.go

@@ -0,0 +1,68 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package otlprelay
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"os"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// Dial opens the ateom half of the relay: a gRPC connection over atelet's unix
+// socket, to be handed to the OTLP exporters via serverboot's ExporterConn.
+//
+// It returns (nil, nil) when sockPath is empty or absent, which the caller reads
+// as "export directly instead". The existence check is what makes the fallback
+// deterministic at startup: grpc.NewClient is lazy, so a connection to a missing
+// socket would be created happily and only fail later, per export, with the
+// telemetry already lost. Losing spans is not worth failing ateom over either,
+// hence a fallback rather than an error.
+//
+// The connection is plaintext by design. A unix socket cannot leave the node, so
+// there is no transport to protect; access is controlled by the socket's file
+// permissions instead (see socketMode).
+func Dial(ctx context.Context, sockPath string) (*grpc.ClientConn, error) {
+	if sockPath == "" {
+		return nil, nil
+	}
+	if err := validateSocketPath(sockPath); err != nil {
+		return nil, err
+	}
+	if _, err := os.Stat(sockPath); err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			slog.WarnContext(ctx, "OTLP relay socket absent, exporting telemetry directly over the pod network",
+				slog.String("socket", sockPath))
+			return nil, nil
+		}
+		return nil, fmt.Errorf("while checking the OTLP relay socket %q: %w", sockPath, err)
+	}
+
+	// gRPC resolves a "unix://" target to a unix socket dialer natively, so the
+	// OTLP exporters above this connection are unchanged: OTLP is gRPC, and gRPC
+	// needs only a reliable byte stream.
+	conn, err := grpc.NewClient("unix://"+sockPath,
+		grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return nil, fmt.Errorf("while dialing the OTLP relay socket %q: %w", sockPath, err)
+	}
+	slog.InfoContext(ctx, "Exporting telemetry through the atelet OTLP relay", slog.String("socket", sockPath))
+	return conn, nil
+}