From 77d168a8ed670aa16198d6a6cbe7a60768d70694 Mon Sep 17 00:00:00 2001 From: "Lenny (Temporal)" Date: Wed, 9 Sep 2026 16:24:13 -0700 Subject: [PATCH] Document Cloud Run OTel and Worker identity setup --- .../workers/serverless-workers/cloud-run.mdx | 105 +++++++++++- .../workers/serverless-workers/cloud-run.mdx | 123 +++++++++++++- .../workers/serverless-workers/cloud-run.mdx | 152 +++++++++++++++++- .../workers/serverless-workers/cloud-run.mdx | 121 +++++++++++++- .../serverless-workers/cloud-run/index.mdx | 39 ++--- 5 files changed, 505 insertions(+), 35 deletions(-) diff --git a/docs/develop/dotnet/workers/serverless-workers/cloud-run.mdx b/docs/develop/dotnet/workers/serverless-workers/cloud-run.mdx index 465e5982af..b7114211de 100644 --- a/docs/develop/dotnet/workers/serverless-workers/cloud-run.mdx +++ b/docs/develop/dotnet/workers/serverless-workers/cloud-run.mdx @@ -23,7 +23,7 @@ import { ReleaseNoteHeader } from '@site/src/components'; On a [GCP Cloud Run worker pool](https://cloud.google.com/run/docs/resource-model#worker-pools), you run a standard long-lived Temporal Worker. Register Workflows and Activities the same way you would with any other .NET Worker, and Temporal Cloud scales the pool up and down as work arrives and drains. -A Cloud Run Worker needs no Cloud Run-specific package. +A Cloud Run Worker needs no Cloud Run-specific runtime or handler. The one addition to a standard Worker is [Worker Versioning](/worker-versioning), which is required for Serverless Workers. For the end-to-end deployment guide covering the Worker Pool, IAM, and compute configuration, see [Deploy a Serverless Worker on GCP Cloud Run](/production-deployment/worker-deployments/serverless-workers/cloud-run). @@ -135,7 +135,104 @@ public static string Process(IReadOnlyList items) For how scale-in decisions are made, see [Serverless Workers on GCP Cloud Run](/serverless-workers/cloud-run#lifecycle). -## Add observability {/* #add-observability */} +## Configure OpenTelemetry {/* #opentelemetry */} + +Configure the .NET OpenTelemetry exporter and the Temporal Runtime to send telemetry by OTLP to `localhost:4317`. The following code sample configures the Worker only. You must separately run an OTLP-compatible receiver at that address. In a Cloud Run Worker Pool, run that receiver as a sidecar. + + +[src/OpenTelemetry/CoreSdkForwarding/Program.cs](https://github.com/temporalio/samples-dotnet/blob/main/src/OpenTelemetry/CoreSdkForwarding/Program.cs) +```cs +var resourceBuilder = ResourceBuilder. + CreateDefault(). + AddService("TemporalioSamples.OpenTelemetry", serviceInstanceId: instanceId); + +using var tracerProvider = Sdk. + CreateTracerProviderBuilder(). + SetResourceBuilder(resourceBuilder). + AddSource(TracingInterceptor.ClientSource.Name, TracingInterceptor.WorkflowsSource.Name, TracingInterceptor.ActivitiesSource.Name). + AddOtlpExporter(). + Build(); + +// Shared by the client and by Core SDK log forwarding below. The OpenTelemetry provider exports +// logs to the dashboard alongside the traces and metrics. +using var loggerFactory = LoggerFactory.Create(builder => + builder. + AddSimpleConsole(options => options.TimestampFormat = "[HH:mm:ss] "). + AddOpenTelemetry(options => + { + options.SetResourceBuilder(resourceBuilder); + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + options.AddOtlpExporter(); + }). + SetMinimumLevel(LogLevel.Information)); + +// Create a client to localhost on default namespace +var connectOptions = ClientEnvConfig.LoadClientConnectOptions(); +connectOptions.TargetHost ??= "localhost:7233"; +connectOptions.LoggerFactory = loggerFactory; +connectOptions.Interceptors = new[] { new TracingInterceptor() }; +connectOptions.Runtime = new TemporalRuntime(new TemporalRuntimeOptions() +{ + Telemetry = new TelemetryOptions() + { + Metrics = new MetricsOptions() + { + OpenTelemetry = new OpenTelemetryOptions() + { + Url = new Uri("http://localhost:4317"), + }, + }, + Logging = new LoggingOptions() + { + // Core SDK logs default to WARN; lowered here so there is more to see. + Filter = new TelemetryFilterOptions(core: TelemetryFilterOptions.Level.Info), + + // The Core SDK writes its logs to the console itself unless Forwarding is set, in + // which case they go to this ILogger instead. + Forwarding = new LogForwardingOptions(loggerFactory.CreateLogger("Temporalio.Core")), + }, + }, +}); +var client = await TemporalClient.ConnectAsync(connectOptions); +``` + + +The [OpenTelemetry sample](https://github.com/temporalio/samples-dotnet/tree/main/src/OpenTelemetry) shows the tracing, metrics, and log-export configuration. Its Docker Compose file runs the .NET Aspire Dashboard locally and exposes its OTLP endpoint on port `4317`; it does not define a Cloud Run sidecar. Configure your Cloud Run sidecar to export the received telemetry to your backend. + +## Set a Worker identity {/* #worker-identity */} + +Use `WorkerIdPlugin` to identify each Worker instance as `@`. The plugin reads Cloud Run environment variables and instance metadata when the Client connects, then Workers created from that Client inherit the identity. + + +[src/Gcp/CloudRun/WorkerId/Program.cs](https://github.com/temporalio/samples-dotnet/blob/main/src/Gcp/CloudRun/WorkerId/Program.cs) +```cs +var address = GetEnvironmentVariable("TEMPORAL_ADDRESS") ?? "localhost:7233"; +var temporalNamespace = GetEnvironmentVariable("TEMPORAL_NAMESPACE") ?? "default"; +var taskQueue = GetEnvironmentVariable("TEMPORAL_TASK_QUEUE") ?? "cloud-run-worker-sample"; + +using var loggerFactory = LoggerFactory.Create(builder => builder. + AddSimpleConsole(options => options.TimestampFormat = "[HH:mm:ss] "). + SetMinimumLevel(LogLevel.Information)); +var logger = loggerFactory.CreateLogger("CloudRunWorkerId"); + +// Register the Cloud Run plugin once on the client. At connect time it reads the Cloud Run instance +// id from the metadata server, and the worker pool / service name and revision from the environment, +// then sets the client Identity to the worker identity "{instanceId}@{revision}" (unless one was +// already configured). Every worker created from this client inherits that identity. The plugin only +// sets the worker identity; it does not configure anything else. +// +// NOTE: this requires the process to be running on a Cloud Run worker pool or service. Running it +// elsewhere throws at connect time because the metadata server is unreachable. +var clientOptions = new TemporalClientConnectOptions(address) +{ + Namespace = temporalNamespace, + LoggerFactory = loggerFactory, + Plugins = new[] { new WorkerIdPlugin() }, +}; + +var client = await TemporalClient.ConnectAsync(clientOptions); +``` + -A Cloud Run Worker emits the same traces and metrics as a Worker anywhere else. -For how to configure metrics export and OpenTelemetry tracing interceptors, see [Observability - .NET SDK](/develop/dotnet/platform/observability) and the [SDK metrics reference](/references/sdk-metrics). +The plugin requires the Cloud Run metadata server, so it fails when the Worker runs outside Cloud Run. For a complete example, see the [.NET Cloud Run Worker Id sample](https://github.com/temporalio/samples-dotnet/pull/219). diff --git a/docs/develop/go/workers/serverless-workers/cloud-run.mdx b/docs/develop/go/workers/serverless-workers/cloud-run.mdx index 101b6469ae..34e1f705c3 100644 --- a/docs/develop/go/workers/serverless-workers/cloud-run.mdx +++ b/docs/develop/go/workers/serverless-workers/cloud-run.mdx @@ -23,7 +23,7 @@ import { ReleaseNoteHeader } from '@site/src/components'; On a [GCP Cloud Run worker pool](https://cloud.google.com/run/docs/resource-model#worker-pools), you run a standard long-lived Temporal Worker. Register Workflows and Activities the same way you would with any other Go Worker, and Temporal Cloud scales the pool up and down as work arrives and drains. -A Cloud Run Worker needs no Cloud Run-specific package. +A Cloud Run Worker needs no Cloud Run-specific runtime or handler. The one addition to a standard Worker is [Worker Versioning](/worker-versioning), which is required for Serverless Workers. For the end-to-end deployment guide covering the Worker Pool, IAM, and compute configuration, see [Deploy a Serverless Worker on GCP Cloud Run](/production-deployment/worker-deployments/serverless-workers/cloud-run). @@ -112,7 +112,122 @@ func MyActivity(ctx context.Context, input MyInput) (string, error) { For how scale-in decisions are made, see [Serverless Workers on GCP Cloud Run](/serverless-workers/cloud-run#lifecycle). -## Add observability {/* #add-observability */} +## Configure OpenTelemetry {/* #opentelemetry */} -A Cloud Run Worker emits the same traces and metrics as a Worker anywhere else. -For how to configure metrics export and OpenTelemetry tracing interceptors, see [Observability - Go SDK](/develop/go/platform/observability) and the [SDK metrics reference](/references/sdk-metrics). +Run an OpenTelemetry Collector as a sidecar in the Worker Pool. The Cloud Run OpenTelemetry plugin exports metrics and traces by OTLP gRPC to the Collector at `localhost:4317`. By default, it derives the service name from `OTEL_SERVICE_NAME`, `CLOUD_RUN_WORKER_POOL`, or `K_SERVICE`. + +Create the plugin before connecting, then add it to the Client options. Client plugins that implement `worker.Plugin` also apply to Workers created from that Client: + + +[gcp/cloudrun/otel/worker/main.go](https://github.com/temporalio/samples-go/blob/gcp-cloud-run-otel/gcp/cloudrun/otel/worker/main.go) +```go +// ... + otelPlugin, err := otel.NewPlugin(ctx, otel.PluginOptions{}) + if err != nil { + log.Fatalln("Unable to create OpenTelemetry plugin", err) + } + + // Load the Temporal connection from the environment (see temporal.toml or the + // TEMPORAL_* environment variables) and install the plugin. Client plugins + // that also implement worker.Plugin are applied to workers automatically. + clientOptions, err := envconfig.LoadDefaultClientOptions() + if err != nil { + log.Fatalln("Unable to load Temporal client options", err) + } + clientOptions.Plugins = append(clientOptions.Plugins, otelPlugin) + + c, err := client.Dial(clientOptions) + if err != nil { + log.Fatalln("Unable to create Temporal client", err) + } +``` + + +Configure the Collector sidecar to receive OTLP gRPC on `localhost:4317`, export traces to Google Cloud, and export metrics to Google Managed Service for Prometheus: + + +[gcp/cloudrun/otel/otel-collector-config.yaml](https://github.com/temporalio/samples-go/blob/main/gcp/cloudrun/otel/otel-collector-config.yaml) +```yaml +# Google-Built OpenTelemetry Collector configuration for a Cloud Run worker pool +# sidecar. The Temporal worker exports OTLP gRPC to localhost:4317; this collector +# adds GCP resource attributes and exports to Google Cloud. +# +# Metrics use googlemanagedprometheus with NO batch processor: batching can merge +# periodic and forced-shutdown snapshots of the same cumulative series into a +# single Google Monitoring write, which Managed Service for Prometheus rejects as +# duplicate data. Traces may be batched independently. +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + +processors: + # Guard the sidecar against unbounded memory growth. + memory_limiter: + check_interval: 1s + limit_percentage: 65 + spike_limit_percentage: 20 + # Detect Google Cloud resource attributes (project, region, revision, ...). + resourcedetection: + detectors: [gcp] + timeout: 10s + # Batch is used ONLY for traces. + batch: + send_batch_size: 200 + timeout: 5s + +exporters: + debug: + googlemanagedprometheus: + googlecloud: + +extensions: + # Health check used as the container startup probe. Bind on all interfaces + # so the Cloud Run startup probe can reach it. + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + pipelines: + # No batch processor in the metrics pipeline. + metrics: + receivers: [otlp] + processors: [memory_limiter, resourcedetection] + exporters: [googlemanagedprometheus, debug] + traces: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, batch] + exporters: [googlecloud, debug] + telemetry: + logs: + level: info +``` + + +On shutdown, stop the Worker and call `otelPlugin.Shutdown` with a deadline shorter than Cloud Run's termination window so telemetry can flush. For a Collector configuration that exports traces to Google Cloud and metrics to Google Managed Service for Prometheus, see the [Go Cloud Run OpenTelemetry sample](https://github.com/temporalio/samples-go/pull/528). + +## Set a Worker identity {/* #worker-identity */} + +Use the Cloud Run Worker Id plugin to identify each Worker instance as `@`. The plugin reads Cloud Run environment variables and instance metadata once when the Client connects, then Workers created from that Client inherit the identity. + + +[gcp/cloudrun/workerid/worker/main.go](https://github.com/temporalio/samples-go/blob/main/gcp/cloudrun/workerid/worker/main.go) +```go +// ... + plugin := workerid.NewPlugin(workerid.PluginOptions{}) + clientOptions := client.Options{ + HostPort: getenv("TEMPORAL_ADDRESS", client.DefaultHostPort), + Namespace: getenv("TEMPORAL_NAMESPACE", client.DefaultNamespace), + Plugins: []client.Plugin{plugin}, + } + + c, err := client.Dial(clientOptions) + if err != nil { + log.Fatalf("Unable to create Temporal client (is this running on a Cloud Run worker pool or service?): %v", err) + } +``` + + +The plugin requires the Cloud Run metadata server, so it fails when the Worker runs outside Cloud Run. For a complete example, see the [Go Cloud Run Worker Id sample](https://github.com/temporalio/samples-go/pull/531). diff --git a/docs/develop/java/workers/serverless-workers/cloud-run.mdx b/docs/develop/java/workers/serverless-workers/cloud-run.mdx index bc718fa966..54905a107a 100644 --- a/docs/develop/java/workers/serverless-workers/cloud-run.mdx +++ b/docs/develop/java/workers/serverless-workers/cloud-run.mdx @@ -23,7 +23,7 @@ import { ReleaseNoteHeader } from '@site/src/components'; On a [GCP Cloud Run worker pool](https://cloud.google.com/run/docs/resource-model#worker-pools), you run a standard long-lived Temporal Worker. Register Workflows and Activities the same way you would with any other Java Worker, and Temporal Cloud scales the pool up and down as work arrives and drains. -A Cloud Run Worker needs no Cloud Run-specific package. +A Cloud Run Worker needs no Cloud Run-specific runtime or handler. The one addition to a standard Worker is [Worker Versioning](/worker-versioning), which is required for Serverless Workers. For the end-to-end deployment guide covering the Worker Pool, IAM, and compute configuration, see [Deploy a Serverless Worker on GCP Cloud Run](/production-deployment/worker-deployments/serverless-workers/cloud-run). @@ -149,7 +149,151 @@ public class GreetingActivitiesImpl implements GreetingActivities { For how scale-in decisions are made, see [Serverless Workers on GCP Cloud Run](/serverless-workers/cloud-run#lifecycle). -## Add observability {/* #add-observability */} +## Configure OpenTelemetry {/* #opentelemetry */} -A Cloud Run Worker emits the same traces and metrics as a Worker anywhere else. -For how to configure metrics export and OpenTelemetry tracing interceptors, see [Observability - Java SDK](/develop/java/platform/observability) and the [SDK metrics reference](/references/sdk-metrics). +Run an OpenTelemetry Collector as a sidecar in the Worker Pool. The Cloud Run OpenTelemetry plugin exports metrics and traces by OTLP gRPC to the Collector at `localhost:4317`. It configures the endpoint and service name from Cloud Run defaults. + +Create the plugin and register it on `WorkflowServiceStubsOptions` before you create the Client: + + +[gcp/cloud-run/opentelemetry/src/main/java/io/temporal/samples/gcp/cloudrun/CloudRunWorker.java](https://github.com/temporalio/samples-java/blob/gcp-cloud-run-otel/gcp/cloud-run/opentelemetry/src/main/java/io/temporal/samples/gcp/cloudrun/CloudRunWorker.java) +```java +ClientConfigProfile profile = ClientConfigProfile.load(); +CloudRunOpenTelemetryPlugin telemetryPlugin = CloudRunOpenTelemetryPlugin.newBuilder().build(); + +WorkflowServiceStubsOptions serviceOptions = + WorkflowServiceStubsOptions.newBuilder(profile.toWorkflowServiceStubsOptions()) + .setPlugins(telemetryPlugin) + .build(); +WorkflowServiceStubs service = WorkflowServiceStubs.newServiceStubs(serviceOptions); +``` + + +Configure the Collector sidecar to receive OTLP gRPC on `localhost:4317`, export traces to Google Cloud, and export metrics to Google Managed Service for Prometheus: + + +[gcp/cloud-run/opentelemetry/collector-config.yaml](https://github.com/temporalio/samples-java/blob/main/gcp/cloud-run/opentelemetry/collector-config.yaml) +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + +processors: + # Batch traces for throughput. Do not add this processor to the cumulative metrics pipeline: + # a shutdown flush can otherwise be batched with a recent periodic export of the same series. + batch/traces: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 5s + memory_limiter: + # This is the collector's memory polling cadence, not the SDK metric export interval. + check_interval: 1s + limit_percentage: 65 + spike_limit_percentage: 20 + resourcedetection: + detectors: [gcp] + timeout: 10s + # Avoid collisions with labels that Google Managed Service for Prometheus adds. + transform/collision: + metric_statements: + - context: datapoint + statements: + - set(attributes["exported_location"], attributes["location"]) + - delete_key(attributes, "location") + - set(attributes["exported_cluster"], attributes["cluster"]) + - delete_key(attributes, "cluster") + - set(attributes["exported_namespace"], attributes["namespace"]) + - delete_key(attributes, "namespace") + - set(attributes["exported_job"], attributes["job"]) + - delete_key(attributes, "job") + - set(attributes["exported_instance"], attributes["instance"]) + - delete_key(attributes, "instance") + - set(attributes["exported_project_id"], attributes["project_id"]) + - delete_key(attributes, "project_id") + # The Telemetry API expects the Google Cloud project in gcp.project_id. + transform/set_project_id: + error_mode: ignore + trace_statements: + - set(resource.attributes["gcp.project_id"], resource.attributes["gcp.project.id"]) where resource.attributes["gcp.project.id"] != nil + - set(resource.attributes["gcp.project_id"], resource.attributes["cloud.account.id"]) where resource.attributes["gcp.project_id"] == nil and resource.attributes["cloud.account.id"] != nil + +exporters: + googlemanagedprometheus: + # Google Cloud's supported OTLP path for traces is the Telemetry API. + otlp: + endpoint: telemetry.googleapis.com:443 + compression: none + balancer_name: pick_first + auth: + authenticator: googleclientauth + +extensions: + # Cloud Run container dependencies require a startup probe. This endpoint is also used for the + # collector liveness probe in worker-pool.yaml. + health_check: + endpoint: 0.0.0.0:13133 + googleclientauth: + +service: + extensions: + - health_check + - googleclientauth + pipelines: + metrics/otlp: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/collision] + exporters: [googlemanagedprometheus] + traces: + receivers: [otlp] + processors: [memory_limiter, resourcedetection, transform/set_project_id, batch/traces] + exporters: [otlp] + # Feed collector self-metrics back through the metrics pipeline. + telemetry: + metrics: + readers: + - periodic: + exporter: + otlp: + protocol: grpc + endpoint: http://localhost:4317 + insecure: true +``` + + +On shutdown, stop the `WorkerFactory`, then run `telemetryPlugin.newFlushHook()` before the Cloud Run termination window ends. For a Collector configuration that exports traces to Google Cloud and metrics to Google Managed Service for Prometheus, see the [Java Cloud Run OpenTelemetry sample](https://github.com/temporalio/samples-java/pull/792). + +## Set a Worker identity {/* #worker-identity */} + +Use `WorkerIdPlugin` to identify each Worker instance as `@`. Fetch the Cloud Run metadata during process startup, then register the plugin on `WorkflowClientOptions`. Workers created from that Client inherit the identity. + + +[gcp/cloud-run/workerid/src/main/java/io/temporal/samples/gcp/cloudrun/workerid/CloudRunWorker.java](https://github.com/temporalio/samples-java/blob/main/gcp/cloud-run/workerid/src/main/java/io/temporal/samples/gcp/cloudrun/workerid/CloudRunWorker.java) +```java +GoogleCloudRunMetadata metadata = GoogleCloudRunMetadata.fetch(); + +String address = envOrDefault(ADDRESS_ENV, DEFAULT_ADDRESS); +String namespace = envOrDefault(NAMESPACE_ENV, DEFAULT_NAMESPACE); +String taskQueue = envOrDefault(TASK_QUEUE_ENV, DEFAULT_TASK_QUEUE); + +// Plaintext connection to the Temporal Service. Configure TLS or an API key here for a secured +// Service such as Temporal Cloud. +WorkflowServiceStubs service = + WorkflowServiceStubs.newServiceStubs( + WorkflowServiceStubsOptions.newBuilder().setTarget(address).build()); + +// Register WorkerIdPlugin on the client. It sets the derived worker identity +// ({instanceId}@{revision}) on the client, and workers created from the client inherit it. +// Passing the already-fetched metadata avoids a second call to the Cloud Run metadata server. +WorkflowClient client = + WorkflowClient.newInstance( + service, + WorkflowClientOptions.newBuilder() + .setNamespace(namespace) + .setPlugins(new WorkerIdPlugin(metadata)) + .build()); +``` + + +Fetching the metadata fails when the Worker runs outside Cloud Run. For a complete example, see the [Java Cloud Run Worker Id sample](https://github.com/temporalio/samples-java/pull/795). diff --git a/docs/develop/python/workers/serverless-workers/cloud-run.mdx b/docs/develop/python/workers/serverless-workers/cloud-run.mdx index 22d20370c4..f5e1a5182c 100644 --- a/docs/develop/python/workers/serverless-workers/cloud-run.mdx +++ b/docs/develop/python/workers/serverless-workers/cloud-run.mdx @@ -23,7 +23,7 @@ import { ReleaseNoteHeader } from '@site/src/components'; On a [GCP Cloud Run worker pool](https://cloud.google.com/run/docs/resource-model#worker-pools), you run a standard long-lived Temporal Worker. Register Workflows and Activities the same way you would with any other Python Worker, and Temporal Cloud scales the pool up and down as work arrives and drains. -A Cloud Run Worker needs no Cloud Run-specific package. +A Cloud Run Worker needs no Cloud Run-specific runtime or handler. The one addition to a standard Worker is [Worker Versioning](/worker-versioning), which is required for Serverless Workers. For the end-to-end deployment guide covering the Worker Pool, IAM, and compute configuration, see [Deploy a Serverless Worker on GCP Cloud Run](/production-deployment/worker-deployments/serverless-workers/cloud-run). @@ -129,7 +129,120 @@ async def my_activity(items: list[str]) -> str: For how scale-in decisions are made, see [Serverless Workers on GCP Cloud Run](/serverless-workers/cloud-run#lifecycle). -## Add observability {/* #add-observability */} +## Configure OpenTelemetry {/* #opentelemetry */} + +Run an OpenTelemetry Collector as a sidecar in the Worker Pool. The Cloud Run OpenTelemetry plugin exports metrics and traces by OTLP gRPC to the Collector at `localhost:4317`. It configures the endpoint, service name, Core SDK metrics, and tracer provider from Cloud Run defaults. + +Create the plugin and pass it to `Client.connect`. The sample reads its connection settings from the environment: + + +[gcp/cloud_run/opentelemetry/worker.py](https://github.com/temporalio/samples-python/blob/gcp-cloud-run-otel/gcp/cloud_run/opentelemetry/worker.py) +```py +# Endpoint, service name, Core metrics, and tracer provider all use the GCP +# plugin defaults. The opt-in adds named Temporal operation spans. +plugin = OpenTelemetryPlugin(add_temporal_spans=True) +client = await Client.connect( + settings.address, + namespace=settings.namespace, + api_key=settings.api_key, + # TLS for Temporal Cloud (api key present); plaintext for a dev server. + tls=bool(settings.api_key), + plugins=[plugin], +) +``` + + +Configure the Collector sidecar to receive OTLP gRPC on `localhost:4317`, export traces to Google Cloud, and export metrics to Google Managed Service for Prometheus: + + +[gcp/cloud_run/opentelemetry/collector-config.yaml](https://github.com/temporalio/samples-python/blob/main/gcp/cloud_run/opentelemetry/collector-config.yaml) +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: localhost:4317 + +processors: + batch/traces: + send_batch_max_size: 200 + send_batch_size: 200 + timeout: 5s + memory_limiter: + check_interval: 1s + limit_percentage: 65 + spike_limit_percentage: 20 + resource_detection: + detectors: [gcp] + timeout: 10s + transform/collision: + metric_statements: + - context: datapoint + statements: + - set(attributes["exported_location"], attributes["location"]) + - delete_key(attributes, "location") + - set(attributes["exported_cluster"], attributes["cluster"]) + - delete_key(attributes, "cluster") + - set(attributes["exported_namespace"], attributes["namespace"]) + - delete_key(attributes, "namespace") + - set(attributes["exported_job"], attributes["job"]) + - delete_key(attributes, "job") + - set(attributes["exported_instance"], attributes["instance"]) + - delete_key(attributes, "instance") + - set(attributes["exported_project_id"], attributes["project_id"]) + - delete_key(attributes, "project_id") + transform/set_project_id: + error_mode: ignore + trace_statements: + - set(resource.attributes["gcp.project_id"], resource.attributes["gcp.project.id"]) where resource.attributes["gcp.project.id"] != nil + - set(resource.attributes["gcp.project_id"], resource.attributes["cloud.account.id"]) where resource.attributes["gcp.project_id"] == nil and resource.attributes["cloud.account.id"] != nil + +exporters: + googlemanagedprometheus: + otlp_grpc: + endpoint: telemetry.googleapis.com:443 + compression: none + balancer_name: pick_first + auth: + authenticator: googleclientauth + +extensions: + googleclientauth: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [googleclientauth, health_check] + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, resource_detection, transform/collision] + exporters: [googlemanagedprometheus] + traces: + receivers: [otlp] + processors: + [memory_limiter, resource_detection, transform/set_project_id, batch/traces] + exporters: [otlp_grpc] +``` + + +Wait until the Collector accepts connections before starting the Worker. On shutdown, call `plugin.shutdown()` after the Worker stops so traces can flush. For a Collector configuration that exports traces to Google Cloud and metrics to Google Managed Service for Prometheus, see the [Python Cloud Run OpenTelemetry sample](https://github.com/temporalio/samples-python/pull/353). + +## Set a Worker identity {/* #worker-identity */} + +Use `WorkerIDPlugin` to identify each Worker instance as `@`. The plugin reads Cloud Run environment variables and instance metadata when the Client connects, then Workers created from that Client inherit the identity. + + +[gcp/cloud_run/worker_id/worker.py](https://github.com/temporalio/samples-python/blob/main/gcp/cloud_run/worker_id/worker.py) +```py +client = await Client.connect( + settings.address, + namespace=settings.namespace, + plugins=[WorkerIDPlugin()], + api_key=settings.api_key, + tls=settings.tls, +) +``` + -A Cloud Run Worker emits the same traces and metrics as a Worker anywhere else. -For how to configure metrics export and OpenTelemetry tracing interceptors, see [Observability - Python SDK](/develop/python/platform/observability) and the [SDK metrics reference](/references/sdk-metrics). +The plugin requires the Cloud Run metadata server, so it fails when the Worker runs outside Cloud Run. For a complete example, see the [Python Cloud Run Worker Id sample](https://github.com/temporalio/samples-python/pull/356). diff --git a/docs/production-deployment/worker-deployments/serverless-workers/cloud-run/index.mdx b/docs/production-deployment/worker-deployments/serverless-workers/cloud-run/index.mdx index 634db69e53..d8ddf7ae04 100644 --- a/docs/production-deployment/worker-deployments/serverless-workers/cloud-run/index.mdx +++ b/docs/production-deployment/worker-deployments/serverless-workers/cloud-run/index.mdx @@ -77,6 +77,7 @@ import os from temporalio.client import Client from temporalio.common import VersioningBehavior, WorkerDeploymentVersion +from temporalio.contrib.gcp.cloud_run.worker_id import WorkerIDPlugin from temporalio.worker import Worker, WorkerDeploymentConfig from my_workflows import MyWorkflow @@ -88,6 +89,7 @@ async def main() -> None: os.environ["TEMPORAL_ADDRESS"], namespace=os.environ["TEMPORAL_NAMESPACE"], api_key=os.environ.get("TEMPORAL_API_KEY"), + plugins=[WorkerIDPlugin()], tls=True, ) worker = Worker( @@ -127,8 +129,8 @@ class MyWorkflow: ... ``` -For more on the Python Worker setup, see -[Serverless Workers on GCP Cloud Run - Python SDK](/develop/python/workers/serverless-workers/cloud-run). +The example uses the Cloud Run Worker Id plugin to identify each Worker as `@`. For more on the +Python Worker setup, see [Serverless Workers on GCP Cloud Run - Python SDK](/develop/python/workers/serverless-workers/cloud-run). @@ -142,6 +144,7 @@ import ( "go.temporal.io/sdk/client" "go.temporal.io/sdk/contrib/envconfig" + "go.temporal.io/sdk/contrib/gcp/cloudrun/workerid" "go.temporal.io/sdk/worker" "go.temporal.io/sdk/workflow" @@ -149,7 +152,9 @@ import ( ) func main() { - c, err := client.Dial(envconfig.MustLoadDefaultClientOptions()) + clientOptions := envconfig.MustLoadDefaultClientOptions() + clientOptions.Plugins = append(clientOptions.Plugins, workerid.NewPlugin(workerid.PluginOptions{})) + c, err := client.Dial(clientOptions) if err != nil { log.Fatalln("Unable to create client", err) } @@ -192,8 +197,8 @@ w := worker.New(c, os.Getenv("TEMPORAL_TASK_QUEUE"), worker.Options{ If a Version is set and neither is specified, registration panics with `workflow type does not have a versioning behavior`. -For more on the Go Worker setup, see -[Serverless Workers on GCP Cloud Run - Go SDK](/develop/go/workers/serverless-workers/cloud-run). +The example uses the Cloud Run Worker Id plugin to identify each Worker as `@`. For more on the +Go Worker setup, see [Serverless Workers on GCP Cloud Run - Go SDK](/develop/go/workers/serverless-workers/cloud-run). @@ -257,6 +262,8 @@ import io.temporal.client.WorkflowClient; import io.temporal.client.WorkflowClientOptions; import io.temporal.common.VersioningBehavior; import io.temporal.common.WorkerDeploymentVersion; +import io.temporal.gcp.cloudrun.workerid.GoogleCloudRunMetadata; +import io.temporal.gcp.cloudrun.workerid.WorkerIdPlugin; import io.temporal.serviceclient.WorkflowServiceStubs; import io.temporal.serviceclient.WorkflowServiceStubsOptions; import io.temporal.worker.Worker; @@ -276,11 +283,13 @@ public class WorkerMain { .addApiKey(() -> apiKey) .build()); + GoogleCloudRunMetadata metadata = GoogleCloudRunMetadata.fetch(); WorkflowClient client = WorkflowClient.newInstance( service, WorkflowClientOptions.newBuilder() .setNamespace(System.getenv("TEMPORAL_NAMESPACE")) + .setPlugins(new WorkerIdPlugin(metadata)) .build()); WorkerFactory factory = WorkerFactory.newInstance(client); @@ -309,20 +318,22 @@ Each Workflow must have a [versioning behavior](/worker-versioning#versioning-be `AUTO_UPGRADE`. Set it per Workflow with the `@WorkflowVersioningBehavior` annotation, or set a Worker-level default with `setDefaultVersioningBehavior` as shown above. -For more on the Java Worker setup, see -[Serverless Workers on GCP Cloud Run - Java SDK](/develop/java/workers/serverless-workers/cloud-run). +The example uses the Cloud Run Worker Id plugin to identify each Worker as `@`. For more on the +Java Worker setup, see [Serverless Workers on GCP Cloud Run - Java SDK](/develop/java/workers/serverless-workers/cloud-run). ```csharp using Temporalio.Client; +using Temporalio.Extensions.Gcp.CloudRun.WorkerId; using Temporalio.Worker; var client = await TemporalClient.ConnectAsync(new(Environment.GetEnvironmentVariable("TEMPORAL_ADDRESS")!) { Namespace = Environment.GetEnvironmentVariable("TEMPORAL_NAMESPACE")!, ApiKey = Environment.GetEnvironmentVariable("TEMPORAL_API_KEY"), + Plugins = new[] { new WorkerIdPlugin() }, Tls = new(), }); @@ -344,8 +355,8 @@ Each Workflow must have a [versioning behavior](/worker-versioning#versioning-be `AutoUpgrade`. Set it per Workflow with `[Workflow(VersioningBehavior = ...)]`, or set a Worker-level default with `DefaultVersioningBehavior` as shown above. -For more on the .NET Worker setup, see -[Serverless Workers on GCP Cloud Run - .NET SDK](/develop/dotnet/workers/serverless-workers/cloud-run). +The example uses the Cloud Run Worker Id plugin to identify each Worker as `@`. For more on the +.NET Worker setup, see [Serverless Workers on GCP Cloud Run - .NET SDK](/develop/dotnet/workers/serverless-workers/cloud-run). @@ -448,16 +459,6 @@ For more on the Rust Worker setup, see -:::tip - -Workers on Cloud Run use the same code as a traditional long-lived Worker, so the SDK defaults the -[Worker Identity](/workers#worker-identity) to the process ID and hostname. On Cloud Run that resolves to -`1@localhost`, which makes a Worker harder to identify in the Temporal UI. Set your own Worker Identity, built from the -[Cloud Run environment variables](https://cloud.google.com/run/docs/container-contract#worker-pools-env-vars), to help -identify your Serverless Workers. - -::: - ## 2. Deploy to a Cloud Run Worker Pool {/* #deploy-worker-pool */} Containerize the Worker, push the image to Artifact Registry, and create the Worker Pool.