From f065585683688fd2ca975e46082286a2dfa47ac8 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Thu, 11 Jun 2026 13:51:02 +0200 Subject: [PATCH 1/5] Stamp Honeycomb SampleRate from tracestate hint and export 5xx error tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Cloudflare worker in front of Electric head-samples successful requests at 1:N (currently 1:20) and keeps all >= 500 responses at sampleRate 1, weighting its exported spans accordingly. Electric only received the sampled/not-sampled bit via traceparent, so: 1. Electric spans carried no SampleRate attribute and electric-region aggregates under-reported worker traffic ~20x; 2. ~95% of error traces had no server-side spans at all, because the worker's keep-on-error decision happens at export time, after a traceparent with sampled=0 was already propagated. The worker now sends its rate hint on every proxied request via the W3C tracestate header (member: `electric=rate:`). This commit makes Electric honor it: * TraceContextPlug parses the hint (missing/unparseable/rate < 1 hints are ignored) and stashes it in the conn together with the remote parent span context and its sampled flag. * For sampled remote parents, ServeShapePlug stamps `SampleRate` = N (status < 500) or `SampleRate` = 1 (status >= 500) on the Plug_shape_get root span once the response status is known; the shape_get.plug.stream_chunk child spans get the same attribute at creation. * For unsampled remote parents, a 5xx response now synthesizes a single root request span at response time with `SampleRate` = 1, carrying the standard root-span attributes, backdated to the request start, and parented onto the remote span context with the sampled trace flag forced on — so the parent-based sampler records it and it lands in the same trace as the worker's kept-on-error spans. Unsampled successful requests still export nothing. Direct traffic (no remote parent) is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/tracestate-sample-rate.md | 11 + .../lib/electric/plug/serve_shape_plug.ex | 64 +++- .../lib/electric/plug/trace_context_plug.ex | 90 +++++- .../lib/electric/shapes/api/response.ex | 8 +- .../lib/electric/telemetry/open_telemetry.ex | 78 +++++ .../serve_shape_plug_sample_rate_test.exs | 306 ++++++++++++++++++ .../electric/plug/trace_context_plug_test.exs | 139 ++++++++ 7 files changed, 693 insertions(+), 3 deletions(-) create mode 100644 .changeset/tracestate-sample-rate.md create mode 100644 packages/sync-service/test/electric/plug/serve_shape_plug_sample_rate_test.exs create mode 100644 packages/sync-service/test/electric/plug/trace_context_plug_test.exs diff --git a/.changeset/tracestate-sample-rate.md b/.changeset/tracestate-sample-rate.md new file mode 100644 index 0000000000..7855634666 --- /dev/null +++ b/.changeset/tracestate-sample-rate.md @@ -0,0 +1,11 @@ +--- +"@core/sync-service": patch +--- + +Honor an upstream head-sampling rate hint received via the W3C `tracestate` header (`electric=rate:`) on shape GET requests: + +- When the remote parent trace is sampled, the `Plug_shape_get` root span (and the `shape_get.plug.stream_chunk` child spans) are stamped with Honeycomb's `SampleRate` attribute — `N` for responses with status < 500 and `1` for 5xx responses — so Honeycomb weights aggregates over Electric's spans by the upstream sampling rate instead of under-reporting traffic ~N-fold. + +- When the remote parent trace is NOT sampled and the request ends in a 5xx response, a single root request span is now synthesized and exported with `SampleRate=1` in the same trace as the upstream's spans (same trace_id, parented on the remote span), so server-side error telemetry is no longer lost to upstream head-sampling. Unsampled successful requests still export nothing. + +Requests without a remote trace context, or with a missing/invalid rate hint, behave exactly as before. diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index 2f3609e862..32c51a2622 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -32,6 +32,7 @@ defmodule Electric.Plug.ServeShapePlug do use Plug.Builder + alias Electric.Plug.TraceContextPlug alias Electric.ShapeCache alias Electric.ShapeCache.ShapeStatus alias Electric.Shapes.Api @@ -469,7 +470,68 @@ defmodule Electric.Plug.ServeShapePlug do } ) - add_span_attrs_from_conn(conn) + conn + |> add_span_attrs_from_conn() + |> stamp_sample_rate() + end + + # Stamp Honeycomb's `SampleRate` attribute on the root request span and handle the + # error tail of remote-unsampled traces. + # + # The upstream proxy head-samples successful requests at 1:N (propagated to us via the + # `tracestate: electric=rate:N` hint) and keeps all >= 500 responses at rate 1. We + # mirror that here, per request: + # + # * remote parent sampled: the root span is recording — stamp `SampleRate` = N for + # status < 500 and `SampleRate` = 1 for status >= 500, so Honeycomb weights + # aggregates over Electric spans the same way it weights the worker's; + # + # * remote parent NOT sampled: the parent-based sampler dropped all spans for this + # request, which is the volume win for the (vast) majority of successful requests. + # But for status >= 500 we still want server-side telemetry: synthesize a single + # root span carrying the final request attributes with `SampleRate` = 1, parented + # onto the remote span so it lands in the same trace as the upstream's + # kept-on-error spans; + # + # * no remote parent (direct traffic): nothing to do — spans are recorded and + # exported unweighted, as before. + defp stamp_sample_rate(conn) do + case TraceContextPlug.trace_context(conn) do + %{parent_sampled?: true} -> + case TraceContextPlug.sample_rate_attrs(conn, conn.status) do + attrs when map_size(attrs) > 0 -> OpenTelemetry.add_span_attributes(attrs) + _ -> :ok + end + + %{parent_sampled?: false, parent_span_ctx: parent_span_ctx} -> + if is_integer(conn.status) and conn.status >= 500 do + export_unsampled_error_span(conn, parent_span_ctx) + end + + nil -> + :ok + end + + conn + end + + # The request ran under a remote-unsampled trace (no recording span exists) but ended + # in a 5xx — export one root span after the fact so the error is visible server-side. + # `SampleRate` is hardcoded to 1: error responses ignore the rate hint, mirroring the + # upstream's keep-all-errors-at-rate-1 semantics. + defp export_unsampled_error_span(conn, parent_span_ctx) do + attributes = + conn + |> open_telemetry_attrs() + |> Map.put("SampleRate", 1) + + OpenTelemetry.export_unsampled_remote_span( + "Plug_shape_get", + attributes, + parent_span_ctx, + start_time: get_in(conn.private, [:electric_telemetry_span, :start_time]), + error: conn.assigns[:error_str] || "HTTP #{conn.status}" + ) end defp get_handle(%{response: %{shape_handle: shape_handle}}), do: shape_handle diff --git a/packages/sync-service/lib/electric/plug/trace_context_plug.ex b/packages/sync-service/lib/electric/plug/trace_context_plug.ex index f0b0a38277..6f28b4ccc3 100644 --- a/packages/sync-service/lib/electric/plug/trace_context_plug.ex +++ b/packages/sync-service/lib/electric/plug/trace_context_plug.ex @@ -1,10 +1,48 @@ defmodule Electric.Plug.TraceContextPlug do @moduledoc """ A plug that extracts trace context from incoming HTTP headers and sets it as the parent span. + + In addition to the standard W3C `traceparent` extraction, this plug parses Electric's + sample-rate hint from the `tracestate` header. An upstream proxy (e.g. the Cloudflare + worker in front of Electric Cloud) that head-samples requests at a rate of 1:N tells us + about that rate via a tracestate member of the form: + + tracestate: electric=rate: + + The hint, together with the remote parent span context and its sampled flag, is stored + in the conn so that downstream plugs can: + + * stamp Honeycomb's `SampleRate` attribute on exported spans, making Honeycomb weight + aggregates by the upstream sampling rate (see `sample_rate_attrs/2`); + + * export an error-tail span for 5xx responses even when the remote parent was not + sampled (see `Electric.Telemetry.OpenTelemetry.export_unsampled_remote_span/4`). + + Hints that are missing, unparseable or have a rate below 1 are ignored. """ @behaviour Plug + alias Electric.Telemetry.OpenTelemetry + + @private_key :electric_trace_context + @tracestate_key "electric" + @sample_rate_attr "SampleRate" + + @typedoc """ + Remote trace context extracted from the request headers. + + * `:parent_span_ctx` - the span context extracted from `traceparent` + * `:parent_sampled?` - whether the remote parent has the W3C `sampled` flag set + * `:sample_rate_hint` - the upstream 1:N sampling rate parsed from `tracestate`, + or `nil` when absent/invalid + """ + @type trace_context :: %{ + parent_span_ctx: OpenTelemetry.span_ctx(), + parent_sampled?: boolean(), + sample_rate_hint: pos_integer() | nil + } + def init(opts), do: opts def call(%Plug.Conn{req_headers: headers} = conn, _opts) do @@ -21,7 +59,57 @@ defmodule Electric.Plug.TraceContextPlug do span_ctx -> # Parent found, set as current span :otel_tracer.set_current_span(span_ctx) - conn + + Plug.Conn.put_private(conn, @private_key, %{ + parent_span_ctx: span_ctx, + parent_sampled?: OpenTelemetry.span_ctx_sampled?(span_ctx), + sample_rate_hint: sample_rate_hint(span_ctx) + }) + end + end + + @doc """ + The remote trace context extracted from the request headers by this plug, or `nil` + when the request did not carry a (valid) `traceparent` header. + """ + @spec trace_context(Plug.Conn.t()) :: trace_context() | nil + def trace_context(%Plug.Conn{private: private}), do: private[@private_key] + + @doc """ + Span attributes carrying Honeycomb's `SampleRate` for a response with the given status. + + Honeycomb's OTLP ingest natively reads an integer span attribute named `SampleRate` + and weights aggregates by it. Successful responses inherit the upstream sampling rate + from the tracestate hint, while error (>= 500) responses are stamped with a rate of 1: + they mirror the upstream's keep-all-errors-at-rate-1 semantics. + + Returns an empty map when the request carried no usable rate hint, leaving the spans + unweighted as before. + """ + @spec sample_rate_attrs(Plug.Conn.t(), integer() | nil) :: %{ + optional(String.t()) => pos_integer() + } + def sample_rate_attrs(conn, status) do + case trace_context(conn) do + %{sample_rate_hint: rate} when is_integer(rate) -> + %{@sample_rate_attr => effective_sample_rate(rate, status)} + + _ -> + %{} + end + end + + defp effective_sample_rate(_rate, status) when is_integer(status) and status >= 500, do: 1 + defp effective_sample_rate(rate, _status), do: rate + + defp sample_rate_hint(span_ctx) do + with value when is_binary(value) <- + OpenTelemetry.tracestate_value(span_ctx, @tracestate_key), + "rate:" <> rate_str <- value, + {rate, ""} when rate >= 1 <- Integer.parse(rate_str) do + rate + else + _ -> nil end end end diff --git a/packages/sync-service/lib/electric/shapes/api/response.ex b/packages/sync-service/lib/electric/shapes/api/response.ex index f7276e6cd7..aa1a3131ea 100644 --- a/packages/sync-service/lib/electric/shapes/api/response.ex +++ b/packages/sync-service/lib/electric/shapes/api/response.ex @@ -427,6 +427,12 @@ defmodule Electric.Shapes.Api.Response do validate_response_finalized!(response) stack_id = Api.stack_id(response) + + # Per-chunk spans inherit Honeycomb's SampleRate from the upstream tracestate hint + # (when present), same as the root request span — see + # Electric.Plug.TraceContextPlug.sample_rate_attrs/2. + sample_rate_attrs = Electric.Plug.TraceContextPlug.sample_rate_attrs(conn, status) + conn = Plug.Conn.send_chunked(conn, status) {conn, bytes_sent} = @@ -436,7 +442,7 @@ defmodule Electric.Shapes.Api.Response do OpenTelemetry.with_span( "shape_get.plug.stream_chunk", - [chunk_size: chunk_size], + Map.put(sample_rate_attrs, :chunk_size, chunk_size), stack_id, fn -> case Plug.Conn.chunk(conn, chunk) do diff --git a/packages/sync-service/lib/electric/telemetry/open_telemetry.ex b/packages/sync-service/lib/electric/telemetry/open_telemetry.ex index 1821b7b0d9..6240fbacbc 100644 --- a/packages/sync-service/lib/electric/telemetry/open_telemetry.ex +++ b/packages/sync-service/lib/electric/telemetry/open_telemetry.ex @@ -32,6 +32,14 @@ defmodule Electric.Telemetry.OpenTelemetry do alias Electric.Telemetry.Sampler alias Electric.Telemetry.IntervalTimer + require Record + + Record.defrecordp( + :span_ctx_record, + :span_ctx, + Record.extract(:span_ctx, from_lib: "opentelemetry_api/include/opentelemetry.hrl") + ) + @typep span_name :: String.t() @typep attr_name :: String.t() @typep span_attrs :: :opentelemetry.attributes_map() @@ -376,6 +384,76 @@ defmodule Electric.Telemetry.OpenTelemetry do Process.get(@interval_timer_key, []) end + @doc """ + Whether the W3C `sampled` flag is set in the trace flags of the given span context. + """ + @spec span_ctx_sampled?(span_ctx()) :: boolean() + def span_ctx_sampled?(span_ctx) when Record.is_record(span_ctx, :span_ctx) do + Bitwise.band(span_ctx_record(span_ctx, :trace_flags), 1) == 1 + end + + @doc """ + Look up the value of the member with the given key in the W3C tracestate carried + by the given span context. + + Returns the member's value as a string, or `nil` when the tracestate has no such + member. + """ + @spec tracestate_value(span_ctx(), String.t()) :: String.t() | nil + def tracestate_value(span_ctx, key) when Record.is_record(span_ctx, :span_ctx) do + case :otel_tracestate.get(key, span_ctx_record(span_ctx, :tracestate)) do + value when is_binary(value) and value != "" -> value + [_ | _] = value -> List.to_string(value) + _ -> nil + end + end + + @doc """ + Create and immediately export a finished span as a child of a remote *unsampled* + parent span context. + + The parent-based OTel sampler drops every span of a trace whose remote parent has + the `sampled` flag unset, so by the time we decide that a request is worth + exporting after all (e.g. it resulted in a 5xx response), no recording span exists. + This function retroactively synthesizes one: it copies the remote parent span + context with the `sampled` trace flag forced on and starts a recording span under + that copy, so the parent-based sampler takes its `remote_parent_sampled` branch + (`always_on` by default). The synthesized span keeps the remote trace_id and uses + the remote span id as its parent, so it shows up in the same trace as the spans + recorded by the upstream service. + + Options: + + * `:start_time` - native monotonic timestamp for the span's start (defaults to + the current time, producing a zero-duration span) + * `:error` - a message to record as the span's error status + """ + @spec export_unsampled_remote_span(span_name(), span_attrs(), span_ctx(), keyword()) :: :ok + def export_unsampled_remote_span(name, attributes, parent_span_ctx, opts \\ []) + when Record.is_record(parent_span_ctx, :span_ctx) do + trace_flags = span_ctx_record(parent_span_ctx, :trace_flags) + sampled_parent = span_ctx_record(parent_span_ctx, trace_flags: Bitwise.bor(trace_flags, 1)) + + parent_ctx = :otel_tracer.set_current_span(:otel_ctx.new(), sampled_parent) + + span_opts = %{ + attributes: Map.new(attributes), + links: [], + is_recording: true, + start_time: opts[:start_time] || :opentelemetry.timestamp(), + kind: :internal + } + + span_ctx = :otel_tracer.start_span(parent_ctx, tracer(), name, span_opts) + + if message = opts[:error] do + :otel_span.set_status(span_ctx, :error, message) + end + + :otel_span.end_span(span_ctx) + :ok + end + @doc """ Add an error event to the current span. """ diff --git a/packages/sync-service/test/electric/plug/serve_shape_plug_sample_rate_test.exs b/packages/sync-service/test/electric/plug/serve_shape_plug_sample_rate_test.exs new file mode 100644 index 0000000000..67757590b4 --- /dev/null +++ b/packages/sync-service/test/electric/plug/serve_shape_plug_sample_rate_test.exs @@ -0,0 +1,306 @@ +# These tests exercise real span creation/export through the :opentelemetry SDK, which is +# only a dependency when building for the telemetry target (MIX_TARGET=application, the +# configuration CI runs the test suite with). +if Electric.telemetry_enabled?() do + defmodule Electric.Plug.ServeShapePlugSampleRateTest do + # async: false — reconfigures the global :opentelemetry application to export + # finished spans to the test process. + use ExUnit.Case, async: false + use Repatch.ExUnit + + alias Electric.Plug.ServeShapePlug + alias Electric.Plug.TraceContextPlug + alias Electric.Postgres.Lsn + alias Electric.Replication.LogOffset + alias Electric.Shapes.Api + alias Electric.Shapes.Shape + + import Support.ComponentSetup + + import Support.TestUtils, + only: [ + set_status_to_active: 1, + patch_shape_cache: 1, + expect_shape_cache: 1, + patch_storage: 1, + expect_storage: 1 + ] + + require Record + + Record.defrecordp( + :span_record, + :span, + Record.extract(:span, from_lib: "opentelemetry/include/otel_span.hrl") + ) + + @inspector Support.StubInspector.new( + tables: ["users"], + columns: [ + %{name: "id", type: "int8", pk_position: 0, type_id: {20, 1}}, + %{name: "value", type: "text", pk_position: nil, type_id: {28, 1}} + ] + ) + + @test_shape %Shape{ + root_table: {"public", "users"}, + root_table_id: :erlang.phash2({"public", "users"}), + root_column_count: 2, + root_pk: ["id"], + selected_columns: ["id", "value"], + explicitly_selected_columns: ["id", "value"], + flags: %{selects_all_columns: true} + } + @test_shape_handle "test-shape-handle" + @test_opts %{foo: "bar"} + @before_all_offset LogOffset.before_all() + @first_offset LogOffset.first() + @test_offset LogOffset.new(Lsn.from_integer(100), 0) + + @receive_timeout 2000 + + @trace_id_hex "0af7651916cd43dd8448eb211c80319c" + @parent_span_id_hex "b7ad6b7169203331" + @trace_id String.to_integer(@trace_id_hex, 16) + @parent_span_id String.to_integer(@parent_span_id_hex, 16) + + @sampled_headers [ + {"traceparent", "00-#{@trace_id_hex}-#{@parent_span_id_hex}-01"}, + {"tracestate", "electric=rate:20"} + ] + @unsampled_headers [ + {"traceparent", "00-#{@trace_id_hex}-#{@parent_span_id_hex}-00"}, + {"tracestate", "electric=rate:20"} + ] + + @moduletag :tmp_dir + + setup [ + :with_stack_id_from_test, + :with_registry, + :with_persistent_kv, + :with_pure_file_storage, + :with_status_monitor, + :with_shape_cleaner, + :with_lsn_tracker + ] + + setup ctx do + # Restart the :opentelemetry application with a simple processor that exports + # every finished span as a `{:span, span}` message to the test process. + restart_opentelemetry([ + {:otel_simple_processor, %{exporter: {:otel_exporter_pid, self()}}} + ]) + + on_exit(fn -> restart_opentelemetry([]) end) + + {:via, _, {registry_name, registry_key}} = + Electric.Shapes.Supervisor.name(ctx.stack_id) + + {:ok, _} = Registry.register(registry_name, registry_key, nil) + set_status_to_active(ctx) + + %{plug_opts: build_plug_opts(ctx)} + end + + defp restart_opentelemetry(processors) do + Application.stop(:opentelemetry) + + # The OTel API caches per-application tracers in persistent terms. A cached + # tracer embeds the span-processor pipeline of the tracer provider it was + # created against, and opentelemetry's create_application_tracers keeps stale + # cache entries across an application restart — so without this, spans started + # via OpentelemetryTelemetry (which uses the per-application tracer) would + # still go to the previous provider's processors. Erase the cache so tracers + # are re-created against the new provider on app start. + for {key, _} <- :persistent_term.get(), + match?({:opentelemetry, _, :tracer, _}, key) do + :persistent_term.erase(key) + end + + Application.put_env(:opentelemetry, :processors, processors) + {:ok, _} = Application.ensure_all_started(:opentelemetry) + end + + defp build_plug_opts(ctx) do + Api.plug_opts( + stack_id: ctx.stack_id, + inspector: @inspector, + feature_flags: [], + stack_ready_timeout: 100, + long_poll_timeout: 20_000, + sse_timeout: 20_000, + max_age: 60, + stale_age: 300, + max_concurrent_requests: %{initial: 300, existing: 10_000} + ) + end + + defp mock_successful_snapshot_response do + patch_storage(for_shape: fn @test_shape_handle, _opts -> @test_opts end) + + expect_shape_cache( + get_or_create_shape_handle: fn @test_shape, _stack_id, _opts -> + {@test_shape_handle, @test_offset} + end, + await_snapshot_start: fn @test_shape_handle, _ -> :started end + ) + + patch_shape_cache(has_shape?: fn @test_shape_handle, _opts -> true end) + + expect_storage( + get_chunk_end_log_offset: fn @before_all_offset, _ -> @first_offset end, + get_log_stream: fn @before_all_offset, @first_offset, @test_opts -> + [Jason.encode!(%{key: "log", value: "foo", headers: %{}, offset: @first_offset})] + end + ) + end + + # Mirrors the production plug order: TraceContextPlug runs in the router pipeline + # before ServeShapePlug, in the same process. + defp request(ctx, headers) do + headers + |> Enum.reduce( + Plug.Test.conn(:get, "/?offset=-1", %{"table" => "public.users"}), + fn {key, value}, conn -> Plug.Conn.put_req_header(conn, key, value) end + ) + |> TraceContextPlug.call([]) + |> ServeShapePlug.call(ctx.plug_opts) + end + + # 5xx requests go through ServeShapePlug's error handling, which re-raises after + # sending the 500 response (so outer layers see the error too). + defp request_expecting_crash(ctx, headers) do + try do + request(ctx, headers) + catch + _kind, _reason -> :ok + end + end + + defp crash_load_shape_info do + Repatch.patch(Api, :load_shape_info, fn _request -> + raise RuntimeError, "simulated crash" + end) + end + + defp attrs_map(attributes), do: :otel_attributes.map(attributes) + + test "sampled remote parent + success: root span stamped with SampleRate=N", ctx do + mock_successful_snapshot_response() + + conn = request(ctx, @sampled_headers) + assert conn.status == 200 + + assert_receive {:span, + span_record( + name: "Plug_shape_get", + trace_id: @trace_id, + parent_span_id: @parent_span_id, + attributes: attributes + )}, + @receive_timeout + + assert attrs_map(attributes)["SampleRate"] == 20 + end + + test "sampled remote parent + success: stream_chunk child spans stamped too", ctx do + mock_successful_snapshot_response() + + conn = request(ctx, @sampled_headers) + assert conn.status == 200 + + assert_receive {:span, + span_record( + name: "shape_get.plug.stream_chunk", + trace_id: @trace_id, + attributes: attributes + )}, + @receive_timeout + + assert attrs_map(attributes)["SampleRate"] == 20 + end + + test "sampled remote parent + 5xx: root span stamped with SampleRate=1", ctx do + crash_load_shape_info() + + request_expecting_crash(ctx, @sampled_headers) + + assert_receive {:span, + span_record( + name: "Plug_shape_get", + trace_id: @trace_id, + parent_span_id: @parent_span_id, + attributes: attributes + )}, + @receive_timeout + + assert attrs_map(attributes)["SampleRate"] == 1 + end + + test "unsampled remote parent + success: no spans exported", ctx do + mock_successful_snapshot_response() + + conn = request(ctx, @unsampled_headers) + assert conn.status == 200 + + refute_receive {:span, _}, 100 + end + + test "unsampled remote parent + 5xx: exactly one root span exported in the same trace", + ctx do + crash_load_shape_info() + + request_expecting_crash(ctx, @unsampled_headers) + + assert_receive {:span, + span_record( + name: "Plug_shape_get", + trace_id: @trace_id, + parent_span_id: @parent_span_id, + start_time: start_time, + end_time: end_time, + status: status, + attributes: attributes + )}, + @receive_timeout + + attrs = attrs_map(attributes) + assert attrs["SampleRate"] == 1 + # Synthesized at response time but backdated to the request start. + assert start_time <= end_time + # Standard root-span attributes are carried over. + assert attrs["shape_req.is_error"] == true + assert Map.has_key?(attrs, "shape.root_table") + # The span records the error. + assert {:status, :error, _message} = status + + # The error-tail span is the only thing exported for this trace. + refute_receive {:span, _}, 100 + end + + test "no remote parent: spans exported without a SampleRate attribute", ctx do + mock_successful_snapshot_response() + + conn = request(ctx, []) + assert conn.status == 200 + + assert_receive {:span, span_record(name: "Plug_shape_get", attributes: attributes)}, + @receive_timeout + + refute Map.has_key?(attrs_map(attributes), "SampleRate") + end + + test "no remote parent + 5xx: no SampleRate and no synthesized duplicates", ctx do + crash_load_shape_info() + + request_expecting_crash(ctx, []) + + assert_receive {:span, span_record(name: "Plug_shape_get", attributes: attributes)}, + @receive_timeout + + refute Map.has_key?(attrs_map(attributes), "SampleRate") + refute_receive {:span, span_record(name: "Plug_shape_get")}, 100 + end + end +end diff --git a/packages/sync-service/test/electric/plug/trace_context_plug_test.exs b/packages/sync-service/test/electric/plug/trace_context_plug_test.exs new file mode 100644 index 0000000000..9e79999eac --- /dev/null +++ b/packages/sync-service/test/electric/plug/trace_context_plug_test.exs @@ -0,0 +1,139 @@ +# The W3C trace context propagator is configured by the :opentelemetry SDK application, +# which is only a dependency when building for the telemetry target (MIX_TARGET=application, +# the configuration CI runs the test suite with). +if Electric.telemetry_enabled?() do + defmodule Electric.Plug.TraceContextPlugTest do + use ExUnit.Case, async: true + + alias Electric.Plug.TraceContextPlug + + @trace_id_hex "0af7651916cd43dd8448eb211c80319c" + @parent_span_id_hex "b7ad6b7169203331" + + defp call(headers) do + headers + |> Enum.reduce(Plug.Test.conn(:get, "/v1/shape"), fn {key, value}, conn -> + Plug.Conn.put_req_header(conn, key, value) + end) + |> TraceContextPlug.call([]) + end + + defp traceparent(flags), do: "00-#{@trace_id_hex}-#{@parent_span_id_hex}-#{flags}" + + test "no traceparent: conn passes through with no trace context" do + conn = call([]) + assert TraceContextPlug.trace_context(conn) == nil + end + + test "tracestate without traceparent is ignored" do + conn = call([{"tracestate", "electric=rate:20"}]) + assert TraceContextPlug.trace_context(conn) == nil + end + + test "sampled traceparent with rate hint" do + conn = call([{"traceparent", traceparent("01")}, {"tracestate", "electric=rate:20"}]) + + assert %{parent_sampled?: true, sample_rate_hint: 20, parent_span_ctx: span_ctx} = + TraceContextPlug.trace_context(conn) + + assert span_ctx != :undefined + # The extracted remote parent is installed as the current span for this process. + assert :otel_tracer.current_span_ctx() == span_ctx + end + + test "unsampled traceparent still parses the rate hint" do + conn = call([{"traceparent", traceparent("00")}, {"tracestate", "electric=rate:20"}]) + + assert %{parent_sampled?: false, sample_rate_hint: 20} = + TraceContextPlug.trace_context(conn) + end + + test "traceparent without tracestate: no rate hint" do + conn = call([{"traceparent", traceparent("01")}]) + + assert %{parent_sampled?: true, sample_rate_hint: nil} = + TraceContextPlug.trace_context(conn) + end + + test "electric member is found among other tracestate members" do + conn = + call([ + {"traceparent", traceparent("01")}, + {"tracestate", "congo=t61rcWkgMzE,electric=rate:42,rojo=00f067aa0ba902b7"} + ]) + + assert %{sample_rate_hint: 42} = TraceContextPlug.trace_context(conn) + end + + test "whitespace around tracestate members is tolerated" do + conn = + call([ + {"traceparent", traceparent("01")}, + {"tracestate", "congo=t61rcWkgMzE , electric=rate:7"} + ]) + + assert %{sample_rate_hint: 7} = TraceContextPlug.trace_context(conn) + end + + test "rate of exactly 1 is accepted" do + conn = call([{"traceparent", traceparent("01")}, {"tracestate", "electric=rate:1"}]) + assert %{sample_rate_hint: 1} = TraceContextPlug.trace_context(conn) + end + + for tracestate <- [ + # rate < 1 + "electric=rate:0", + "electric=rate:-5", + # non-integer / trailing garbage + "electric=rate:1.5", + "electric=rate:20x", + "electric=rate:abc", + "electric=rate:", + # wrong member format + "electric=ratio:20", + "electric=20", + # no electric member at all + "congo=t61rcWkgMzE", + "notelectric=rate:20" + ] do + test "invalid or missing hint in #{inspect(tracestate)} is ignored" do + conn = + call([{"traceparent", traceparent("01")}, {"tracestate", unquote(tracestate)}]) + + assert %{parent_sampled?: true, sample_rate_hint: nil} = + TraceContextPlug.trace_context(conn) + end + end + + describe "sample_rate_attrs/2" do + test "uses the hinted rate for non-5xx statuses" do + conn = call([{"traceparent", traceparent("01")}, {"tracestate", "electric=rate:20"}]) + + assert TraceContextPlug.sample_rate_attrs(conn, 200) == %{"SampleRate" => 20} + assert TraceContextPlug.sample_rate_attrs(conn, 304) == %{"SampleRate" => 20} + assert TraceContextPlug.sample_rate_attrs(conn, 404) == %{"SampleRate" => 20} + assert TraceContextPlug.sample_rate_attrs(conn, 499) == %{"SampleRate" => 20} + # Status not yet known: default to the success-path rate. + assert TraceContextPlug.sample_rate_attrs(conn, nil) == %{"SampleRate" => 20} + end + + test "5xx statuses override the hint with rate 1" do + conn = call([{"traceparent", traceparent("01")}, {"tracestate", "electric=rate:20"}]) + + assert TraceContextPlug.sample_rate_attrs(conn, 500) == %{"SampleRate" => 1} + assert TraceContextPlug.sample_rate_attrs(conn, 503) == %{"SampleRate" => 1} + end + + test "returns no attributes without a rate hint" do + conn = call([{"traceparent", traceparent("01")}]) + assert TraceContextPlug.sample_rate_attrs(conn, 200) == %{} + assert TraceContextPlug.sample_rate_attrs(conn, 500) == %{} + end + + test "returns no attributes without a remote parent" do + conn = call([]) + assert TraceContextPlug.sample_rate_attrs(conn, 500) == %{} + end + end + end +end From 11082d7038517bec48531f70c4cd5bb67081d0f7 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Thu, 11 Jun 2026 14:01:04 +0200 Subject: [PATCH 2/5] Share the SampleRate attribute name via TraceContextPlug.sample_rate_attr/0 --- packages/sync-service/lib/electric/plug/serve_shape_plug.ex | 2 +- .../sync-service/lib/electric/plug/trace_context_plug.ex | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index 32c51a2622..f6e42424e3 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -523,7 +523,7 @@ defmodule Electric.Plug.ServeShapePlug do attributes = conn |> open_telemetry_attrs() - |> Map.put("SampleRate", 1) + |> Map.put(TraceContextPlug.sample_rate_attr(), 1) OpenTelemetry.export_unsampled_remote_span( "Plug_shape_get", diff --git a/packages/sync-service/lib/electric/plug/trace_context_plug.ex b/packages/sync-service/lib/electric/plug/trace_context_plug.ex index 6f28b4ccc3..b1cd9bad9a 100644 --- a/packages/sync-service/lib/electric/plug/trace_context_plug.ex +++ b/packages/sync-service/lib/electric/plug/trace_context_plug.ex @@ -102,6 +102,12 @@ defmodule Electric.Plug.TraceContextPlug do defp effective_sample_rate(_rate, status) when is_integer(status) and status >= 500, do: 1 defp effective_sample_rate(rate, _status), do: rate + @doc """ + The name of the span attribute Honeycomb reads as the sampling weight. + """ + @spec sample_rate_attr() :: String.t() + def sample_rate_attr, do: @sample_rate_attr + defp sample_rate_hint(span_ctx) do with value when is_binary(value) <- OpenTelemetry.tracestate_value(span_ctx, @tracestate_key), From 7fe29396c67efb7d190daa2dd1b9def0cb10128c Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Thu, 11 Jun 2026 15:15:45 +0200 Subject: [PATCH 3/5] Address review nits: uniform string attr keys, note on test global state --- packages/sync-service/lib/electric/shapes/api/response.ex | 2 +- .../test/electric/plug/serve_shape_plug_sample_rate_test.exs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/sync-service/lib/electric/shapes/api/response.ex b/packages/sync-service/lib/electric/shapes/api/response.ex index aa1a3131ea..b2c12032f8 100644 --- a/packages/sync-service/lib/electric/shapes/api/response.ex +++ b/packages/sync-service/lib/electric/shapes/api/response.ex @@ -442,7 +442,7 @@ defmodule Electric.Shapes.Api.Response do OpenTelemetry.with_span( "shape_get.plug.stream_chunk", - Map.put(sample_rate_attrs, :chunk_size, chunk_size), + Map.put(sample_rate_attrs, "chunk_size", chunk_size), stack_id, fn -> case Plug.Conn.chunk(conn, chunk) do diff --git a/packages/sync-service/test/electric/plug/serve_shape_plug_sample_rate_test.exs b/packages/sync-service/test/electric/plug/serve_shape_plug_sample_rate_test.exs index 67757590b4..2f480f3945 100644 --- a/packages/sync-service/test/electric/plug/serve_shape_plug_sample_rate_test.exs +++ b/packages/sync-service/test/electric/plug/serve_shape_plug_sample_rate_test.exs @@ -103,6 +103,11 @@ if Electric.telemetry_enabled?() do %{plug_opts: build_plug_opts(ctx)} end + # NOTE: this mutates VM-global telemetry state (stops/starts the :opentelemetry + # app and erases its persistent-term tracer cache). The module is `async: false` + # and `on_exit` restores the default (empty) processor pipeline, so other test + # modules are not affected — but tests in this module must not assume any OTel + # state set up outside of it. defp restart_opentelemetry(processors) do Application.stop(:opentelemetry) From b6d48d76712f57b31c59f1680e72d8d1a066aeed Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Sat, 13 Jun 2026 01:21:34 +0200 Subject: [PATCH 4/5] Use vendor-neutral language in telemetry docs and comments --- .changeset/tracestate-sample-rate.md | 2 +- .../lib/electric/plug/serve_shape_plug.ex | 7 +++--- .../lib/electric/plug/trace_context_plug.ex | 23 ++++++++++--------- .../lib/electric/shapes/api/response.ex | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.changeset/tracestate-sample-rate.md b/.changeset/tracestate-sample-rate.md index 7855634666..72dd842b9d 100644 --- a/.changeset/tracestate-sample-rate.md +++ b/.changeset/tracestate-sample-rate.md @@ -4,7 +4,7 @@ Honor an upstream head-sampling rate hint received via the W3C `tracestate` header (`electric=rate:`) on shape GET requests: -- When the remote parent trace is sampled, the `Plug_shape_get` root span (and the `shape_get.plug.stream_chunk` child spans) are stamped with Honeycomb's `SampleRate` attribute — `N` for responses with status < 500 and `1` for 5xx responses — so Honeycomb weights aggregates over Electric's spans by the upstream sampling rate instead of under-reporting traffic ~N-fold. +- When the remote parent trace is sampled, the `Plug_shape_get` root span (and the `shape_get.plug.stream_chunk` child spans) are stamped with the `SampleRate` attribute — `N` for responses with status < 500 and `1` for 5xx responses — so tracing backends that understand sampling weights scale aggregates over Electric's spans by the upstream sampling rate instead of under-reporting traffic ~N-fold. - When the remote parent trace is NOT sampled and the request ends in a 5xx response, a single root request span is now synthesized and exported with `SampleRate=1` in the same trace as the upstream's spans (same trace_id, parented on the remote span), so server-side error telemetry is no longer lost to upstream head-sampling. Unsampled successful requests still export nothing. diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index f6e42424e3..af84e90304 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -475,7 +475,7 @@ defmodule Electric.Plug.ServeShapePlug do |> stamp_sample_rate() end - # Stamp Honeycomb's `SampleRate` attribute on the root request span and handle the + # Stamp the `SampleRate` attribute on the root request span and handle the # error tail of remote-unsampled traces. # # The upstream proxy head-samples successful requests at 1:N (propagated to us via the @@ -483,8 +483,9 @@ defmodule Electric.Plug.ServeShapePlug do # mirror that here, per request: # # * remote parent sampled: the root span is recording — stamp `SampleRate` = N for - # status < 500 and `SampleRate` = 1 for status >= 500, so Honeycomb weights - # aggregates over Electric spans the same way it weights the worker's; + # status < 500 and `SampleRate` = 1 for status >= 500, so sampling-aware tracing + # backends weight aggregates over Electric spans consistently with the upstream + # proxy's own spans; # # * remote parent NOT sampled: the parent-based sampler dropped all spans for this # request, which is the volume win for the (vast) majority of successful requests. diff --git a/packages/sync-service/lib/electric/plug/trace_context_plug.ex b/packages/sync-service/lib/electric/plug/trace_context_plug.ex index b1cd9bad9a..520201eaea 100644 --- a/packages/sync-service/lib/electric/plug/trace_context_plug.ex +++ b/packages/sync-service/lib/electric/plug/trace_context_plug.ex @@ -3,17 +3,18 @@ defmodule Electric.Plug.TraceContextPlug do A plug that extracts trace context from incoming HTTP headers and sets it as the parent span. In addition to the standard W3C `traceparent` extraction, this plug parses Electric's - sample-rate hint from the `tracestate` header. An upstream proxy (e.g. the Cloudflare - worker in front of Electric Cloud) that head-samples requests at a rate of 1:N tells us - about that rate via a tracestate member of the form: + sample-rate hint from the `tracestate` header. An upstream proxy or gateway that + head-samples requests at a rate of 1:N tells us about that rate via a tracestate + member of the form: tracestate: electric=rate: The hint, together with the remote parent span context and its sampled flag, is stored in the conn so that downstream plugs can: - * stamp Honeycomb's `SampleRate` attribute on exported spans, making Honeycomb weight - aggregates by the upstream sampling rate (see `sample_rate_attrs/2`); + * stamp the `SampleRate` attribute on exported spans, letting tracing backends that + understand sampling weights scale aggregates by the upstream sampling rate + (see `sample_rate_attrs/2`); * export an error-tail span for 5xx responses even when the remote parent was not sampled (see `Electric.Telemetry.OpenTelemetry.export_unsampled_remote_span/4`). @@ -76,12 +77,12 @@ defmodule Electric.Plug.TraceContextPlug do def trace_context(%Plug.Conn{private: private}), do: private[@private_key] @doc """ - Span attributes carrying Honeycomb's `SampleRate` for a response with the given status. + Span attributes carrying the sampling weight for a response with the given status. - Honeycomb's OTLP ingest natively reads an integer span attribute named `SampleRate` - and weights aggregates by it. Successful responses inherit the upstream sampling rate - from the tracestate hint, while error (>= 500) responses are stamped with a rate of 1: - they mirror the upstream's keep-all-errors-at-rate-1 semantics. + Tracing backends that support weighted sampling read an integer span attribute named + `SampleRate` and scale aggregates by it. Successful responses inherit the upstream + sampling rate from the tracestate hint, while error (>= 500) responses are stamped + with a rate of 1: they mirror the upstream's keep-all-errors-at-rate-1 semantics. Returns an empty map when the request carried no usable rate hint, leaving the spans unweighted as before. @@ -103,7 +104,7 @@ defmodule Electric.Plug.TraceContextPlug do defp effective_sample_rate(rate, _status), do: rate @doc """ - The name of the span attribute Honeycomb reads as the sampling weight. + The name of the span attribute that carries the sampling weight. """ @spec sample_rate_attr() :: String.t() def sample_rate_attr, do: @sample_rate_attr diff --git a/packages/sync-service/lib/electric/shapes/api/response.ex b/packages/sync-service/lib/electric/shapes/api/response.ex index b2c12032f8..6a99d06335 100644 --- a/packages/sync-service/lib/electric/shapes/api/response.ex +++ b/packages/sync-service/lib/electric/shapes/api/response.ex @@ -428,7 +428,7 @@ defmodule Electric.Shapes.Api.Response do stack_id = Api.stack_id(response) - # Per-chunk spans inherit Honeycomb's SampleRate from the upstream tracestate hint + # Per-chunk spans inherit the `SampleRate` attribute from the upstream tracestate hint # (when present), same as the root request span — see # Electric.Plug.TraceContextPlug.sample_rate_attrs/2. sample_rate_attrs = Electric.Plug.TraceContextPlug.sample_rate_attrs(conn, status) From 25b7688dddf61a889e4d499c8f03b40df92e8486 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Sat, 13 Jun 2026 01:31:51 +0200 Subject: [PATCH 5/5] Fold sample-rate stamping into add_span_attrs_from_conn --- .../lib/electric/plug/serve_shape_plug.ex | 111 ++++++++---------- 1 file changed, 49 insertions(+), 62 deletions(-) diff --git a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex index af84e90304..82c0337fde 100644 --- a/packages/sync-service/lib/electric/plug/serve_shape_plug.ex +++ b/packages/sync-service/lib/electric/plug/serve_shape_plug.ex @@ -472,45 +472,62 @@ defmodule Electric.Plug.ServeShapePlug do conn |> add_span_attrs_from_conn() - |> stamp_sample_rate() end - # Stamp the `SampleRate` attribute on the root request span and handle the - # error tail of remote-unsampled traces. + defp get_handle(%{response: %{shape_handle: shape_handle}}), do: shape_handle + defp get_handle(%{request: %{shape_handle: shape_handle}}), do: shape_handle + defp get_handle(_), do: nil + + defp get_live_mode(%{response: %{params: %{live: live}}}), do: live + defp get_live_mode(%{request: %{params: %{live: live}}}), do: live + defp get_live_mode(_), do: false + + # Used as a metric label for the response-size histogram. The goal is low + # cardinality (one value per configured shape root table), so we only emit + # the validated `table` from request params. If validation never ran (e.g. + # the request was rejected before parse), fall back to nil rather than + # echoing back attacker-controlled query string values. + defp get_root_table(%{request: %{params: %{table: table}}}, _conn) when is_binary(table), + do: table + + defp get_root_table(_assigns, _conn), do: nil + + # Put the request's span attributes — the usual conn-derived attributes plus the + # `SampleRate` sampling weight when an upstream rate hint applies — where they belong. # - # The upstream proxy head-samples successful requests at 1:N (propagated to us via the - # `tracestate: electric=rate:N` hint) and keeps all >= 500 responses at rate 1. We - # mirror that here, per request: + # The full attribute map is always computed the same way; the only branch is on the + # remote trace context extracted by TraceContextPlug, which is fixed for the lifetime + # of the request: # - # * remote parent sampled: the root span is recording — stamp `SampleRate` = N for - # status < 500 and `SampleRate` = 1 for status >= 500, so sampling-aware tracing - # backends weight aggregates over Electric spans consistently with the upstream - # proxy's own spans; + # * remote parent sampled, or no remote parent (direct traffic): the root span is + # recording — add the attributes to it in bulk. Successful responses carry the + # upstream rate `N`, 5xx responses carry `1`, mirroring the upstream proxy's + # keep-all-errors-at-rate-1 weighting (see TraceContextPlug.sample_rate_attrs/2); # # * remote parent NOT sampled: the parent-based sampler dropped all spans for this - # request, which is the volume win for the (vast) majority of successful requests. - # But for status >= 500 we still want server-side telemetry: synthesize a single - # root span carrying the final request attributes with `SampleRate` = 1, parented - # onto the remote span so it lands in the same trace as the upstream's - # kept-on-error spans; + # request — the volume win for the (vast) majority of successful requests. For + # 5xx responses we still export server-side telemetry: the same attribute map + # goes onto a single synthesized root span with `SampleRate` = 1, parented onto + # the remote span so it lands in the same trace as the upstream's kept-on-error + # spans. Anything else exports nothing. # - # * no remote parent (direct traffic): nothing to do — spans are recorded and - # exported unweighted, as before. - defp stamp_sample_rate(conn) do - case TraceContextPlug.trace_context(conn) do - %{parent_sampled?: true} -> - case TraceContextPlug.sample_rate_attrs(conn, conn.status) do - attrs when map_size(attrs) > 0 -> OpenTelemetry.add_span_attributes(attrs) - _ -> :ok - end + # Called both at span start (status not yet known: the rate hint is stamped as-is and + # the 5xx conditions cannot match) and at emit time, when the final attribute values + # overwrite the initial ones. + defp add_span_attrs_from_conn(conn) do + attrs = + conn + |> open_telemetry_attrs() + |> Map.merge(TraceContextPlug.sample_rate_attrs(conn, conn.status)) + case TraceContextPlug.trace_context(conn) do %{parent_sampled?: false, parent_span_ctx: parent_span_ctx} -> if is_integer(conn.status) and conn.status >= 500 do - export_unsampled_error_span(conn, parent_span_ctx) + export_unsampled_error_span(conn, parent_span_ctx, attrs) end - nil -> - :ok + _sampled_or_direct -> + OpenTelemetry.add_span_attributes(attrs) end conn @@ -518,49 +535,19 @@ defmodule Electric.Plug.ServeShapePlug do # The request ran under a remote-unsampled trace (no recording span exists) but ended # in a 5xx — export one root span after the fact so the error is visible server-side. - # `SampleRate` is hardcoded to 1: error responses ignore the rate hint, mirroring the - # upstream's keep-all-errors-at-rate-1 semantics. - defp export_unsampled_error_span(conn, parent_span_ctx) do - attributes = - conn - |> open_telemetry_attrs() - |> Map.put(TraceContextPlug.sample_rate_attr(), 1) - + # `SampleRate` is set to 1 unconditionally: error responses ignore the rate hint (and + # the hint may be absent entirely), mirroring the upstream's keep-all-errors-at-rate-1 + # semantics. + defp export_unsampled_error_span(conn, parent_span_ctx, attrs) do OpenTelemetry.export_unsampled_remote_span( "Plug_shape_get", - attributes, + Map.put(attrs, TraceContextPlug.sample_rate_attr(), 1), parent_span_ctx, start_time: get_in(conn.private, [:electric_telemetry_span, :start_time]), error: conn.assigns[:error_str] || "HTTP #{conn.status}" ) end - defp get_handle(%{response: %{shape_handle: shape_handle}}), do: shape_handle - defp get_handle(%{request: %{shape_handle: shape_handle}}), do: shape_handle - defp get_handle(_), do: nil - - defp get_live_mode(%{response: %{params: %{live: live}}}), do: live - defp get_live_mode(%{request: %{params: %{live: live}}}), do: live - defp get_live_mode(_), do: false - - # Used as a metric label for the response-size histogram. The goal is low - # cardinality (one value per configured shape root table), so we only emit - # the validated `table` from request params. If validation never ran (e.g. - # the request was rejected before parse), fall back to nil rather than - # echoing back attacker-controlled query string values. - defp get_root_table(%{request: %{params: %{table: table}}}, _conn) when is_binary(table), - do: table - - defp get_root_table(_assigns, _conn), do: nil - - defp add_span_attrs_from_conn(conn) do - conn - |> open_telemetry_attrs() - |> OpenTelemetry.add_span_attributes() - - conn - end - defp open_telemetry_attrs(%Conn{assigns: assigns} = conn) do request = Map.get(assigns, :request, %{}) |> bare_map() params = Map.get(request, :params, %{}) |> bare_map()