From ed6c13e44d2babaec9fceb490ee3713bc4560984 Mon Sep 17 00:00:00 2001 From: Tudor Golubenco Date: Fri, 31 Jul 2026 14:59:25 +0200 Subject: [PATCH 1/2] Make tcp keepalive and tcp user timeout configurable --- packages/sync-service/config/runtime.exs | 19 ++++++ packages/sync-service/lib/electric/config.ex | 7 ++ .../connection/manager/connection_resolver.ex | 66 ++++++++++++++++++- website/docs/sync/api/config.md | 50 ++++++++++++++ 4 files changed, 140 insertions(+), 2 deletions(-) diff --git a/packages/sync-service/config/runtime.exs b/packages/sync-service/config/runtime.exs index c122e4f4e1..6dec9711ae 100644 --- a/packages/sync-service/config/runtime.exs +++ b/packages/sync-service/config/runtime.exs @@ -265,6 +265,25 @@ config :electric, prometheus_port: prometheus_port, live_dashboard_port: live_dashboard_port, db_pool_size: env!("ELECTRIC_DB_POOL_SIZE", :integer, nil), + db_tcp_keepalive_idle: + env!( + "ELECTRIC_DATABASE_TCP_KEEPALIVE_IDLE", + &Electric.Config.parse_human_readable_time!/1, + nil + ), + db_tcp_keepalive_interval: + env!( + "ELECTRIC_DATABASE_TCP_KEEPALIVE_INTERVAL", + &Electric.Config.parse_human_readable_time!/1, + nil + ), + db_tcp_keepalive_count: env!("ELECTRIC_DATABASE_TCP_KEEPALIVE_COUNT", :integer, nil), + db_tcp_user_timeout: + env!( + "ELECTRIC_DATABASE_TCP_USER_TIMEOUT", + &Electric.Config.parse_human_readable_time!/1, + nil + ), replication_stream_id: replication_stream_id, replication_slot_temporary?: env!("CLEANUP_REPLICATION_SLOTS_ON_SHUTDOWN", :boolean, nil), replication_slot_temporary_random_name?: diff --git a/packages/sync-service/lib/electric/config.ex b/packages/sync-service/lib/electric/config.ex index 31a0530d2a..b42d8b1c2d 100644 --- a/packages/sync-service/lib/electric/config.ex +++ b/packages/sync-service/lib/electric/config.ex @@ -45,6 +45,13 @@ defmodule Electric.Config do ## Database provided_database_id: "single_stack", db_pool_size: 20, + # TCP-level liveness detection for database connections. All nil by + # default, leaving the OS defaults in place. See + # Electric.Connection.Manager.ConnectionResolver for what these do. + db_tcp_keepalive_idle: nil, + db_tcp_keepalive_interval: nil, + db_tcp_keepalive_count: nil, + db_tcp_user_timeout: nil, replication_stream_id: "default", replication_slot_temporary?: false, replication_slot_temporary_random_name?: false, diff --git a/packages/sync-service/lib/electric/connection/manager/connection_resolver.ex b/packages/sync-service/lib/electric/connection/manager/connection_resolver.ex index 1f0b6a63a1..59f63727e9 100644 --- a/packages/sync-service/lib/electric/connection/manager/connection_resolver.ex +++ b/packages/sync-service/lib/electric/connection/manager/connection_resolver.ex @@ -4,6 +4,14 @@ defmodule Electric.Connection.Manager.ConnectionResolver do require Logger + # Socket option numbers from the Linux headers, for use with :inet's `:raw` + # option. IPPROTO_TCP is from netinet/in.h, the rest from netinet/tcp.h. + @ipproto_tcp 6 + @tcp_keepidle 4 + @tcp_keepintvl 5 + @tcp_keepcnt 6 + @tcp_user_timeout 18 + defmodule Connection do @moduledoc false @behaviour Postgrex.SimpleConnection @@ -173,16 +181,70 @@ defmodule Electric.Connection.Manager.ConnectionResolver do end defp populate_tcp_opts(connection_opts) do - tcp_opts = + inet_opts = if connection_opts[:ipv6] do [:inet6] else [] end - Keyword.put(connection_opts, :socket_options, tcp_opts) + Keyword.put(connection_opts, :socket_options, inet_opts ++ tcp_liveness_opts()) end + # Options that let the kernel notice a connection whose peer has gone away + # without closing it. + # + # The replication connection can sit idle for long stretches and only writes a + # StandbyStatusUpdate every `wal_sender_timeout / 3`. If the peer disappears + # without a FIN or an RST -- a hard-killed container or proxy, a dropped route + # -- the socket stays ESTABLISHED here and those writes are simply + # retransmitted. Nothing surfaces an error until the kernel gives up, which on + # Linux takes `net.ipv4.tcp_retries2` retries (~15 minutes by default). Until + # then replication is stalled with no indication that anything is wrong. + # + # SO_KEEPALIVE makes the kernel probe an idle connection, and TCP_USER_TIMEOUT + # caps how long unacknowledged data may stay outstanding before the connection + # is dropped -- the latter also bounds detection while data *is* being sent, + # which keepalive alone does not. + # + # Everything here is opt-in: with no configuration we emit no options and + # inherit the OS defaults. + defp tcp_liveness_opts do + keepalive_idle = Electric.Config.get_env(:db_tcp_keepalive_idle) + keepalive_interval = Electric.Config.get_env(:db_tcp_keepalive_interval) + keepalive_count = Electric.Config.get_env(:db_tcp_keepalive_count) + user_timeout = Electric.Config.get_env(:db_tcp_user_timeout) + + keepalive_opt = + if is_nil(keepalive_idle) and is_nil(keepalive_interval) and is_nil(keepalive_count) do + [] + else + [{:keepalive, true}] + end + + keepalive_opt ++ + raw_tcp_opt(@tcp_keepidle, ms_to_sec(keepalive_idle)) ++ + raw_tcp_opt(@tcp_keepintvl, ms_to_sec(keepalive_interval)) ++ + raw_tcp_opt(@tcp_keepcnt, keepalive_count) ++ + raw_tcp_opt(@tcp_user_timeout, user_timeout) + end + + # The raw socket options below are Linux-specific: the option numbers differ + # on other platforms and TCP_USER_TIMEOUT has no equivalent at all. Skip them + # elsewhere so a developer on macOS gets a working connection rather than an + # obscure einval, while :keepalive (a portable inet option) still applies. + defp raw_tcp_opt(_opt, nil), do: [] + + defp raw_tcp_opt(opt, value) when is_integer(value) do + case :os.type() do + {:unix, :linux} -> [{:raw, @ipproto_tcp, opt, <>}] + _ -> [] + end + end + + defp ms_to_sec(nil), do: nil + defp ms_to_sec(ms) when is_integer(ms), do: max(div(ms, 1000), 1) + defp mutate_based_on_error(%Postgrex.Error{message: "ssl not available"} = error, conn_opts) do maybe_fallback_to_no_ssl(conn_opts, error) end diff --git a/website/docs/sync/api/config.md b/website/docs/sync/api/config.md index 5add4bd4a1..0ffe541efd 100644 --- a/website/docs/sync/api/config.md +++ b/website/docs/sync/api/config.md @@ -96,6 +96,56 @@ How many connections Electric opens as a pool for handling shape queries. +### ELECTRIC_DATABASE_TCP_KEEPALIVE_IDLE + + + +Enable TCP keepalive on database connections and set how long a connection may sit idle before the kernel starts sending probes. + +By default Electric leaves the operating system's settings in place. Setting this lets an idle replication connection detect the loss and reconnect sooner. + +See also `ELECTRIC_DATABASE_TCP_USER_TIMEOUT`, which bounds detection while data is actually in flight. + + + +### ELECTRIC_DATABASE_TCP_KEEPALIVE_INTERVAL + + + +How long to wait between individual TCP keepalive probes. Enables keepalive if set on its own. + + + +### ELECTRIC_DATABASE_TCP_KEEPALIVE_COUNT + + + +How many unanswered TCP keepalive probes before the connection is dropped. Enables keepalive if set on its own. + + + +### ELECTRIC_DATABASE_TCP_USER_TIMEOUT + + + +The maximum time data may remain unacknowledged before the connection is dropped (Linux `TCP_USER_TIMEOUT`). + +This complements the keepalive settings above. Keepalive probes only run while a connection is idle, whereas this bounds how long Electric keeps retransmitting a write to a peer that has stopped responding. + + + ### ELECTRIC_DATABASE_CA_CERTIFICATE_FILE Date: Fri, 31 Jul 2026 15:03:53 +0200 Subject: [PATCH 2/2] Simplify comment --- .../connection/manager/connection_resolver.ex | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/sync-service/lib/electric/connection/manager/connection_resolver.ex b/packages/sync-service/lib/electric/connection/manager/connection_resolver.ex index 59f63727e9..39dd80d756 100644 --- a/packages/sync-service/lib/electric/connection/manager/connection_resolver.ex +++ b/packages/sync-service/lib/electric/connection/manager/connection_resolver.ex @@ -191,16 +191,7 @@ defmodule Electric.Connection.Manager.ConnectionResolver do Keyword.put(connection_opts, :socket_options, inet_opts ++ tcp_liveness_opts()) end - # Options that let the kernel notice a connection whose peer has gone away - # without closing it. - # - # The replication connection can sit idle for long stretches and only writes a - # StandbyStatusUpdate every `wal_sender_timeout / 3`. If the peer disappears - # without a FIN or an RST -- a hard-killed container or proxy, a dropped route - # -- the socket stays ESTABLISHED here and those writes are simply - # retransmitted. Nothing surfaces an error until the kernel gives up, which on - # Linux takes `net.ipv4.tcp_retries2` retries (~15 minutes by default). Until - # then replication is stalled with no indication that anything is wrong. + # Options for configuring TCP keepalives and TCP user timeout. # # SO_KEEPALIVE makes the kernel probe an idle connection, and TCP_USER_TIMEOUT # caps how long unacknowledged data may stay outstanding before the connection