Add TCP keepalive options for database connections

packages/sync-service/config/runtime.exs

@@ -265,6 +265,25 @@ config :electric,
   prometheus_port: prometheus_port,
   live_dashboard_port: live_dashboard_port,
   db_pool_size: env!("ELECTRIC_DB_POOL_SIZE", :integer, nil),
+  db_tcp_keepalive_idle:
+    env!(
+      "ELECTRIC_DATABASE_TCP_KEEPALIVE_IDLE",
+      &Electric.Config.parse_human_readable_time!/1,
+      nil
+    ),
+  db_tcp_keepalive_interval:
+    env!(
+      "ELECTRIC_DATABASE_TCP_KEEPALIVE_INTERVAL",
+      &Electric.Config.parse_human_readable_time!/1,
+      nil
+    ),
+  db_tcp_keepalive_count: env!("ELECTRIC_DATABASE_TCP_KEEPALIVE_COUNT", :integer, nil),
+  db_tcp_user_timeout:
+    env!(
+      "ELECTRIC_DATABASE_TCP_USER_TIMEOUT",
+      &Electric.Config.parse_human_readable_time!/1,
+      nil
+    ),
   replication_stream_id: replication_stream_id,
   replication_slot_temporary?: env!("CLEANUP_REPLICATION_SLOTS_ON_SHUTDOWN", :boolean, nil),
   replication_slot_temporary_random_name?:

packages/sync-service/lib/electric/config.ex

@@ -45,6 +45,13 @@ defmodule Electric.Config do
     ## Database
     provided_database_id: "single_stack",
     db_pool_size: 20,
+    # TCP-level liveness detection for database connections. All nil by
+    # default, leaving the OS defaults in place. See
+    # Electric.Connection.Manager.ConnectionResolver for what these do.
+    db_tcp_keepalive_idle: nil,
+    db_tcp_keepalive_interval: nil,
+    db_tcp_keepalive_count: nil,
+    db_tcp_user_timeout: nil,
     replication_stream_id: "default",
     replication_slot_temporary?: false,
     replication_slot_temporary_random_name?: false,

packages/sync-service/lib/electric/connection/manager/connection_resolver.ex

@@ -4,6 +4,14 @@ defmodule Electric.Connection.Manager.ConnectionResolver do
 
   require Logger
 
+  # Socket option numbers from the Linux headers, for use with :inet's `:raw`
+  # option. IPPROTO_TCP is from netinet/in.h, the rest from netinet/tcp.h.
+  @ipproto_tcp 6
+  @tcp_keepidle 4
+  @tcp_keepintvl 5
+  @tcp_keepcnt 6
+  @tcp_user_timeout 18
+
   defmodule Connection do
     @moduledoc false
     @behaviour Postgrex.SimpleConnection
@@ -173,16 +181,61 @@ defmodule Electric.Connection.Manager.ConnectionResolver do
   end
 
   defp populate_tcp_opts(connection_opts) do
-    tcp_opts =
+    inet_opts =
       if connection_opts[:ipv6] do
         [:inet6]
       else
         []
       end
 
-    Keyword.put(connection_opts, :socket_options, tcp_opts)
+    Keyword.put(connection_opts, :socket_options, inet_opts ++ tcp_liveness_opts())
+  end
+
+  # Options for configuring TCP keepalives and TCP user timeout.
+  #
+  # SO_KEEPALIVE makes the kernel probe an idle connection, and TCP_USER_TIMEOUT
+  # caps how long unacknowledged data may stay outstanding before the connection
+  # is dropped -- the latter also bounds detection while data *is* being sent,
+  # which keepalive alone does not.
+  #
+  # Everything here is opt-in: with no configuration we emit no options and
+  # inherit the OS defaults.
+  defp tcp_liveness_opts do
+    keepalive_idle = Electric.Config.get_env(:db_tcp_keepalive_idle)
+    keepalive_interval = Electric.Config.get_env(:db_tcp_keepalive_interval)
+    keepalive_count = Electric.Config.get_env(:db_tcp_keepalive_count)
+    user_timeout = Electric.Config.get_env(:db_tcp_user_timeout)
+
+    keepalive_opt =
+      if is_nil(keepalive_idle) and is_nil(keepalive_interval) and is_nil(keepalive_count) do
+        []
+      else
+        [{:keepalive, true}]
+      end
+
+    keepalive_opt ++
+      raw_tcp_opt(@tcp_keepidle, ms_to_sec(keepalive_idle)) ++
+      raw_tcp_opt(@tcp_keepintvl, ms_to_sec(keepalive_interval)) ++
+      raw_tcp_opt(@tcp_keepcnt, keepalive_count) ++
+      raw_tcp_opt(@tcp_user_timeout, user_timeout)
   end
 
+  # The raw socket options below are Linux-specific: the option numbers differ
+  # on other platforms and TCP_USER_TIMEOUT has no equivalent at all. Skip them
+  # elsewhere so a developer on macOS gets a working connection rather than an
+  # obscure einval, while :keepalive (a portable inet option) still applies.
+  defp raw_tcp_opt(_opt, nil), do: []
+
+  defp raw_tcp_opt(opt, value) when is_integer(value) do
+    case :os.type() do
+      {:unix, :linux} -> [{:raw, @ipproto_tcp, opt, <<value::native-32>>}]
+      _ -> []
+    end
+  end
+
+  defp ms_to_sec(nil), do: nil
+  defp ms_to_sec(ms) when is_integer(ms), do: max(div(ms, 1000), 1)
+
   defp mutate_based_on_error(%Postgrex.Error{message: "ssl not available"} = error, conn_opts) do
     maybe_fallback_to_no_ssl(conn_opts, error)
   end

website/docs/sync/api/config.md

@@ -96,6 +96,56 @@ How many connections Electric opens as a pool for handling shape queries.
 
 </EnvVarConfig>
 
+### ELECTRIC_DATABASE_TCP_KEEPALIVE_IDLE
+
+<EnvVarConfig
+    name="ELECTRIC_DATABASE_TCP_KEEPALIVE_IDLE"
+    optional="true"
+    example="30s">
+
+Enable TCP keepalive on database connections and set how long a connection may sit idle before the kernel starts sending probes.
+
+By default Electric leaves the operating system's settings in place. Setting this lets an idle replication connection detect the loss and reconnect sooner.
+
+See also `ELECTRIC_DATABASE_TCP_USER_TIMEOUT`, which bounds detection while data is actually in flight.
+
+</EnvVarConfig>
+
+### ELECTRIC_DATABASE_TCP_KEEPALIVE_INTERVAL
+
+<EnvVarConfig
+    name="ELECTRIC_DATABASE_TCP_KEEPALIVE_INTERVAL"
+    optional="true"
+    example="10s">
+
+How long to wait between individual TCP keepalive probes. Enables keepalive if set on its own.
+
+</EnvVarConfig>
+
+### ELECTRIC_DATABASE_TCP_KEEPALIVE_COUNT
+
+<EnvVarConfig
+    name="ELECTRIC_DATABASE_TCP_KEEPALIVE_COUNT"
+    optional="true"
+    example="3">
+
+How many unanswered TCP keepalive probes before the connection is dropped. Enables keepalive if set on its own.
+
+</EnvVarConfig>
+
+### ELECTRIC_DATABASE_TCP_USER_TIMEOUT
+
+<EnvVarConfig
+    name="ELECTRIC_DATABASE_TCP_USER_TIMEOUT"
+    optional="true"
+    example="60s">
+
+The maximum time data may remain unacknowledged before the connection is dropped (Linux `TCP_USER_TIMEOUT`).
+
+This complements the keepalive settings above. Keepalive probes only run while a connection is idle, whereas this bounds how long Electric keeps retransmitting a write to a peer that has stopped responding.
+
+</EnvVarConfig>
+
 ### ELECTRIC_DATABASE_CA_CERTIFICATE_FILE
 
 <EnvVarConfig