libsql-server: fix unbounded memory growth from idle-evicted histogram handles

libsql-server/src/connection/mod.rs

@@ -12,8 +12,8 @@ use crate::auth::Authenticated;
 use crate::error::Error;
 use crate::http::user::timing::sample_time;
 use crate::metrics::{
-    CONCURRENT_CONNECTIONS_COUNT, CONNECTION_ALIVE_DURATION, CONNECTION_CREATE_TIME,
-    TOTAL_RESPONSE_SIZE_HIST,
+    record_connection_alive_duration, record_connection_create_time,
+    record_total_response_size_before_lock, CONCURRENT_CONNECTIONS_COUNT,
 };
 use crate::namespace::meta_store::MetaStore;
 use crate::namespace::NamespaceName;
@@ -360,8 +360,9 @@ impl<F: MakeConnection> MakeConnection for MakeThrottledConnection<F> {
             "Available semaphore units: {}",
             self.semaphore.available_permits()
         );
-        TOTAL_RESPONSE_SIZE_HIST
-            .record(TOTAL_RESPONSE_SIZE.load(std::sync::atomic::Ordering::Relaxed) as f64);
+        record_total_response_size_before_lock(
+            TOTAL_RESPONSE_SIZE.load(std::sync::atomic::Ordering::Relaxed) as f64,
+        );
         let units = self.units_to_take();
         let waiters_guard = WaitersGuard::new(&self.waiters);
         if (waiters_guard.waiters.load(Ordering::Relaxed) as u64) >= self.max_concurrent_requests {
@@ -389,7 +390,7 @@ impl<F: MakeConnection> MakeConnection for MakeThrottledConnection<F> {
         let inner = self.connection_maker.create().await?;
 
         CONCURRENT_CONNECTIONS_COUNT.increment(1.0);
-        CONNECTION_CREATE_TIME.record(before_create.elapsed());
+        record_connection_create_time(before_create.elapsed());
 
         Ok(TrackedConnection {
             permit,
@@ -411,7 +412,7 @@ impl<T> Drop for TrackedConnection<T> {
     fn drop(&mut self) {
         sample_time("connection-duration", self.created_at.elapsed());
         CONCURRENT_CONNECTIONS_COUNT.decrement(1.0);
-        CONNECTION_ALIVE_DURATION.record(self.created_at.elapsed());
+        record_connection_alive_duration(self.created_at.elapsed());
     }
 }
 

libsql-server/src/http/admin/mod.rs

@@ -59,6 +59,15 @@ impl<C> FromRef<Arc<AppState<C>>> for Metrics {
 
 static PROM_HANDLE: Mutex<OnceCell<PrometheusHandle>> = Mutex::new(OnceCell::new());
 
+/// Prometheus recorder configuration shared by the admin server and the metrics tests.
+///
+/// Idle metrics (any kind) are evicted `idle_timeout` after their last update. Histograms are
+/// therefore recorded through the `metrics` macros (see `crate::metrics`), never through cached
+/// handles, so that an evicted histogram is re-registered on its next sample instead of leaking.
+pub(crate) fn prometheus_builder(idle_timeout: Duration) -> PrometheusBuilder {
+    PrometheusBuilder::new().idle_timeout(metrics_util::MetricKindMask::ALL, Some(idle_timeout))
+}
+
 pub async fn run<A, C>(
     acceptor: A,
     user_http_server: Arc<hrana::http::Server>,
@@ -80,19 +89,18 @@ where
         let lock = PROM_HANDLE.lock();
         let prom_handle = lock.get_or_init(|| {
             tracing::info!("initializing prometheus metrics");
-            let b = PrometheusBuilder::new().idle_timeout(
-                metrics_util::MetricKindMask::ALL,
-                Some(Duration::from_secs(120)),
-            );
+            let b = prometheus_builder(Duration::from_secs(120));
 
-            if let Some(app_label) = app_label {
+            let handle = if let Some(app_label) = app_label {
                 b.add_global_label("app", app_label)
                     .add_global_label("version", ver)
                     .install_recorder()
                     .unwrap()
             } else {
                 b.install_recorder().unwrap()
-            }
+            };
+            crate::metrics::describe_histograms();
+            handle
         });
 
         tokio::task::spawn(async move {