fix: raise RLIMIT_NOFILE soft limit at startup

Cargo.toml

@@ -70,6 +70,7 @@ vergen-git2 = { version = "9", features = ["rustc"] }
 rayon = "1.11"
 rand = "0.10"
 rocksdb = "0.24"
+libc = "0.2"
 reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
 eyre = "0.6"
 

bin/ethlambda/Cargo.toml

@@ -33,5 +33,7 @@ eyre.workspace = true
 
 tikv-jemallocator.workspace = true
 
+libc.workspace = true
+
 [build-dependencies]
 vergen-git2.workspace = true

bin/ethlambda/src/fd_limit.rs

@@ -0,0 +1,147 @@
+// Copyright 2016-2020 Parity Technologies (UK) Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Ported from ethrex (`crates/common/fd_limit.rs`); the `Outcome` enum was
+// dropped because the binary's call site doesn't need it.
+
+/// Errors that happen when trying to raise file descriptor resource limit
+#[derive(Debug, thiserror::Error)]
+#[allow(clippy::enum_variant_names)]
+pub enum Error {
+    /// Failed to call sysctl to get max supported value configured in sysctl
+    #[error("Failed to call sysctl to get max supported value configured in sysctl: {0}")]
+    #[cfg(any(target_os = "macos", target_os = "ios"))]
+    FailedToCallSysctl(std::io::Error),
+    /// Failed to get current limit
+    #[error("Failed to get current limit: {0}")]
+    FailedToGetLimit(std::io::Error),
+    /// Failed to set new limit
+    #[error("Failed to set new limit ({from}->{to}): {error}")]
+    FailedToSetLimit {
+        /// Current limit
+        from: u64,
+        /// New desired limit
+        to: u64,
+        /// Low level OS error
+        error: std::io::Error,
+    },
+}
+
+/// Raise the soft open file descriptor resource limit to the smaller of the
+/// kernel limit and the hard resource limit.
+///
+/// darwin_fd_limit exists to work around an issue where launchctl on Mac OS X
+/// defaults the rlimit maxfiles to 256/unlimited. The default soft limit of 256
+/// ends up being far too low for our multithreaded scheduler testing, depending
+/// on the number of cores available.
+#[cfg(any(target_os = "macos", target_os = "ios"))]
+#[allow(clippy::useless_conversion, non_camel_case_types)]
+pub fn raise_fd_limit() -> Result<(), Error> {
+    use std::cmp;
+    use std::io;
+    use std::mem::size_of_val;
+    use std::ptr::null_mut;
+
+    unsafe {
+        static CTL_KERN: libc::c_int = 1;
+        static KERN_MAXFILESPERPROC: libc::c_int = 29;
+
+        // The strategy here is to fetch the current resource limits, read the
+        // kern.maxfilesperproc sysctl value, and bump the soft resource limit for
+        // maxfiles up to the sysctl value.
+
+        // Fetch the kern.maxfilesperproc value
+        let mut mib: [libc::c_int; 2] = [CTL_KERN, KERN_MAXFILESPERPROC];
+        let mut maxfiles: libc::c_int = 0;
+        let mut size: libc::size_t = size_of_val(&maxfiles) as libc::size_t;
+        if libc::sysctl(
+            &mut mib[0],
+            2,
+            &mut maxfiles as *mut _ as *mut _,
+            &mut size,
+            null_mut(),
+            0,
+        ) != 0
+        {
+            return Err(Error::FailedToCallSysctl(io::Error::last_os_error()));
+        }
+
+        // Fetch the current resource limits
+        let mut rlim = libc::rlimit {
+            rlim_cur: 0,
+            rlim_max: 0,
+        };
+        if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 {
+            return Err(Error::FailedToGetLimit(io::Error::last_os_error()));
+        }
+
+        let old_value = rlim.rlim_cur;
+
+        // Bump the soft limit to the smaller of kern.maxfilesperproc and the hard
+        // limit
+        rlim.rlim_cur = cmp::min(maxfiles as libc::rlim_t, rlim.rlim_max);
+
+        // Set our newly-increased resource limit
+        if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
+            return Err(Error::FailedToSetLimit {
+                from: old_value.into(),
+                to: rlim.rlim_cur.into(),
+                error: io::Error::last_os_error(),
+            });
+        }
+
+        Ok(())
+    }
+}
+
+/// Raise the soft open file descriptor resource limit to the hard resource
+/// limit.
+#[cfg(target_os = "linux")]
+#[allow(clippy::useless_conversion, non_camel_case_types)]
+pub fn raise_fd_limit() -> Result<(), Error> {
+    use std::io;
+
+    unsafe {
+        // Fetch the current resource limits
+        let mut rlim = libc::rlimit {
+            rlim_cur: 0,
+            rlim_max: 0,
+        };
+        if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 {
+            return Err(Error::FailedToGetLimit(io::Error::last_os_error()));
+        }
+
+        let old_value = rlim.rlim_cur;
+
+        // Set soft limit to hard limit
+        rlim.rlim_cur = rlim.rlim_max;
+
+        // Set our newly-increased resource limit
+        if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 {
+            return Err(Error::FailedToSetLimit {
+                from: old_value.into(),
+                to: rlim.rlim_cur.into(),
+                error: io::Error::last_os_error(),
+            });
+        }
+
+        Ok(())
+    }
+}
+
+/// Does nothing on unsupported platform
+#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "linux")))]
+pub fn raise_fd_limit() -> Result<(), Error> {
+    Ok(())
+}

bin/ethlambda/src/main.rs

@@ -1,4 +1,5 @@
 mod checkpoint_sync;
+mod fd_limit;
 mod version;
 
 #[cfg(not(target_env = "msvc"))]
@@ -142,6 +143,13 @@ async fn main() -> eyre::Result<()> {
 
     info!(version = version::CLIENT_VERSION, "Starting ethlambda");
 
+    // Raise the soft open-file-descriptor limit to the hard limit. RocksDB
+    // keeps an unbounded table cache (`set_max_open_files(-1)`), so on
+    // containerized hosts with the default ulimit of 1024 the store
+    // eventually panics with `EMFILE`. Fail fast at startup rather than
+    // stall days later when the cache outgrows the limit.
+    fd_limit::raise_fd_limit().wrap_err("failed to raise RLIMIT_NOFILE")?;
+
     // Hive lean spec-asset suites boot the client with
     // HIVE_LEAN_TEST_DRIVER=1 so it skips the consensus/p2p stack and
     // exposes only the `/lean/v0/test_driver/...` endpoints driven by the