diff --git a/Cargo.lock b/Cargo.lock index 6c480657..2512ee5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2017,6 +2017,7 @@ dependencies = [ "ethlambda-types", "eyre", "hex", + "libc", "libssz", "libssz-types", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 5cc0cb47..467971ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,7 @@ vergen-git2 = { version = "9", features = ["rustc"] } rayon = "1.11" rand = "0.10" rocksdb = "0.24" +libc = "0.2" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } eyre = "0.6" diff --git a/bin/ethlambda/Cargo.toml b/bin/ethlambda/Cargo.toml index e5e22ee9..bb1f61bb 100644 --- a/bin/ethlambda/Cargo.toml +++ b/bin/ethlambda/Cargo.toml @@ -33,5 +33,7 @@ eyre.workspace = true tikv-jemallocator.workspace = true +libc.workspace = true + [build-dependencies] vergen-git2.workspace = true diff --git a/bin/ethlambda/src/fd_limit.rs b/bin/ethlambda/src/fd_limit.rs new file mode 100644 index 00000000..2df5b390 --- /dev/null +++ b/bin/ethlambda/src/fd_limit.rs @@ -0,0 +1,147 @@ +// Copyright 2016-2020 Parity Technologies (UK) Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Ported from ethrex (`crates/common/fd_limit.rs`); the `Outcome` enum was +// dropped because the binary's call site doesn't need it. + +/// Errors that happen when trying to raise file descriptor resource limit +#[derive(Debug, thiserror::Error)] +#[allow(clippy::enum_variant_names)] +pub enum Error { + /// Failed to call sysctl to get max supported value configured in sysctl + #[error("Failed to call sysctl to get max supported value configured in sysctl: {0}")] + #[cfg(any(target_os = "macos", target_os = "ios"))] + FailedToCallSysctl(std::io::Error), + /// Failed to get current limit + #[error("Failed to get current limit: {0}")] + FailedToGetLimit(std::io::Error), + /// Failed to set new limit + #[error("Failed to set new limit ({from}->{to}): {error}")] + FailedToSetLimit { + /// Current limit + from: u64, + /// New desired limit + to: u64, + /// Low level OS error + error: std::io::Error, + }, +} + +/// Raise the soft open file descriptor resource limit to the smaller of the +/// kernel limit and the hard resource limit. +/// +/// darwin_fd_limit exists to work around an issue where launchctl on Mac OS X +/// defaults the rlimit maxfiles to 256/unlimited. The default soft limit of 256 +/// ends up being far too low for our multithreaded scheduler testing, depending +/// on the number of cores available. +#[cfg(any(target_os = "macos", target_os = "ios"))] +#[allow(clippy::useless_conversion, non_camel_case_types)] +pub fn raise_fd_limit() -> Result<(), Error> { + use std::cmp; + use std::io; + use std::mem::size_of_val; + use std::ptr::null_mut; + + unsafe { + static CTL_KERN: libc::c_int = 1; + static KERN_MAXFILESPERPROC: libc::c_int = 29; + + // The strategy here is to fetch the current resource limits, read the + // kern.maxfilesperproc sysctl value, and bump the soft resource limit for + // maxfiles up to the sysctl value. + + // Fetch the kern.maxfilesperproc value + let mut mib: [libc::c_int; 2] = [CTL_KERN, KERN_MAXFILESPERPROC]; + let mut maxfiles: libc::c_int = 0; + let mut size: libc::size_t = size_of_val(&maxfiles) as libc::size_t; + if libc::sysctl( + &mut mib[0], + 2, + &mut maxfiles as *mut _ as *mut _, + &mut size, + null_mut(), + 0, + ) != 0 + { + return Err(Error::FailedToCallSysctl(io::Error::last_os_error())); + } + + // Fetch the current resource limits + let mut rlim = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 { + return Err(Error::FailedToGetLimit(io::Error::last_os_error())); + } + + let old_value = rlim.rlim_cur; + + // Bump the soft limit to the smaller of kern.maxfilesperproc and the hard + // limit + rlim.rlim_cur = cmp::min(maxfiles as libc::rlim_t, rlim.rlim_max); + + // Set our newly-increased resource limit + if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 { + return Err(Error::FailedToSetLimit { + from: old_value.into(), + to: rlim.rlim_cur.into(), + error: io::Error::last_os_error(), + }); + } + + Ok(()) + } +} + +/// Raise the soft open file descriptor resource limit to the hard resource +/// limit. +#[cfg(target_os = "linux")] +#[allow(clippy::useless_conversion, non_camel_case_types)] +pub fn raise_fd_limit() -> Result<(), Error> { + use std::io; + + unsafe { + // Fetch the current resource limits + let mut rlim = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + if libc::getrlimit(libc::RLIMIT_NOFILE, &mut rlim) != 0 { + return Err(Error::FailedToGetLimit(io::Error::last_os_error())); + } + + let old_value = rlim.rlim_cur; + + // Set soft limit to hard limit + rlim.rlim_cur = rlim.rlim_max; + + // Set our newly-increased resource limit + if libc::setrlimit(libc::RLIMIT_NOFILE, &rlim) != 0 { + return Err(Error::FailedToSetLimit { + from: old_value.into(), + to: rlim.rlim_cur.into(), + error: io::Error::last_os_error(), + }); + } + + Ok(()) + } +} + +/// Does nothing on unsupported platform +#[cfg(not(any(target_os = "macos", target_os = "ios", target_os = "linux")))] +pub fn raise_fd_limit() -> Result<(), Error> { + Ok(()) +} diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index d361b969..2c16c9e1 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1,4 +1,5 @@ mod checkpoint_sync; +mod fd_limit; mod version; #[cfg(not(target_env = "msvc"))] @@ -142,6 +143,13 @@ async fn main() -> eyre::Result<()> { info!(version = version::CLIENT_VERSION, "Starting ethlambda"); + // Raise the soft open-file-descriptor limit to the hard limit. RocksDB + // keeps an unbounded table cache (`set_max_open_files(-1)`), so on + // containerized hosts with the default ulimit of 1024 the store + // eventually panics with `EMFILE`. Fail fast at startup rather than + // stall days later when the cache outgrows the limit. + fd_limit::raise_fd_limit().wrap_err("failed to raise RLIMIT_NOFILE")?; + // Hive lean spec-asset suites boot the client with // HIVE_LEAN_TEST_DRIVER=1 so it skips the consensus/p2p stack and // exposes only the `/lean/v0/test_driver/...` endpoints driven by the