fix(runtime-core): bound the stop-path disk-handle scan at 30 s with an injectable probe (HACK-1328)

packages/runtime-core/README.md

@@ -1514,7 +1514,12 @@ clean `down` keeps it clean. Pools stopped before this marker existed qualify af
 next clean `down`. Clean also requires a creation receipt, rootfs pin, both disk
 identities and declared sizes, the recorded executable and user, a dead recorded
 provider and no pending owner update; these receipt checks are read-only, take no
-provider lock and run around every sample, including `runtime probe`.
+provider lock and run around every sample, including `runtime probe`. The disk-handle
+scan (`lsof` over both disks) that proves the provider's release is bounded at 30 s on
+the stop paths (`down`, `recover`) and in release checks, well above the slowest scan
+measured on a loaded host. A scan that still does not finish ends the stop as
+`provider_timeout_uncertain`, without the marker. `scripts/measure-lsof-scan.ts`
+re-measures a host.
 
 The reserve is used only when a start needs it. Once the startup lease is held, a
 start whose every qualifying sample, including the lease recheck, also fits the 58 GiB

packages/runtime-core/src/provider/lifecycle.rs

@@ -1,4 +1,5 @@
 mod admission_pool;
+mod disk_handles;
 pub mod host_filesystem;
 mod interrupted;
 mod operating_guard;
@@ -1997,24 +1998,26 @@ impl Drop for VmLock {
     }
 }
 
-/// How long `down`, `recover` and other stop paths wait for `lsof` over the owned disks.
+/// The bound [`lock_absent_disks`] gives the disk-handle scan: the production stop-path
+/// bound [`disk_handles::STOP_HANDLE_WAIT`].
 #[cfg(not(test))]
-const STOP_HANDLE_WAIT: Duration = Duration::from_secs(5);
-/// The unit-test build of [`STOP_HANDLE_WAIT`]. Release-check fixtures prove what the scan
+const STOP_SCAN_WAIT: Duration = disk_handles::STOP_HANDLE_WAIT;
+/// The unit-test build of [`STOP_SCAN_WAIT`]. Release-check fixtures prove what the scan
 /// reports (status 1, no output), not how fast it reports it, and they run beside other
-/// tests' whole-host scans; on a loaded host one scan took longer than 5 s and the
-/// fixture failed as `provider_timeout_uncertain` before its result under test (HACK-1457).
-/// Tests of the timeout itself pass their own bound to [`lock_absent_disks_within`].
+/// tests' work; on a loaded host one scan took longer than 5 s and the fixture failed as
+/// `provider_timeout_uncertain` before its result under test (HACK-1457). The production
+/// bounds themselves are tested on a fake clock in [`disk_handles`].
 #[cfg(test)]
-const STOP_HANDLE_WAIT: Duration = super::test_executable::LOADED_HOST_SCAN_WAIT;
+const STOP_SCAN_WAIT: Duration = super::test_executable::LOADED_HOST_SCAN_WAIT;
 
 /// Keep this guard through alias restoration and the stopped receipt.
 fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result<VmLock, CandidateError> {
-    lock_absent_disks_within(candidate, owner, STOP_HANDLE_WAIT)
+    lock_absent_disks_within(candidate, owner, STOP_SCAN_WAIT)
 }
 
-/// [`lock_absent_disks`] with a caller-chosen bound on the `lsof` disk-handle scan. A scan
-/// that does not finish within `handle_wait` fails as `provider_timeout_uncertain`.
+/// [`lock_absent_disks`] with a caller-chosen bound on the disk-handle scan
+/// ([`disk_handles::disks_closed`]). A scan that does not finish within `handle_wait` fails
+/// as `provider_timeout_uncertain`.
 fn lock_absent_disks_within(
     candidate: &Candidate,
     owner: &Owner,
@@ -2041,20 +2044,12 @@ fn lock_absent_disks_within(
         ));
     }
     let vm_lock = VmLock(vm_lock);
-    let handles = process::capture(
-        process::clean_command(Path::new("/usr/sbin/lsof"))
-            .args(["-n", "-P", "-t", "--"])
-            .arg(directory.join("storage.raw"))
-            .arg(directory.join("overlay.raw")),
+    disk_handles::disks_closed(
+        &disk_handles::Lsof::host(),
+        &process::MonotonicClock::new(),
+        &[directory.join("storage.raw"), directory.join("overlay.raw")],
         handle_wait,
     )?;
-    if handles.status.code() != Some(1) || !handles.stdout.is_empty() || !handles.stderr.is_empty()
-    {
-        return Err(CandidateError::new(
-            "stop_uncertain",
-            "Cannot confirm that all handles to owned disks are closed.",
-        ));
-    }
     verify_disks(candidate, owner)?;
     Ok(vm_lock)
 }

packages/runtime-core/src/provider/lifecycle/admission_pool.rs

@@ -29,20 +29,13 @@
 //! needs a [`StartPermit`], which only the pre-boot admission ([`admit_boot`]) followed by
 //! the final leased recheck ([`BootAdmission::release`]) issues.
 use super::{
-    Owner, admission, binary, io, lock_absent_disks_within, root, sockets_absent,
+    Owner, admission, binary, disk_handles, io, lock_absent_disks_within, root, sockets_absent,
     verify_disk_allocation, verify_disks, verify_live,
 };
 use crate::provider::{Profile, config_audit, identity, state};
 use crate::{Candidate, CandidateError};
 use serde_json::Value;
 use std::fs;
-use std::time::Duration;
-
-/// How long a release check waits for `lsof` over the owned disks. `lsof` scans every
-/// process, so it is slow on a loaded host (5.4–6.5 s at about 1,850 processes and load
-/// 30–80). The wait is longer than `down`'s because only a start that needs the reserve runs
-/// it, under the startup lease; a scan that still does not finish refuses the reserve.
-const RELEASE_HANDLE_WAIT: Duration = Duration::from_secs(30);
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 enum Basis {
@@ -251,7 +244,7 @@ fn released(
 ) -> Result<(), StoppedRefusal> {
     let check = |error: CandidateError| StoppedRefusal::from_check(&error);
     lease.verify_path(&root(candidate)).map_err(check)?;
-    lock_absent_disks_within(candidate, owner, RELEASE_HANDLE_WAIT).map_err(check)?;
+    lock_absent_disks_within(candidate, owner, disk_handles::RELEASE_HANDLE_WAIT).map_err(check)?;
     sockets_absent(owner).map_err(check)?;
     config_audit::verify(candidate, owner).map_err(check)
 }