diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index b5f309b4c..661d2d4d4 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -366,12 +366,20 @@ v4 state is enabled. Runtime startup is a further explicit operation subject to host resource admission. Read `--help` for the candidate command surface. The development profile reserves the declared 32 GiB storage disk and 10 GiB -overlay ceiling plus 16 GiB of host disk space before startup. Its unchanged memory -budget is a 10 GiB free-plus-file-cache estimate: 6 GiB guest, 2 GiB provisional -provider overhead and 2 GiB host reserve. Normal and warning macOS pressure can -qualify only with that headroom, normal thermal/performance state and stable -swapouts. Development samples before the startup lease and again under the lease -before allocation, keeping the same swapout baseline across both windows. +overlay ceiling plus 16 GiB of host disk space before a fresh startup. Starting +again on a verified running pool, or restarting an exactly clean stopped pool whose +last `down` completed, reuses existing disks and reserves only the 16 GiB host +reserve; `runtime probe --profile development --json` reports the basis in +`disk_budget_basis`, and `retained_reserve_refusal` names why a stopped pool's +disks were charged again. A restart uses that reserve only when free disk is +below the fresh budget; with enough room it starts on the fresh budget as before. +Any other stopped pool keeps the fresh budget. The +unchanged memory budget is a 10 GiB free-plus-file-cache estimate: 6 GiB guest, +2 GiB provisional provider overhead and 2 GiB host reserve. Normal and warning +macOS pressure can qualify only with that headroom, normal thermal/performance +state and stable swapouts. Development samples before the startup lease and again +under the lease before allocation, keeping the same swapout baseline across both +windows. Critical, unknown or incomplete observations refuse; there is no bypass option. It reports host load without refusing solely because unrelated jobs raise the one-minute average. The research profile keeps its normal-only pressure policy, diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index eb308245d..6455b7936 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -1461,8 +1461,54 @@ host reserve, with memory-pressure, memory-headroom and thermal checks unchanged requires matching profile, creation receipt, native process/PID-file identity, machine name and both disks' current identities and declared sizes. Every sample and the acquired startup lease recheck the selected owner; changed ownership -refuses, and a reserve-qualified request cannot enter VM create or boot. Stopped, -missing or unproved capacity retains fresh-allocation requirements or refuses. +refuses, and a reserve-qualified request cannot enter VM create or boot. Missing or +unproved capacity retains fresh-allocation requirements or refuses. + +Restarting an exactly clean stopped Development pool also requires only the 16 GiB +host reserve (`verified-stopped-owned-pool-host-reserve`), because it reuses its +retained disks instead of allocating new ones. Clean means phase `stopped` plus the +clean-stop marker that only a completed `down` records, in the same receipt write as +that phase, after the guest acknowledged flush and shutdown of a quiesced engine and the +provider's absence was proven. The marker names the stopped boot's guest boot and engine +identity and must match both the receipt and the quiescence receipt. Every other phase +write and every new boot clear it, so an interrupted, uncertain (`stop_uncertain`), +recovered-unclean, before-engine or failed-boot stop never qualifies, even after a start +refused before boot rewrites the phase to `stopped`; a start refused before boot after a +clean `down` keeps it clean. Pools stopped before this marker existed qualify after their +next clean `down`. Clean also requires a creation receipt, rootfs pin, both disk +identities and declared sizes, the recorded executable and user, a dead recorded +provider and no pending owner update; these receipt checks are read-only, take no +provider lock and run around every sample, including `runtime probe`. + +The reserve is used only when a start needs it. Once the startup lease is held, a +start whose every qualifying sample, including the lease recheck, also fits the 58 GiB +fresh budget drops the stopped selection: it records those samples on the fresh basis +and continues exactly as a start without a selection, with no release checks. Only a +start whose free disk lies between the host reserve and the fresh budget keeps the +selection. `runtime probe` takes no lease and settles no basis, so it reports the reserve +basis for a qualifying stopped pool whatever its headroom; the start's `admission.json` +records the basis that start used. Under the lease, before any restart effect and again immediately before +`machine start`, its release checks also require a free VM lock, no open disk handle +(`lsof`, bounded at 30 seconds because it scans every process and is slow on a loaded +host), no owned listener and an unchanged audited provider configuration. The short-HOME +alias is checked on every owner load, and the rootfs content against its pin before +boot. + +Each refusal maps to a closed code: `retained_reserve_foreign_owner`, +`retained_reserve_active_owner`, `retained_reserve_dirty_owner` or +`retained_reserve_stale_owner`. A created stopped pool that fails the receipt checks +keeps the 58 GiB fresh-allocation budget, and the admission report carries +`retained_reserve_refusal` with that code; the reserve is never relaxed. A selected pool +that fails a later check refuses the start with the code. Because only a start that +needs the reserve runs the release checks, a release-check refusal, including an `lsof` +scan that times out, never refuses a start whose samples all fit the fresh budget. `machine create` requires an +authority only a start without a reserve selection obtains, and `machine start` requires +one issued only by the pre-boot admission on the qualifying basis followed by the final +leased recheck. The start sequence itself is typed: the check before any restart effect +needs the basis that only the leased settle step returns, and both `machine create` and +the pre-boot admission need the authority that only that check returns, so a start that +skips or reorders a step does not compile. Pressure, thermal, swapout and +memory-headroom checks are unchanged. ### Native persistent-data identity codec diff --git a/packages/runtime-core/src/provider/admission.rs b/packages/runtime-core/src/provider/admission.rs index 53d918756..ff2f49ba6 100644 --- a/packages/runtime-core/src/provider/admission.rs +++ b/packages/runtime-core/src/provider/admission.rs @@ -101,6 +101,10 @@ pub struct Admission { pub swapouts: Option, pub admitted: bool, pub reasons: Vec, + /// For a fresh-budget sample beside a created stopped Development pool: the closed code + /// explaining why its retained disks did not qualify for the host reserve. + #[serde(skip_serializing_if = "Option::is_none")] + pub retained_reserve_refusal: Option<&'static str>, } pub fn parse_free_memory(text: &str) -> Result { @@ -233,36 +237,93 @@ pub fn probe(path: &Path) -> Result { } pub fn probe_for(path: &Path, profile: Profile) -> Result { - probe_with_disk_budget(path, profile, false) + probe_with_disk_budget(path, profile, OwnedPool::Fresh) } -/// Only the lifecycle's independently verified live development pool may use this budget. -pub(super) fn probe_owned_development(path: &Path) -> Result { - probe_with_disk_budget(path, Profile::Development, true) +/// Which existing Development disk allocation an admission may treat as already paid. +/// Only the lifecycle's admission-pool selection names an owned pool, after proving it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum OwnedPool { + /// No proven allocation: charge the declared storage, overlay and host reserve. + Fresh, + /// An independently verified live development pool. + Running, + /// An exactly clean stopped development pool whose verified retained disks a + /// restart reuses without `machine create`. + Stopped, } -fn disk_budget(profile: Profile, owned_live: bool) -> (u64, &'static str) { - if profile == Profile::Development && owned_live { - ( +/// Only the lifecycle's independently verified development pool may use this budget. +pub(super) fn probe_owned_development( + path: &Path, + owned: OwnedPool, +) -> Result { + probe_with_disk_budget(path, Profile::Development, owned) +} + +fn disk_budget(profile: Profile, owned: OwnedPool) -> (u64, &'static str) { + match (profile, owned) { + (Profile::Research, _) => (disk_floor_bytes(profile), "research-floor"), + (Profile::Development, OwnedPool::Fresh) => ( + disk_floor_bytes(profile), + "new-pool-storage-overlay-host-reserve", + ), + (Profile::Development, OwnedPool::Running) => ( DEVELOPMENT_HOST_DISK_RESERVE_GIB * 1024 * 1024 * 1024, "verified-live-owned-pool-host-reserve", - ) - } else { - ( - disk_floor_bytes(profile), - if profile == Profile::Research { - "research-floor" - } else { - "new-pool-storage-overlay-host-reserve" - }, - ) + ), + (Profile::Development, OwnedPool::Stopped) => ( + DEVELOPMENT_HOST_DISK_RESERVE_GIB * 1024 * 1024 * 1024, + "verified-stopped-owned-pool-host-reserve", + ), } } +/// The fixed refusal for free disk below the selected budget, or none when it fits. +fn disk_shortfall(profile: Profile, owned: OwnedPool, available: u64) -> Option<&'static str> { + if available >= disk_budget(profile, owned).0 { + return None; + } + Some(match (profile, owned) { + (Profile::Research, _) => "Free disk is below the 100 GiB research floor.", + (Profile::Development, OwnedPool::Fresh) => { + "Free disk is below the development VM storage and host reserve budget." + } + (Profile::Development, OwnedPool::Running) => { + "Free disk is below the host reserve for the verified running development pool." + } + (Profile::Development, OwnedPool::Stopped) => { + "Free disk is below the host reserve for the verified stopped development pool." + } + }) +} + +/// Whether an admitted Development sample also passes the unchanged fresh-allocation budget. +/// Free disk is the only predicate that depends on the disk basis, so an admitted sample +/// whose observed free disk fits the fresh budget is admitted on that budget too. +pub(super) fn fits_fresh_budget(sample: &Admission) -> bool { + sample.profile == Profile::Development + && sample.admitted + && sample.reasons.is_empty() + && sample.disk_free_bytes.is_some_and(|free| { + disk_shortfall(Profile::Development, OwnedPool::Fresh, free).is_none() + }) +} + +/// Record a sample that [`fits_fresh_budget`] on the fresh-allocation budget, exactly as a +/// start without an owned pool would have recorded the same observation. +pub(super) fn rebase_fresh(sample: &mut Admission) { + debug_assert!(fits_fresh_budget(sample)); + let (minimum, basis) = disk_budget(Profile::Development, OwnedPool::Fresh); + sample.minimum_disk_free_bytes = minimum; + sample.disk_budget_basis = basis; + sample.retained_reserve_refusal = None; +} + fn probe_with_disk_budget( path: &Path, profile: Profile, - owned_live: bool, + owned: OwnedPool, ) -> Result { let supported = cfg!(all(target_os = "macos", target_arch = "aarch64")); let mut result = Admission { @@ -279,14 +340,15 @@ fn probe_with_disk_budget( memory_pressure_normal: false, memory_pressure: MemoryPressure::from_level(None), disk_free_bytes: None, - minimum_disk_free_bytes: disk_budget(profile, owned_live).0, - disk_budget_basis: disk_budget(profile, owned_live).1, + minimum_disk_free_bytes: disk_budget(profile, owned).0, + disk_budget_basis: disk_budget(profile, owned).1, one_minute_load: None, load_ceiling: None, thermal_normal: false, swapouts: None, admitted: false, reasons: Vec::new(), + retained_reserve_refusal: None, }; if !supported { result @@ -369,19 +431,8 @@ fn probe_with_disk_budget( let available = u64::try_from(available) .map_err(|_| CandidateError::new("admission_unavailable", "Invalid disk headroom."))?; result.disk_free_bytes = Some(available); - if available < result.minimum_disk_free_bytes { - result.reasons.push( - match profile { - Profile::Research => "Free disk is below the 100 GiB research floor.", - Profile::Development if owned_live => { - "Free disk is below the host reserve for the verified running development pool." - } - Profile::Development => { - "Free disk is below the development VM storage and host reserve budget." - } - } - .into(), - ); + if let Some(reason) = disk_shortfall(profile, owned, available) { + result.reasons.push(reason.into()); } let thermal = run( clean_command(Path::new("/usr/bin/pmset")).args(["-g", "therm"]), @@ -509,13 +560,119 @@ mod tests { const GIB: u64 = 1024 * 1024 * 1024; assert_eq!(disk_floor_bytes(Profile::Research), 100 * GIB); assert_eq!(disk_floor_bytes(Profile::Development), 58 * GIB); - assert_eq!(disk_budget(Profile::Development, true).0, 16 * GIB); - assert_eq!(disk_budget(Profile::Research, true).0, 100 * GIB); - assert_eq!(disk_budget(Profile::Development, false).0, 58 * GIB); + assert_eq!( + disk_budget(Profile::Development, OwnedPool::Running).0, + 16 * GIB + ); + assert_eq!( + disk_budget(Profile::Research, OwnedPool::Running).0, + 100 * GIB + ); + assert_eq!( + disk_budget(Profile::Development, OwnedPool::Fresh).0, + 58 * GIB + ); assert_eq!(load_ceiling(Profile::Research), Some(8.0)); assert_eq!(load_ceiling(Profile::Development), None); } + #[test] + fn stopped_owned_pool_charges_only_the_host_reserve_and_refuses_below_it() { + const GIB: u64 = 1024 * 1024 * 1024; + let stopped = OwnedPool::Stopped; + assert_eq!( + disk_budget(Profile::Development, stopped), + (16 * GIB, "verified-stopped-owned-pool-host-reserve") + ); + // The observed October 2 restart: 58,010,112,000 free bytes with retained disks. + assert_eq!( + disk_shortfall(Profile::Development, stopped, 58_010_112_000), + None + ); + assert_eq!( + disk_shortfall(Profile::Development, OwnedPool::Fresh, 58_010_112_000), + Some("Free disk is below the development VM storage and host reserve budget.") + ); + // The reserve boundary is inclusive and never waived for a stopped pool. + assert_eq!( + disk_shortfall(Profile::Development, stopped, 16 * GIB), + None + ); + assert_eq!( + disk_shortfall(Profile::Development, stopped, 16 * GIB - 1), + Some("Free disk is below the host reserve for the verified stopped development pool.") + ); + assert!(disk_shortfall(Profile::Development, stopped, 0).is_some()); + // Research keeps its frozen floor whatever ownership is claimed. + assert_eq!( + disk_budget(Profile::Research, stopped), + (100 * GIB, "research-floor") + ); + assert!(disk_shortfall(Profile::Research, stopped, 58_010_112_000).is_some()); + assert_eq!( + disk_budget(Profile::Development, OwnedPool::Fresh).1, + "new-pool-storage-overlay-host-reserve" + ); + assert_eq!( + disk_budget(Profile::Development, OwnedPool::Running).1, + "verified-live-owned-pool-host-reserve" + ); + } + + #[test] + fn only_an_admitted_development_sample_with_fresh_headroom_rebases_to_the_fresh_budget() { + const GIB: u64 = 1024 * 1024 * 1024; + let reserve = |free: u64| Admission { + profile: Profile::Development, + host_supported: true, + free_memory_bytes: Some(10 * GIB), + minimum_free_memory_bytes: 10 * GIB, + free_plus_file_cache_estimate_bytes: Some(10 * GIB), + memory_budget_basis: "free-plus-file-cache-estimate", + memory_pressure: MemoryPressure::from_level(Some(1)), + memory_pressure_normal: true, + disk_free_bytes: Some(free), + minimum_disk_free_bytes: 16 * GIB, + disk_budget_basis: "verified-stopped-owned-pool-host-reserve", + one_minute_load: Some(0.0), + load_ceiling: None, + thermal_normal: true, + swapouts: Some(7), + admitted: true, + reasons: Vec::new(), + retained_reserve_refusal: None, + }; + // The fresh boundary is inclusive, like the probe's own comparison. + let mut sample = reserve(58 * GIB); + assert!(fits_fresh_budget(&sample)); + rebase_fresh(&mut sample); + assert_eq!(sample.minimum_disk_free_bytes, 58 * GIB); + assert_eq!( + sample.disk_budget_basis, + "new-pool-storage-overlay-host-reserve" + ); + assert!(sample.admitted && sample.reasons.is_empty()); + assert_eq!(sample.swapouts, Some(7)); + // The October 2 restart's headroom needs the reserve; it never rebases. + assert!(!fits_fresh_budget(&reserve(58 * GIB - 1))); + assert!(!fits_fresh_budget(&reserve(58_010_112_000))); + let mut unobserved = reserve(100 * GIB); + unobserved.disk_free_bytes = None; + assert!(!fits_fresh_budget(&unobserved)); + // A refused sample stays refused whatever its headroom. + let mut refused = reserve(100 * GIB); + refused.admitted = false; + assert!(!fits_fresh_budget(&refused)); + let mut explained = reserve(100 * GIB); + explained + .reasons + .push("Thermal/performance status is not confirmed normal.".into()); + assert!(!fits_fresh_budget(&explained)); + let mut research = reserve(200 * GIB); + research.profile = Profile::Research; + assert!(!fits_fresh_budget(&research)); + } + #[test] fn operating_diagnostics_identify_each_failed_predicate_and_observations() { let floor = 2 * 1024 * 1024 * 1024; diff --git a/packages/runtime-core/src/provider/admission/pressure_tests.rs b/packages/runtime-core/src/provider/admission/pressure_tests.rs index 35878422a..467cdf2d7 100644 --- a/packages/runtime-core/src/provider/admission/pressure_tests.rs +++ b/packages/runtime-core/src/provider/admission/pressure_tests.rs @@ -23,6 +23,7 @@ fn observation(profile: Profile, level: Option, headroom: u64) -> Admission swapouts: Some(42), admitted: true, reasons: Vec::new(), + retained_reserve_refusal: None, } } diff --git a/packages/runtime-core/src/provider/lifecycle.rs b/packages/runtime-core/src/provider/lifecycle.rs index 654a460d7..9f8220318 100644 --- a/packages/runtime-core/src/provider/lifecycle.rs +++ b/packages/runtime-core/src/provider/lifecycle.rs @@ -113,7 +113,14 @@ fn invoke(candidate: &Candidate, owner: &Owner, args: &[&str]) -> Result Result<(), CandidateError> { + if value != "stopped" { + owner.clean_stop = None; + } owner.phase = value.into(); owner.save(candidate) } @@ -839,8 +846,8 @@ pub fn probe_with_profile( candidate: &Candidate, profile: super::Profile, ) -> Result { - let selected = admission_pool::select(candidate, profile)?; - admission_pool::probe(candidate, profile, selected.as_ref()) + let choice = admission_pool::select(candidate, profile)?; + admission_pool::probe(candidate, profile, &choice) } pub fn up_with_bridge( @@ -1125,9 +1132,10 @@ fn start_pool( )); } // Admission before aliases, provider commands, disks or VM effects. Only - // independently verified live ownership avoids charging a second VM's disks. - let admission_owner = admission_pool::select(candidate, profile)?; - let admission = admission_pool::probe(candidate, profile, admission_owner.as_ref())?; + // independently verified ownership of existing disks (a live pool, or an exactly + // clean stopped pool this start reuses) avoids charging a second VM's disks. + let mut admission_owner = admission_pool::select(candidate, profile)?; + let admission = admission_pool::probe(candidate, profile, &admission_owner)?; if !admission.admitted { return Err(CandidateError::new( "admission_rejected", @@ -1138,13 +1146,13 @@ fn start_pool( let baseline_swapouts = admission.swapouts; let mut samples = vec![admission]; samples.extend(admission::sample_after(profile, baseline_swapouts, || { - admission_pool::probe(candidate, profile, admission_owner.as_ref()) + admission_pool::probe(candidate, profile, &admission_owner) })?); samples } else { // Retain the frozen Research window and baseline semantics. admission::sample_with(profile, || { - admission_pool::probe(candidate, profile, admission_owner.as_ref()) + admission_pool::probe(candidate, profile, &admission_owner) })? }; artifact::verify(candidate)?; @@ -1155,12 +1163,13 @@ fn start_pool( STARTUP_LEASE_WAIT, profile, &samples, - || admission_pool::probe(candidate, profile, admission_owner.as_ref()), + || admission_pool::probe(candidate, profile, &admission_owner), )?; - if let Some(selected) = &admission_owner { - selected.reverify(candidate)?; - } samples.extend(fresh_samples); + // Under the lease: the selection is rechecked. A stopped pool keeps the reserve only if + // a sample needs it, and then its provider release (VM lock, disk handles, listeners, + // provider configuration) is proven; otherwise this start uses the fresh budget. + let settled = admission_pool::settle(candidate, &mut admission_owner, &lock, &mut samples)?; #[cfg(target_os = "macos")] if let Some(guard) = &retained_guard { guard.verify(candidate)?; @@ -1220,7 +1229,7 @@ fn start_pool( super::dependency_socket::verify(candidate, &owner)?; state::write(&root(candidate).join("admission.json"), &samples)?; if owner.phase == "running" { - if let Some(selected) = &admission_owner { + if let Some(selected) = settled.selected() { selected.reverify(candidate)?; } #[cfg(target_os = "macos")] @@ -1231,12 +1240,8 @@ fn start_pool( audit_boot(candidate, &owner)?; return status(candidate); } - if admission_owner.is_some() { - return Err(CandidateError::new( - "admission_owner_changed", - "Reserve-qualified ownership no longer names a running pool; no create or boot was admitted.", - )); - } + // Only a clean stopped selection continues, as a restart of its verified retained disks. + let restart = admission_pool::restart_only(candidate, settled, &owner, &lock)?; if ![ "initializing", "stopped", @@ -1262,6 +1267,8 @@ fn start_pool( } prepare_rootfs(candidate, &mut owner)?; if !owner.created { + // A reserve-qualified start never reaches `machine create`. + let fresh = admission_pool::fresh_create(&restart)?; // SmolVM clones the expanded templates in `home/.smolvm` into the new disks at the // machine's first start. Choose them (stock, or an activated prepared base) and verify // them before the phase records a create attempt; `verify_templates` checks again @@ -1315,31 +1322,26 @@ fn start_pool( ]); } } - invoke( - candidate, - &owner, - &arguments.iter().map(String::as_str).collect::>(), - )?; + create_machine(candidate, &owner, &arguments, fresh)?; super::config_audit::pin_created_network(candidate, &mut owner)?; owner.created = true; owner.save(candidate)?; } - // A fresh admission immediately before boot closes installation/lock delays. - let admission = admission::probe_for(&candidate.checkout, profile)?; - if !admission.admitted { - phase(candidate, &mut owner, "stopped")?; - return Err(CandidateError::new( - "admission_rejected", - admission.reasons.join(" "), - )); - } - if admission.swapouts != samples.last().and_then(|sample| sample.swapouts) { - phase(candidate, &mut owner, "stopped")?; - return Err(CandidateError::new( - "admission_rejected", - "Swapouts changed after qualification; refusing boot.", - )); - } + // A fresh admission immediately before boot, on the qualification's basis, closes + // installation/lock delays. `machine start` needs the permit it leads to. + let boot = match admission_pool::admit_boot( + candidate, + profile, + &restart, + samples.last().and_then(|sample| sample.swapouts), + )? { + admission_pool::BootGate::Admitted(boot) => boot, + admission_pool::BootGate::Refused(error) => { + // Nothing booted: a clean stop stays clean, an unclean one stays unclean. + phase(candidate, &mut owner, "stopped")?; + return Err(error); + } + }; if let Err(error) = super::config_audit::verify(candidate, &owner) { phase(candidate, &mut owner, "stopped-before-engine")?; return Err(error); @@ -1357,6 +1359,8 @@ fn start_pool( if let Some(guard) = &retained_guard { guard.verify(candidate)?; } + // The last check before `machine start` releases the VM lock to the provider. + let permit = boot.release(candidate, &lock)?; let previous_boot = owner.begin_boot(); owner.reclamation = Some(boot_reclamation_policy()); // The previous provider was proven dead above. Until this start records its own provider, @@ -1364,11 +1368,7 @@ fn start_pool( // old identity. owner.process = None; phase(candidate, &mut owner, "booting")?; - invoke( - candidate, - &owner, - &["machine", "start", "--name", &owner.machine], - )?; + start_machine(candidate, &owner, permit)?; let observed = recorded_process(candidate, &owner)?; identity::verify( &observed, @@ -1399,6 +1399,35 @@ fn start_pool( result } +/// `machine create` allocates new disks; only a start without a reserve selection may. +fn create_machine( + candidate: &Candidate, + owner: &Owner, + arguments: &[String], + _fresh: admission_pool::FreshCreate, +) -> Result<(), CandidateError> { + invoke( + candidate, + owner, + &arguments.iter().map(String::as_str).collect::>(), + ) + .map(drop) +} + +/// `machine start` hands the VM lock to the provider; only a released boot admission may. +fn start_machine( + candidate: &Candidate, + owner: &Owner, + _permit: admission_pool::StartPermit, +) -> Result<(), CandidateError> { + invoke( + candidate, + owner, + &["machine", "start", "--name", &owner.machine], + ) + .map(drop) +} + fn stop_failed_boot( process: &identity::ProcessIdentity, binary: &Path, @@ -1897,20 +1926,25 @@ sync"#, "Provider remains alive after TERM; no force-kill or false stopped receipt.", )); } - finish_stopped( - candidate, - &mut owner, - if before_engine { - "stopped-before-engine" - } else if failed_engine { - "stopped-after-engine-failure" - } else { - "stopped" - }, - )?; + if before_engine { + finish_stopped(candidate, &mut owner, "stopped-before-engine")?; + } else if failed_engine { + finish_stopped(candidate, &mut owner, "stopped-after-engine-failure")?; + } else { + finish_quiesced_stop(candidate, &mut owner)?; + } status(candidate) } +/// `down`'s completion after the guest acknowledged flush and shutdown of a quiesced engine +/// and the provider exited: the clean-stop marker names this boot and is written with the +/// `stopped` phase once the provider's release is proven. If any proof fails, nothing is +/// written and the receipt keeps its in-progress phase. +fn finish_quiesced_stop(candidate: &Candidate, owner: &mut Owner) -> Result<(), CandidateError> { + owner.clean_stop = state::CleanStop::of(owner); + finish_stopped(candidate, owner, "stopped") +} + fn finish_stopped( candidate: &Candidate, owner: &mut Owner, @@ -1963,8 +1997,21 @@ impl Drop for VmLock { } } +/// How long `down`, `recover` and other stop paths wait for `lsof` over the owned disks. +const STOP_HANDLE_WAIT: Duration = Duration::from_secs(5); + /// Keep this guard through alias restoration and the stopped receipt. fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result { + lock_absent_disks_within(candidate, owner, STOP_HANDLE_WAIT) +} + +/// [`lock_absent_disks`] with a caller-chosen bound on the `lsof` disk-handle scan. A scan +/// that does not finish within `handle_wait` fails as `provider_timeout_uncertain`. +fn lock_absent_disks_within( + candidate: &Candidate, + owner: &Owner, + handle_wait: Duration, +) -> Result { let directory = owner.real_data_dir(candidate)?; let vm_lock = OpenOptions::new() .read(true) @@ -1991,7 +2038,7 @@ fn lock_absent_disks(candidate: &Candidate, owner: &Owner) -> Result Result<(), CandidateError> { let mut sockets = vec!["agent.sock".to_owned(), "docker.sock".to_owned()]; if let Some(bridge) = owner.application_bridge { sockets.extend(bridge.guest_paths().into_iter().map(|path| { @@ -2050,13 +2109,7 @@ fn finish_absent_locked( } } } - for name in ["storage.raw", "overlay.raw"] { - File::open(owner.real_data_dir(candidate)?.join(name)) - .map_err(io)? - .sync_all() - .map_err(io)?; - } - phase(candidate, owner, value) + Ok(()) } pub fn recover(candidate: &Candidate) -> Result { diff --git a/packages/runtime-core/src/provider/lifecycle/admission_pool.rs b/packages/runtime-core/src/provider/lifecycle/admission_pool.rs index e25dcfb5c..dc7c26b83 100644 --- a/packages/runtime-core/src/provider/lifecycle/admission_pool.rs +++ b/packages/runtime-core/src/provider/lifecycle/admission_pool.rs @@ -1,25 +1,158 @@ -//! A host-reserve budget is valid only for one independently verified live pool. -use super::{Owner, admission, io, root, verify_disk_allocation, verify_live}; +//! A host-reserve disk budget is valid only for one independently verified pool whose +//! declared disks already exist: a live running pool, or an exactly clean stopped +//! Development pool that a restart reuses without `machine create`. +//! +//! Every stopped-pool refusal at selection keeps the unchanged fresh-allocation budget and +//! reports its closed code, so an ineligible owner is never admitted on the reserve. A +//! stopped selection is proven in two layers: +//! +//! * Receipt checks ([`stopped`]) are read-only and run around every admission sample, +//! including the read-only `runtime probe`: the `down`-completed clean-stop marker of the +//! last boot, its quiescence receipt, creation, rootfs pin, disk identities and sizes, +//! the recorded executable and user, and a dead recorded provider. +//! * Release checks ([`released`]) take the provider's VM lock, run `lsof` on the owned +//! disks, probe the owned sockets and audit the provider configuration. They need the +//! startup lease, so they never contend with another lifecycle operation that may be +//! handing the VM lock to a provider; they run once the lease is held, before any restart +//! effect, and immediately before `machine start`. +//! +//! The reserve is used only when a start needs it. Once the lease is held, [`settle`] drops +//! a stopped selection whose every qualifying sample also fits the unchanged fresh budget: +//! that start continues exactly as it would without a selection, on the fresh budget and +//! without release checks. Only a start whose free disk lies between the host reserve and +//! the fresh budget keeps the selection, and it is refused if a release check fails. +//! +//! The start sequence is typed, so a start cannot skip a step: [`restart_only`] needs the +//! [`Settled`] basis that only [`settle`] returns, and both `machine create` and the +//! pre-boot admission need the [`Restart`] that only [`restart_only`] returns. `machine +//! create` also needs a [`FreshCreate`], which no selection can obtain, and `machine start` +//! needs a [`StartPermit`], which only the pre-boot admission ([`admit_boot`]) followed by +//! the final leased recheck ([`BootAdmission::release`]) issues. +use super::{ + Owner, admission, binary, io, lock_absent_disks_within, root, sockets_absent, + verify_disk_allocation, verify_disks, verify_live, +}; +use crate::provider::{Profile, config_audit, identity, state}; use crate::{Candidate, CandidateError}; +use serde_json::Value; use std::fs; +use std::time::Duration; -pub(super) struct Selection(Owner); +/// How long a release check waits for `lsof` over the owned disks. `lsof` scans every +/// process, so it is slow on a loaded host (5.4–6.5 s at about 1,850 processes and load +/// 30–80). The wait is longer than `down`'s because only a start that needs the reserve runs +/// it, under the startup lease; a scan that still does not finish refuses the reserve. +const RELEASE_HANDLE_WAIT: Duration = Duration::from_secs(30); -pub(super) fn select( - candidate: &Candidate, - profile: crate::provider::Profile, -) -> Result, CandidateError> { - if profile != crate::provider::Profile::Development { - return Ok(None); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Basis { + Running, + Stopped, +} + +pub(super) struct Selection { + owner: Owner, + basis: Basis, +} + +/// The admission basis chosen before startup: a verified selection, or the fresh budget +/// with the closed reason a retained stopped pool did not qualify for the reserve. +pub(super) struct Choice { + selection: Option, + refusal: Option, +} + +impl Choice { + pub(super) fn selected(&self) -> Option<&Selection> { + self.selection.as_ref() + } + + #[cfg(all(test, target_os = "macos"))] + pub(super) fn into_selection(self) -> Option { + self.selection + } + + #[cfg(all(test, target_os = "macos"))] + pub(super) fn refusal(&self) -> Option { + self.refusal + } +} + +/// Closed reasons a stopped Development owner cannot use the host-reserve budget. +/// Codes are fixed labels without paths, identities or provider output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum StoppedRefusal { + /// The receipt names another machine, executable, user, provider configuration or + /// unsafe VM lock. + Foreign, + /// A provider process, VM lock holder, disk handle or listener may remain. + Active, + /// The last stop was not a completed quiesced `down` of the last boot, or an owner + /// update is pending. + Dirty, + /// Creation, disk identities, sizes, rootfs or boot identity are missing or drifted. + Stale, +} + +impl StoppedRefusal { + pub(super) fn code(self) -> &'static str { + match self { + Self::Foreign => "retained_reserve_foreign_owner", + Self::Active => "retained_reserve_active_owner", + Self::Dirty => "retained_reserve_dirty_owner", + Self::Stale => "retained_reserve_stale_owner", + } + } + + fn error(self) -> CandidateError { + CandidateError::new( + self.code(), + "Stopped development ownership is not exactly clean; the retained-disk host reserve was not admitted.", + ) + } + + /// Classify a lifecycle verification failure into a closed refusal. Missing, replaced + /// or mis-sized state is stale; any other failure, including an absence probe that + /// could not complete (for example a slow `lsof`), leaves the provider possibly active. + fn from_check(error: &CandidateError) -> Self { + match error.code { + "foreign_state" | "unaudited_provider_config" => Self::Foreign, + "provider_state" | "disk_identity_mismatch" | "disk_allocation_mismatch" => Self::Stale, + _ => Self::Active, + } + } +} + +pub(super) fn select(candidate: &Candidate, profile: Profile) -> Result { + let fresh = |refusal| Choice { + selection: None, + refusal, + }; + if profile != Profile::Development { + return Ok(fresh(None)); } match fs::symlink_metadata(root(candidate).join("owner.json")) { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(fresh(None)), Err(e) => return Err(io(e)), Ok(_) => {} } let owner = Owner::load(candidate)?; - if owner.profile != profile || owner.phase != "running" { - return Ok(None); + if owner.profile != profile { + return Ok(fresh(None)); + } + if owner.phase != "running" { + return Ok(match stopped(candidate, &owner) { + Ok(()) => Choice { + selection: Some(Selection { + owner, + basis: Basis::Stopped, + }), + refusal: None, + }, + // A never-created owner has no retained disks, so the fresh budget needs no + // explanation; a created one reports why its disks were charged again. + Err(refusal) => fresh(owner.created.then_some(refusal)), + }); } if !owner.created || owner.storage.is_none() || owner.overlay.is_none() { return Err(changed()); @@ -30,46 +163,354 @@ pub(super) fn select( owner.overlay.as_ref().unwrap(), profile, )?; - Ok(Some(Selection(owner))) + Ok(Choice { + selection: Some(Selection { + owner, + basis: Basis::Running, + }), + refusal: None, + }) +} + +/// Receipt checks: whether `owner` records an exactly clean stopped Development pool with +/// its retained disks. +/// +/// Clean means `down` completed a quiesced stop of the last boot and recorded its +/// clean-stop marker with the `stopped` phase. Any other phase write and every new boot +/// clear that marker, so an interrupted, uncertain, unclean, before-engine or failed-boot +/// stop never qualifies, even when a later refused start rewrote the phase to `stopped`. +/// Read-only and lock-free: it never repairs or adopts state, takes no provider lock and +/// runs no host tool. +pub(super) fn stopped(candidate: &Candidate, owner: &Owner) -> Result<(), StoppedRefusal> { + if owner.profile != Profile::Development { + return Err(StoppedRefusal::Foreign); + } + match fs::symlink_metadata(root(candidate).join("owner.pending")) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + _ => return Err(StoppedRefusal::Dirty), + } + if owner.phase != "stopped" { + return Err(StoppedRefusal::Dirty); + } + let (Some(storage), Some(overlay), Some(process), Some(boot), Some(daemon), Some(start)) = ( + owner.storage.as_ref(), + owner.overlay.as_ref(), + owner.process.as_ref(), + owner.guest_boot_id.as_deref(), + owner.daemon_pid, + owner.daemon_start, + ) else { + return Err(StoppedRefusal::Stale); + }; + if !owner.created || owner.rootfs_digest.is_none() { + return Err(StoppedRefusal::Stale); + } + // Only `down` records this marker, after the guest acknowledged flush and shutdown and + // provider absence was proven, in the same receipt write as its `stopped` phase. + if !owner + .clean_stop + .as_ref() + .is_some_and(|clean| clean.names(boot, daemon, start)) + { + return Err(StoppedRefusal::Dirty); + } + // Only `down` writes this receipt, after the guest engine quiesced in this boot. + let quiesced = state::read::(&root(candidate).join("quiescence.json")) + .ok() + .is_some_and(|receipt| { + receipt["guest_boot_id"].as_str() == Some(boot) + && receipt["daemon_pid"].as_u64() == Some(u64::from(daemon)) + && receipt["daemon_start"].as_u64() == Some(start) + && receipt["result"] + .as_str() + .is_some_and(|result| result.starts_with("quiesced")) + }); + if !quiesced { + return Err(StoppedRefusal::Dirty); + } + if process.executable != binary(candidate) || process.uid != unsafe { libc::geteuid() } { + return Err(StoppedRefusal::Foreign); + } + // A live or unobservable recorded PID (including reuse) is never proven absent. + if identity::alive(process.pid).unwrap_or(true) { + return Err(StoppedRefusal::Active); + } + verify_disks(candidate, owner).map_err(|error| StoppedRefusal::from_check(&error))?; + verify_disk_allocation(storage, overlay, owner.profile) + .map_err(|error| StoppedRefusal::from_check(&error)) +} + +/// Release checks, under the startup lease: the provider the receipt names released its +/// machine (free VM lock, no open disk handle, no owned listener) and the provider's +/// persisted configuration is still this pool's audited one. The VM lock is released on +/// return; the provider's own `machine start` reacquires it. +fn released( + candidate: &Candidate, + owner: &Owner, + lease: &state::Lock, +) -> Result<(), StoppedRefusal> { + let check = |error: CandidateError| StoppedRefusal::from_check(&error); + lease.verify_path(&root(candidate)).map_err(check)?; + lock_absent_disks_within(candidate, owner, RELEASE_HANDLE_WAIT).map_err(check)?; + sockets_absent(owner).map_err(check)?; + config_audit::verify(candidate, owner).map_err(check) } fn changed() -> CandidateError { CandidateError::new( "admission_owner_changed", - "Verified live admission ownership changed; no create or boot was admitted.", + "Verified admission ownership changed; no create or boot was admitted.", ) } impl Selection { + pub(super) fn is_stopped(&self) -> bool { + self.basis == Basis::Stopped + } + + /// The selected receipt is unchanged and its basis is still independently proven. + /// For a stopped selection these are the lock-free receipt checks. pub(super) fn reverify(&self, candidate: &Candidate) -> Result<(), CandidateError> { let current = Owner::load(candidate)?; - if current != self.0 || current.phase != "running" { + if current != self.owner { return Err(changed()); } - verify_live(candidate, ¤t)?; - verify_disk_allocation( - current.storage.as_ref().ok_or_else(changed)?, - current.overlay.as_ref().ok_or_else(changed)?, - current.profile, - ) + match self.basis { + Basis::Stopped => stopped(candidate, ¤t).map_err(StoppedRefusal::error), + Basis::Running => { + if current.phase != "running" { + return Err(changed()); + } + verify_live(candidate, ¤t)?; + verify_disk_allocation( + current.storage.as_ref().ok_or_else(changed)?, + current.overlay.as_ref().ok_or_else(changed)?, + current.profile, + ) + } + } + } + + /// [`Self::reverify`] plus, for a stopped selection, the release checks that need the + /// startup lease. + pub(super) fn reverify_leased( + &self, + candidate: &Candidate, + lease: &state::Lock, + ) -> Result<(), CandidateError> { + self.reverify(candidate)?; + if self.basis == Basis::Stopped { + released(candidate, &self.owner, lease).map_err(StoppedRefusal::error)?; + // The receipt cannot have changed under the lease, but bind it after the + // release probes as well. + self.reverify(candidate)?; + } + Ok(()) + } +} + +/// Which disk allocation an admission sample treats as already paid. +fn owned_pool(choice: &Choice) -> admission::OwnedPool { + match choice.selected().map(|selected| selected.basis) { + None => admission::OwnedPool::Fresh, + Some(Basis::Running) => admission::OwnedPool::Running, + Some(Basis::Stopped) => admission::OwnedPool::Stopped, + } +} + +/// The host observation behind every lifecycle admission sample. +fn observe_host( + candidate: &Candidate, + profile: Profile, + owned: admission::OwnedPool, +) -> Result { + match owned { + admission::OwnedPool::Fresh => admission::probe_for(&candidate.checkout, profile), + owned => admission::probe_owned_development(&candidate.checkout, owned), } } +/// One admission sample on `choice`'s basis. A selection is rechecked around the sample; +/// a fresh sample reports why a retained stopped pool did not qualify for the reserve. pub(super) fn probe( candidate: &Candidate, - profile: crate::provider::Profile, - selected: Option<&Selection>, + profile: Profile, + choice: &Choice, +) -> Result { + probe_with(candidate, choice, |owned| { + observe_host(candidate, profile, owned) + }) +} + +fn probe_with( + candidate: &Candidate, + choice: &Choice, + observe: impl FnOnce(admission::OwnedPool) -> Result, ) -> Result { - match selected { - None => admission::probe_for(&candidate.checkout, profile), + let owned = owned_pool(choice); + match choice.selected() { + None => { + let mut observed = observe(owned)?; + observed.retained_reserve_refusal = choice.refusal.map(StoppedRefusal::code); + Ok(observed) + } Some(selected) => { selected.reverify(candidate)?; - let observed = admission::probe_owned_development(&candidate.checkout)?; + let observed = observe(owned)?; selected.reverify(candidate)?; Ok(observed) } } } +/// Once the startup lease is held and before any restart effect, settle the basis this start +/// uses. `samples` are the qualifying samples, including the lease recheck, in the order they +/// will be recorded. +/// +/// A live selection is rechecked. A stopped selection's receipt is rechecked; then, if every +/// sample also fits the unchanged fresh budget, the selection is dropped and the samples are +/// recorded on that budget, so the start runs exactly as one without a selection and no +/// release check runs. Otherwise the reserve is needed, and the release checks must pass +/// under the lease. +pub(super) fn settle<'a>( + candidate: &Candidate, + choice: &'a mut Choice, + lease: &state::Lock, + samples: &mut [admission::Admission], +) -> Result, CandidateError> { + if let Some(selected) = choice.selected() { + selected.reverify(candidate)?; + if selected.is_stopped() { + if !samples.is_empty() && samples.iter().all(admission::fits_fresh_budget) { + samples.iter_mut().for_each(admission::rebase_fresh); + choice.selection = None; + } else { + selected.reverify_leased(candidate, lease)?; + } + } + } + Ok(Settled(choice)) +} + +/// The admission basis this start settled under the startup lease. Only [`settle`] issues +/// it, and [`restart_only`] needs it. +#[derive(Clone, Copy)] +pub(super) struct Settled<'a>(&'a Choice); + +impl Settled<'_> { + pub(super) fn selected(&self) -> Option<&Selection> { + self.0.selected() + } +} + +/// Authority for create or restart effects and for the pre-boot admission. Only +/// [`restart_only`] issues it. +pub(super) struct Restart<'a>(&'a Choice); + +/// Before a restart effect under the startup lease: a selection continues only as a +/// restart of its clean stopped pool's verified retained disks. +pub(super) fn restart_only<'a>( + candidate: &Candidate, + settled: Settled<'a>, + owner: &Owner, + lease: &state::Lock, +) -> Result, CandidateError> { + let choice = settled.0; + let Some(selected) = choice.selected() else { + return Ok(Restart(choice)); + }; + if !selected.is_stopped() || !owner.created || *owner != selected.owner { + return Err(CandidateError::new( + "admission_owner_changed", + "Reserve-qualified ownership no longer names its selected pool; no create or boot was admitted.", + )); + } + selected.reverify_leased(candidate, lease)?; + Ok(Restart(choice)) +} + +/// Authority to run `machine create`: only a start without a reserve selection allocates +/// new disks. +pub(super) struct FreshCreate(()); + +pub(super) fn fresh_create(restart: &Restart<'_>) -> Result { + if restart.0.selected().is_some() { + return Err(CandidateError::new( + "admission_owner_changed", + "A reserve-qualified start never creates a machine; no create or boot was admitted.", + )); + } + Ok(FreshCreate(())) +} + +/// The pre-boot admission passed on this start's basis. +pub(super) struct BootAdmission<'a> { + selection: Option<&'a Selection>, +} + +/// Authority to run `machine start`, issued only by [`BootAdmission::release`]. +pub(super) struct StartPermit(()); + +/// The pre-boot admission's outcome. A refusal leaves the provider untouched; the caller +/// records the stop before returning it. +pub(super) enum BootGate<'a> { + Admitted(BootAdmission<'a>), + Refused(CandidateError), +} + +/// A fresh admission immediately before boot, on the same basis as qualification, closes +/// installation and lock delays: unchanged fresh budget without a selection; a selection's +/// reserve basis rechecked around the sample. Swapouts must not have grown since the +/// qualifying window's last sample. +pub(super) fn admit_boot<'a>( + candidate: &Candidate, + profile: Profile, + restart: &Restart<'a>, + baseline_swapouts: Option, +) -> Result, CandidateError> { + admit_boot_with(candidate, restart.0, baseline_swapouts, |owned| { + observe_host(candidate, profile, owned) + }) +} + +fn admit_boot_with<'a>( + candidate: &Candidate, + choice: &'a Choice, + baseline_swapouts: Option, + observe: impl FnOnce(admission::OwnedPool) -> Result, +) -> Result, CandidateError> { + let admission = probe_with(candidate, choice, observe)?; + if !admission.admitted { + return Ok(BootGate::Refused(CandidateError::new( + "admission_rejected", + admission.reasons.join(" "), + ))); + } + if admission.swapouts != baseline_swapouts { + return Ok(BootGate::Refused(CandidateError::new( + "admission_rejected", + "Swapouts changed after qualification; refusing boot.", + ))); + } + Ok(BootGate::Admitted(BootAdmission { + selection: choice.selected(), + })) +} + +impl BootAdmission<'_> { + /// The last check before `machine start` hands the VM lock to the provider: the + /// selection, if any, is rechecked under the startup lease after every pre-boot effect. + pub(super) fn release( + self, + candidate: &Candidate, + lease: &state::Lock, + ) -> Result { + if let Some(selected) = self.selection { + selected.reverify_leased(candidate, lease)?; + } + Ok(StartPermit(())) + } +} + #[cfg(all(test, target_os = "macos"))] mod tests; diff --git a/packages/runtime-core/src/provider/lifecycle/admission_pool/tests.rs b/packages/runtime-core/src/provider/lifecycle/admission_pool/tests.rs index 54b0b9e8c..52b0b59f1 100644 --- a/packages/runtime-core/src/provider/lifecycle/admission_pool/tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/admission_pool/tests.rs @@ -146,11 +146,13 @@ fn only_a_live_owned_development_pool_selects_the_host_reserve() { let selection = select(&pool.candidate, Profile::Development) .unwrap() + .into_selection() .expect("exact live development ownership"); selection.reverify(&pool.candidate).unwrap(); assert!( select(&pool.candidate, Profile::Research) .unwrap() + .into_selection() .is_none() ); assert_eq!( @@ -176,7 +178,12 @@ fn absent_or_non_running_owners_do_not_select_the_host_reserve() { child: None, }; let candidate = Candidate::discover(&directory).unwrap(); - assert!(select(&candidate, Profile::Development).unwrap().is_none()); + assert!( + select(&candidate, Profile::Development) + .unwrap() + .into_selection() + .is_none() + ); drop(cleanup); for phase in ["stopped", "stopped-before-engine"] { @@ -186,6 +193,7 @@ fn absent_or_non_running_owners_do_not_select_the_host_reserve() { assert!( select(&pool.candidate, Profile::Development) .unwrap() + .into_selection() .is_none() ); } @@ -195,6 +203,7 @@ fn absent_or_non_running_owners_do_not_select_the_host_reserve() { assert!( select(&pool.candidate, Profile::Development) .unwrap() + .into_selection() .is_none() ); } @@ -257,6 +266,7 @@ fn selected_owner_or_provider_changes_refuse_reverification() { let mut pool = Pool::new(); let selection = select(&pool.candidate, Profile::Development) .unwrap() + .into_selection() .unwrap(); pool.owner.guest_boot_id = Some("22222222-2222-2222-2222-222222222222".into()); pool.save(); @@ -268,6 +278,7 @@ fn selected_owner_or_provider_changes_refuse_reverification() { let pool = Pool::new(); let selection = select(&pool.candidate, Profile::Development) .unwrap() + .into_selection() .unwrap(); fs::write(pool.data().join("agent.pid"), b"999999\n1\n").unwrap(); assert!(selection.reverify(&pool.candidate).is_err()); @@ -275,9 +286,938 @@ fn selected_owner_or_provider_changes_refuse_reverification() { let pool = Pool::new(); let selection = select(&pool.candidate, Profile::Development) .unwrap() + .into_selection() .unwrap(); let path = pool.data().join("overlay.raw"); fs::rename(&path, pool.data().join("overlay.retained")).unwrap(); sparse_disk(&path, 10, 2); assert!(selection.reverify(&pool.candidate).is_err()); } + +/// Tests that run the release probes (`lsof` over the owned disks) run one at a time: +/// concurrent whole-host `lsof` scans on a loaded machine can exceed the probe's bounded +/// wait, which correctly refuses as possibly active but is not what these tests examine. +fn serial() -> std::sync::MutexGuard<'static, ()> { + static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(()); + SERIAL + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +/// A pool `down` left exactly clean: provider proven dead after a quiesced engine stop of +/// its last boot, clean-stop marker recorded, retained disks identified, provider +/// configuration audited, VM lock free and no listener or disk handle. +fn stopped_pool() -> Pool { + let mut pool = Pool::new(); + let child = pool.cleanup.child.as_mut().unwrap(); + child.kill().unwrap(); + child.wait().unwrap(); + pool.cleanup.child = None; + pool.owner.phase = "stopped".into(); + pool.owner.daemon_pid = Some(4242); + pool.owner.daemon_start = Some(99); + pool.owner.rootfs_digest = Some("fixture-rootfs".into()); + pool.owner.clean_stop = state::CleanStop::of(&pool.owner); + pool.save(); + quiescence(&pool, BOOT, 4242, 99, "quiesced"); + OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(pool.data().join("vm.lock")) + .unwrap(); + provider_config(&pool, &pool.owner.token); + pool +} + +const BOOT: &str = "11111111-1111-1111-1111-111111111111"; + +fn quiescence(pool: &Pool, boot: &str, daemon: u32, start: u64, result: &str) { + state::write( + &root(&pool.candidate).join("quiescence.json"), + &serde_json::json!({ + "guest_boot_id": boot, "daemon_pid": daemon, "daemon_start": start, "result": result, + }), + ) + .unwrap(); +} + +fn provider_database(pool: &Pool) -> PathBuf { + pool.candidate + .state_root + .join("run/smolvm") + .join("home") + .join("Library/Application Support/smolvm/server/smolvm.db") +} + +/// The provider's persisted machine record, as `machine create` left it for this pool, +/// labelled with `token`. +fn provider_config(pool: &Pool, token: &str) { + let database = provider_database(pool); + fs::create_dir_all(database.parent().unwrap()).unwrap(); + let _ = fs::remove_file(&database); + let record = serde_json::json!({ + "name": pool.owner.machine, "labels": {"hack-local.owner": token}, + "ssh_agent": false, "network": false, "ports": [], "published_sockets": [], + "staged_mounts": [], "remote_volumes": [], "init": [], "env": [], "entrypoint": [], + "cmd": [], + "mounts": [[artifact::engine_root(&pool.candidate), "/opt/hack-engine", true]], + }); + let connection = rusqlite::Connection::open(&database).unwrap(); + connection + .execute_batch("CREATE TABLE vms(name TEXT PRIMARY KEY, data BLOB NOT NULL)") + .unwrap(); + connection + .execute( + "INSERT INTO vms VALUES (?1, ?2)", + rusqlite::params![pool.owner.machine, serde_json::to_vec(&record).unwrap()], + ) + .unwrap(); +} + +fn lease(pool: &Pool) -> state::Lock { + state::Lock::acquire(&root(&pool.candidate)).unwrap() +} + +/// The closed receipt-level refusal for the current receipt. Selection keeps the fresh +/// budget and reports the same closed code for a created pool. +fn refusal(pool: &Pool) -> StoppedRefusal { + let owner = Owner::load(&pool.candidate).unwrap(); + let refusal = stopped(&pool.candidate, &owner).unwrap_err(); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + assert!(choice.selected().is_none()); + if owner.created { + assert_eq!(choice.refusal(), Some(refusal)); + } + refusal +} + +/// The closed code of a refusal only the leased release checks establish: the receipt +/// checks select the pool, and the lease-held recheck refuses it. +fn leased_refusal(pool: &Pool) -> &'static str { + let selection = select(&pool.candidate, Profile::Development) + .unwrap() + .into_selection() + .expect("receipt checks pass"); + selection.reverify(&pool.candidate).unwrap(); + let lease = lease(pool); + selection + .reverify_leased(&pool.candidate, &lease) + .unwrap_err() + .code +} + +fn exclusive_lock(path: &Path) -> fs::File { + use std::os::fd::AsRawFd; + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + // SAFETY: the descriptor stays open for this call; the lock ends when `file` drops. + assert_eq!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }, + 0 + ); + file +} + +#[test] +fn only_a_clean_stopped_owned_development_pool_selects_the_retained_reserve() { + let _serial = serial(); + let pool = stopped_pool(); + let receipt = fs::read(root(&pool.candidate).join("owner.json")).unwrap(); + let storage = identity::disk(&pool.data().join("storage.raw")).unwrap(); + let overlay = identity::disk(&pool.data().join("overlay.raw")).unwrap(); + + let choice = select(&pool.candidate, Profile::Development).unwrap(); + assert_eq!(choice.refusal(), None); + let selection = choice + .into_selection() + .expect("exact clean stopped development ownership"); + assert!(selection.is_stopped()); + selection.reverify(&pool.candidate).unwrap(); + let lease = lease(&pool); + selection.reverify_leased(&pool.candidate, &lease).unwrap(); + selection.reverify_leased(&pool.candidate, &lease).unwrap(); + drop(lease); + assert!( + select(&pool.candidate, Profile::Research) + .unwrap() + .into_selection() + .is_none() + ); + // Selection is read-only: receipt, disks and the provider's VM lock are left as found. + assert_eq!( + fs::read(root(&pool.candidate).join("owner.json")).unwrap(), + receipt + ); + assert_eq!( + identity::disk(&pool.data().join("storage.raw")).unwrap(), + storage + ); + assert_eq!( + identity::disk(&pool.data().join("overlay.raw")).unwrap(), + overlay + ); + drop(exclusive_lock(&pool.data().join("vm.lock"))); + + // A running selection keeps its separate live basis. + let running = Pool::new(); + assert!( + !select(&running.candidate, Profile::Development) + .unwrap() + .into_selection() + .unwrap() + .is_stopped() + ); +} + +#[test] +fn receipt_checks_take_no_provider_lock() { + let _serial = serial(); + // The read-only `runtime probe` and every unleased sample may run while another + // lifecycle operation hands the VM lock to a provider: they neither take nor need it. + let pool = stopped_pool(); + let held = exclusive_lock(&pool.data().join("vm.lock")); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + choice + .selected() + .expect("receipt checks never take the VM lock") + .reverify(&pool.candidate) + .unwrap(); + let observed = probe_with(&pool.candidate, &choice, |owned| { + Ok(sample(owned, true, Some(7))) + }) + .unwrap(); + assert_eq!( + observed.disk_budget_basis, + "verified-stopped-owned-pool-host-reserve" + ); + drop(held); +} + +#[test] +fn foreign_stopped_owner_refuses_the_retained_reserve() { + let _serial = serial(); + let pool = stopped_pool(); + fs::write(pool.data().join("name"), b"hack-foreign0000").unwrap(); + assert_eq!(refusal(&pool), StoppedRefusal::Foreign); + + let mut pool = stopped_pool(); + pool.owner.process.as_mut().unwrap().executable = PathBuf::from("/usr/bin/true"); + pool.save(); + assert_eq!(refusal(&pool), StoppedRefusal::Foreign); + + let mut pool = stopped_pool(); + pool.owner.process.as_mut().unwrap().uid += 1; + pool.save(); + assert_eq!(refusal(&pool), StoppedRefusal::Foreign); + + let pool = stopped_pool(); + fs::hard_link( + pool.data().join("vm.lock"), + pool.data().join("vm.lock.alias"), + ) + .unwrap(); + assert_eq!(leased_refusal(&pool), "retained_reserve_foreign_owner"); + + // The provider's persisted configuration names another owner, or is missing. + let pool = stopped_pool(); + provider_config(&pool, "00000000000000000000000000000000"); + assert_eq!(leased_refusal(&pool), "retained_reserve_foreign_owner"); + let pool = stopped_pool(); + fs::remove_file(provider_database(&pool)).unwrap(); + assert_eq!(leased_refusal(&pool), "retained_reserve_foreign_owner"); + assert_eq!( + StoppedRefusal::Foreign.code(), + "retained_reserve_foreign_owner" + ); +} + +#[test] +fn active_stopped_owner_refuses_the_retained_reserve() { + let _serial = serial(); + // A receipt that says stopped while its recorded provider is still alive. + let mut pool = Pool::new(); + pool.owner.phase = "stopped".into(); + pool.owner.daemon_pid = Some(4242); + pool.owner.daemon_start = Some(99); + pool.owner.rootfs_digest = Some("fixture-rootfs".into()); + pool.owner.clean_stop = state::CleanStop::of(&pool.owner); + pool.save(); + quiescence(&pool, BOOT, 4242, 99, "quiesced"); + assert_eq!(refusal(&pool), StoppedRefusal::Active); + + let pool = stopped_pool(); + let held = exclusive_lock(&pool.data().join("vm.lock")); + assert_eq!(leased_refusal(&pool), "retained_reserve_active_owner"); + drop(held); + + let pool = stopped_pool(); + let handle = fs::File::open(pool.data().join("storage.raw")).unwrap(); + assert_eq!(leased_refusal(&pool), "retained_reserve_active_owner"); + drop(handle); + + let pool = stopped_pool(); + let listener = + std::os::unix::net::UnixListener::bind(pool.owner.data_dir().join("agent.sock")).unwrap(); + assert_eq!(leased_refusal(&pool), "retained_reserve_active_owner"); + drop(listener); + assert_eq!( + StoppedRefusal::Active.code(), + "retained_reserve_active_owner" + ); +} + +#[test] +fn dirty_stopped_owner_refuses_the_retained_reserve() { + let _serial = serial(); + for phase in [ + "recovered-unclean", + "stopped-before-engine", + "stopped-after-engine-failure", + "failed-boot-stopped", + "stopping", + "quiescing", + "booting", + "creating", + "initializing", + ] { + let mut pool = stopped_pool(); + pool.owner.phase = phase.into(); + pool.save(); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty, "{phase}"); + } + + // A phase rewritten to stopped without a quiesced stop of the last boot. + let pool = stopped_pool(); + fs::remove_file(root(&pool.candidate).join("quiescence.json")).unwrap(); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty); + for (boot, daemon, start, result) in [ + ("22222222-2222-2222-2222-222222222222", 4242, 99, "quiesced"), + (BOOT, 4243, 99, "quiesced"), + (BOOT, 4242, 98, "quiesced"), + (BOOT, 4242, 99, "timeout"), + ] { + let pool = stopped_pool(); + quiescence(&pool, boot, daemon, start, result); + assert_eq!( + refusal(&pool), + StoppedRefusal::Dirty, + "{boot} {daemon} {start} {result}" + ); + } + + // A quiesced engine whose `down` never completed, or completed for another boot. + let mut pool = stopped_pool(); + pool.owner.clean_stop = None; + pool.save(); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty); + for (boot, daemon, start) in [ + ("22222222-2222-2222-2222-222222222222", 4242, 99), + (BOOT, 4243, 99), + (BOOT, 4242, 98), + ] { + let mut pool = stopped_pool(); + pool.owner.clean_stop = Some(state::CleanStop { + guest_boot_id: boot.into(), + daemon_pid: daemon, + daemon_start: start, + }); + pool.save(); + assert_eq!( + refusal(&pool), + StoppedRefusal::Dirty, + "{boot} {daemon} {start}" + ); + } + + let pool = stopped_pool(); + fs::write(root(&pool.candidate).join("owner.pending"), b"{}").unwrap(); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty); + assert_eq!(StoppedRefusal::Dirty.code(), "retained_reserve_dirty_owner"); +} + +/// The relabelling path: a later boot's `down` quiesced the engine and then failed +/// (`stop_uncertain`), `recover` recorded `recovered-unclean`, and a start refused before +/// boot rewrote the phase to `stopped`. The quiescence receipt and the boot identity still +/// match, but the stop was never completed, so the pool never qualifies. +#[test] +fn an_uncertain_stop_rewritten_to_stopped_never_qualifies() { + let _serial = serial(); + const LATER: &str = "22222222-2222-2222-2222-222222222222"; + let pool = stopped_pool(); + let candidate = &pool.candidate; + let mut owner = Owner::load(candidate).unwrap(); + // A later boot: `start_pool` begins it and the engine identity is recorded. + owner.begin_boot(); + assert_eq!(owner.clean_stop, None); + super::super::phase(candidate, &mut owner, "booting").unwrap(); + owner.guest_boot_id = Some(LATER.into()); + owner.daemon_pid = Some(5151); + owner.daemon_start = Some(123); + super::super::phase(candidate, &mut owner, "running").unwrap(); + // `down` quiesces that boot and then cannot prove the guest flushed and shut down. + super::super::phase(candidate, &mut owner, "quiescing").unwrap(); + quiescence(&pool, LATER, 5151, 123, "quiesced"); + super::super::phase(candidate, &mut owner, "stopping").unwrap(); + // `recover` proves the provider dead and records an unclean stop. + super::super::finish_stopped(candidate, &mut owner, "recovered-unclean").unwrap(); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty); + // A start refused before boot rewrites the phase to `stopped`. + let mut owner = Owner::load(candidate).unwrap(); + super::super::phase(candidate, &mut owner, "stopped").unwrap(); + assert_eq!(owner.clean_stop, None); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty); + + // The same rewrite after an unclean label on the clean boot itself, with no new boot. + for unclean in ["recovered-unclean", "stopped-before-engine"] { + let pool = stopped_pool(); + let mut owner = Owner::load(&pool.candidate).unwrap(); + if unclean == "recovered-unclean" { + super::super::finish_stopped(&pool.candidate, &mut owner, unclean).unwrap(); + } else { + super::super::phase(&pool.candidate, &mut owner, unclean).unwrap(); + } + super::super::phase(&pool.candidate, &mut owner, "stopped").unwrap(); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty, "{unclean}"); + } +} + +#[test] +fn a_start_refused_before_boot_keeps_a_clean_stop_clean() { + let _serial = serial(); + // Nothing booted since `down`, so the retained disks are exactly as it left them. + let pool = stopped_pool(); + let mut owner = Owner::load(&pool.candidate).unwrap(); + super::super::phase(&pool.candidate, &mut owner, "stopped").unwrap(); + assert!( + select(&pool.candidate, Profile::Development) + .unwrap() + .into_selection() + .unwrap() + .is_stopped() + ); +} + +#[test] +fn only_a_completed_quiesced_down_records_a_clean_stop() { + let _serial = serial(); + // `down` reached `stopping` for the clean boot; the provider has exited. + let unfinished = || { + let mut pool = stopped_pool(); + pool.owner.clean_stop = None; + pool.owner.phase = "stopping".into(); + pool.save(); + pool + }; + let pool = unfinished(); + let mut owner = Owner::load(&pool.candidate).unwrap(); + super::super::finish_quiesced_stop(&pool.candidate, &mut owner).unwrap(); + assert!(owner.clean_stop.is_some()); + assert_eq!(owner.clean_stop, state::CleanStop::of(&owner)); + assert!( + select(&pool.candidate, Profile::Development) + .unwrap() + .into_selection() + .unwrap() + .is_stopped() + ); + + // Recording `stopped` by any other route leaves the stop unproven. + let pool = unfinished(); + let mut owner = Owner::load(&pool.candidate).unwrap(); + super::super::finish_stopped(&pool.candidate, &mut owner, "stopped").unwrap(); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty); + + // When the provider's release cannot be proven, nothing is recorded. + let pool = unfinished(); + let receipt = fs::read(root(&pool.candidate).join("owner.json")).unwrap(); + let held = exclusive_lock(&pool.data().join("vm.lock")); + let mut owner = Owner::load(&pool.candidate).unwrap(); + assert!(super::super::finish_quiesced_stop(&pool.candidate, &mut owner).is_err()); + drop(held); + assert_eq!( + fs::read(root(&pool.candidate).join("owner.json")).unwrap(), + receipt + ); + assert_eq!(refusal(&pool), StoppedRefusal::Dirty); +} + +#[test] +fn stale_or_unknown_stopped_owner_refuses_the_retained_reserve() { + let _serial = serial(); + for missing in [ + "created", "storage", "overlay", "rootfs", "boot", "daemon", "process", + ] { + let mut pool = stopped_pool(); + match missing { + "created" => pool.owner.created = false, + "storage" => pool.owner.storage = None, + "overlay" => pool.owner.overlay = None, + "rootfs" => pool.owner.rootfs_digest = None, + "boot" => pool.owner.guest_boot_id = None, + "daemon" => pool.owner.daemon_start = None, + "process" => pool.owner.process = None, + _ => unreachable!(), + } + pool.save(); + assert_eq!(refusal(&pool), StoppedRefusal::Stale, "{missing}"); + } + + let pool = stopped_pool(); + let path = pool.data().join("storage.raw"); + fs::rename(&path, pool.data().join("storage.retained")).unwrap(); + sparse_disk(&path, 32, 1); + assert_eq!(refusal(&pool), StoppedRefusal::Stale); + + let pool = stopped_pool(); + fs::remove_file(pool.data().join("overlay.raw")).unwrap(); + assert_eq!(refusal(&pool), StoppedRefusal::Stale); + + let mut pool = stopped_pool(); + let path = pool.data().join("overlay.raw"); + OpenOptions::new() + .write(true) + .open(&path) + .unwrap() + .set_len(9 * GIB) + .unwrap(); + pool.owner.overlay = Some(identity::disk(&path).unwrap()); + pool.save(); + assert_eq!(refusal(&pool), StoppedRefusal::Stale); + + let pool = stopped_pool(); + fs::remove_file(pool.data().join("vm.lock")).unwrap(); + assert_eq!(leased_refusal(&pool), "retained_reserve_stale_owner"); + assert_eq!(StoppedRefusal::Stale.code(), "retained_reserve_stale_owner"); +} + +#[test] +fn selected_stopped_owner_drift_refuses_reverification() { + let _serial = serial(); + let mut pool = stopped_pool(); + let selection = select(&pool.candidate, Profile::Development) + .unwrap() + .into_selection() + .unwrap(); + pool.owner.phase = "booting".into(); + pool.save(); + assert_eq!( + selection.reverify(&pool.candidate).err().unwrap().code, + "admission_owner_changed" + ); + + let pool = stopped_pool(); + let selection = select(&pool.candidate, Profile::Development) + .unwrap() + .into_selection() + .unwrap(); + let held = exclusive_lock(&pool.data().join("vm.lock")); + let lease = lease(&pool); + assert_eq!( + selection + .reverify_leased(&pool.candidate, &lease) + .err() + .unwrap() + .code, + "retained_reserve_active_owner" + ); + drop(held); + selection.reverify_leased(&pool.candidate, &lease).unwrap(); + drop(lease); + + let pool = stopped_pool(); + let selection = select(&pool.candidate, Profile::Development) + .unwrap() + .into_selection() + .unwrap(); + let path = pool.data().join("overlay.raw"); + fs::rename(&path, pool.data().join("overlay.retained")).unwrap(); + sparse_disk(&path, 10, 2); + assert_eq!( + selection.reverify(&pool.candidate).err().unwrap().code, + "retained_reserve_stale_owner" + ); + + let pool = stopped_pool(); + let selection = select(&pool.candidate, Profile::Development) + .unwrap() + .into_selection() + .unwrap(); + fs::remove_file(root(&pool.candidate).join("quiescence.json")).unwrap(); + assert_eq!( + selection.reverify(&pool.candidate).err().unwrap().code, + "retained_reserve_dirty_owner" + ); +} + +/// A fixture host sample on `owned`'s disk basis, as the admission module reports it. +fn sample( + owned: admission::OwnedPool, + admitted: bool, + swapouts: Option, +) -> admission::Admission { + let (minimum_disk_free_bytes, disk_budget_basis) = match owned { + admission::OwnedPool::Fresh => (58 * GIB, "new-pool-storage-overlay-host-reserve"), + admission::OwnedPool::Running => (16 * GIB, "verified-live-owned-pool-host-reserve"), + admission::OwnedPool::Stopped => (16 * GIB, "verified-stopped-owned-pool-host-reserve"), + }; + admission::Admission { + profile: Profile::Development, + host_supported: true, + free_memory_bytes: Some(10 * GIB), + minimum_free_memory_bytes: 10 * GIB, + free_plus_file_cache_estimate_bytes: Some(10 * GIB), + memory_budget_basis: "fixture", + memory_pressure: admission::MemoryPressure::from_level(Some(1)), + memory_pressure_normal: true, + disk_free_bytes: Some(30 * GIB), + minimum_disk_free_bytes, + disk_budget_basis, + one_minute_load: Some(0.0), + load_ceiling: None, + thermal_normal: true, + swapouts, + admitted, + reasons: if admitted { + Vec::new() + } else { + vec!["Free disk is below the development VM storage and host reserve budget.".into()] + }, + retained_reserve_refusal: None, + } +} + +#[test] +fn a_fresh_budget_beside_a_created_stopped_pool_reports_why_the_reserve_did_not_apply() { + let _serial = serial(); + let mut pool = stopped_pool(); + pool.owner.phase = "recovered-unclean".into(); + pool.save(); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + let observed = probe_with(&pool.candidate, &choice, |owned| { + assert_eq!(owned, admission::OwnedPool::Fresh); + Ok(sample(owned, false, Some(7))) + }) + .unwrap(); + assert_eq!( + observed.retained_reserve_refusal, + Some("retained_reserve_dirty_owner") + ); + assert_eq!( + serde_json::to_value(&observed).unwrap()["retained_reserve_refusal"], + "retained_reserve_dirty_owner" + ); + + // No retained disks exist yet, so a fresh allocation needs no explanation. + let mut pool = stopped_pool(); + pool.owner.phase = "initializing".into(); + pool.owner.created = false; + pool.save(); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + let observed = probe_with(&pool.candidate, &choice, |owned| { + Ok(sample(owned, true, Some(7))) + }) + .unwrap(); + assert_eq!(observed.retained_reserve_refusal, None); + assert!( + serde_json::to_value(&observed) + .unwrap() + .get("retained_reserve_refusal") + .is_none() + ); +} + +#[test] +fn the_boot_gate_samples_the_qualifying_basis_and_its_swapouts() { + let _serial = serial(); + // A clean stopped selection is sampled on its reserve basis, not the fresh floor. + let pool = stopped_pool(); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + let mut asked = None; + let gate = admit_boot_with(&pool.candidate, &choice, Some(7), |owned| { + asked = Some(owned); + Ok(sample(owned, true, Some(7))) + }) + .unwrap(); + assert_eq!(asked, Some(admission::OwnedPool::Stopped)); + assert!(matches!(gate, BootGate::Admitted(_))); + + // Without a selection the unchanged fresh budget decides; a live selection keeps its own. + let fresh = Choice { + selection: None, + refusal: None, + }; + let mut asked = None; + admit_boot_with(&pool.candidate, &fresh, Some(7), |owned| { + asked = Some(owned); + Ok(sample(owned, true, Some(7))) + }) + .unwrap(); + assert_eq!(asked, Some(admission::OwnedPool::Fresh)); + let running = Pool::new(); + let choice = select(&running.candidate, Profile::Development).unwrap(); + let mut asked = None; + admit_boot_with(&running.candidate, &choice, Some(7), |owned| { + asked = Some(owned); + Ok(sample(owned, true, Some(7))) + }) + .unwrap(); + assert_eq!(asked, Some(admission::OwnedPool::Running)); + + // Refused samples and swapout change since qualification refuse the boot. + let choice = select(&pool.candidate, Profile::Development).unwrap(); + for (admitted, swapouts) in [(false, Some(7)), (true, Some(8)), (true, None)] { + let gate = admit_boot_with(&pool.candidate, &choice, Some(7), |owned| { + Ok(sample(owned, admitted, swapouts)) + }) + .unwrap(); + let BootGate::Refused(error) = gate else { + panic!("{admitted} {swapouts:?} must refuse"); + }; + assert_eq!(error.code, "admission_rejected"); + } +} + +#[test] +fn the_boot_gate_rechecks_the_selection_around_its_sample() { + let _serial = serial(); + let pool = stopped_pool(); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + let result = admit_boot_with(&pool.candidate, &choice, Some(7), |owned| { + // The receipt drifts while the host is sampled. + fs::write(root(&pool.candidate).join("owner.pending"), b"{}").unwrap(); + Ok(sample(owned, true, Some(7))) + }); + assert_eq!(result.err().unwrap().code, "retained_reserve_dirty_owner"); + + let mut pool = stopped_pool(); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + pool.owner.phase = "recovered-unclean".into(); + pool.save(); + let result = admit_boot_with(&pool.candidate, &choice, Some(7), |_| { + panic!("a drifted selection is never sampled") + }); + assert_eq!(result.err().unwrap().code, "admission_owner_changed"); +} + +#[test] +fn machine_start_needs_a_final_leased_recheck_of_the_selection() { + let _serial = serial(); + let pool = stopped_pool(); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + let admitted = || match admit_boot_with(&pool.candidate, &choice, Some(7), |owned| { + Ok(sample(owned, true, Some(7))) + }) + .unwrap() + { + BootGate::Admitted(boot) => boot, + BootGate::Refused(error) => panic!("{}", error.code), + }; + let lease = lease(&pool); + // After the pre-boot sample, a provider takes the VM lock or opens a disk. + let boot = admitted(); + let held = exclusive_lock(&pool.data().join("vm.lock")); + assert_eq!( + boot.release(&pool.candidate, &lease).err().unwrap().code, + "retained_reserve_active_owner" + ); + drop(held); + let boot = admitted(); + let handle = fs::File::open(pool.data().join("overlay.raw")).unwrap(); + assert_eq!( + boot.release(&pool.candidate, &lease).err().unwrap().code, + "retained_reserve_active_owner" + ); + drop(handle); + // Or the receipt changes. + let boot = admitted(); + fs::write(root(&pool.candidate).join("owner.pending"), b"{}").unwrap(); + assert_eq!( + boot.release(&pool.candidate, &lease).err().unwrap().code, + "retained_reserve_dirty_owner" + ); + fs::remove_file(root(&pool.candidate).join("owner.pending")).unwrap(); + admitted().release(&pool.candidate, &lease).unwrap(); +} + +#[test] +fn a_reserve_selection_only_restarts_and_never_creates() { + let _serial = serial(); + let pool = stopped_pool(); + let choice = select(&pool.candidate, Profile::Development).unwrap(); + let lease = lease(&pool); + let owner = Owner::load(&pool.candidate).unwrap(); + // Each case stands for a start whose basis `settle` already settled under the lease. + let settled = Settled(&choice); + let restart = restart_only(&pool.candidate, settled, &owner, &lease).unwrap(); + assert_eq!( + fresh_create(&restart).err().unwrap().code, + "admission_owner_changed" + ); + // The owner the start holds must be the selected, created pool. + let mut uncreated = owner.clone(); + uncreated.created = false; + assert_eq!( + restart_only(&pool.candidate, settled, &uncreated, &lease) + .err() + .unwrap() + .code, + "admission_owner_changed" + ); + let mut other = owner.clone(); + other.guest_boot_id = None; + assert_eq!( + restart_only(&pool.candidate, settled, &other, &lease) + .err() + .unwrap() + .code, + "admission_owner_changed" + ); + // The provider's release is proven under the lease before any restart effect. + let held = exclusive_lock(&pool.data().join("vm.lock")); + assert_eq!( + restart_only(&pool.candidate, settled, &owner, &lease) + .err() + .unwrap() + .code, + "retained_reserve_active_owner" + ); + drop(held); + drop(lease); + + // A live selection never restarts or creates. + let running = Pool::new(); + let choice = select(&running.candidate, Profile::Development).unwrap(); + let lease = state::Lock::acquire(&root(&running.candidate)).unwrap(); + let owner = Owner::load(&running.candidate).unwrap(); + assert_eq!( + restart_only(&running.candidate, Settled(&choice), &owner, &lease) + .err() + .unwrap() + .code, + "admission_owner_changed" + ); + assert!(fresh_create(&Restart(&choice)).is_err()); + + // Without a selection, the unchanged fresh path may create. + let fresh = Choice { + selection: None, + refusal: None, + }; + let restart = restart_only(&running.candidate, Settled(&fresh), &owner, &lease).unwrap(); + fresh_create(&restart).unwrap(); +} + +/// Qualifying samples on `owned`'s basis, each observing `free` bytes of free disk. +fn window(owned: admission::OwnedPool, free: &[u64]) -> Vec { + free.iter() + .map(|&bytes| { + let mut observed = sample(owned, true, Some(7)); + observed.disk_free_bytes = Some(bytes); + observed + }) + .collect() +} + +#[test] +fn a_start_with_fresh_headroom_drops_the_reserve_and_runs_no_release_check() { + let _serial = serial(); + let stopped = admission::OwnedPool::Stopped; + let fresh_floor = 58 * GIB; + let pool = stopped_pool(); + let lease = lease(&pool); + // A provider holds the VM lock, so any release check would refuse. + let held = exclusive_lock(&pool.data().join("vm.lock")); + + // Every sample fits the fresh budget: the start continues exactly as one without a + // selection, so the held lock is never examined. + let mut choice = select(&pool.candidate, Profile::Development).unwrap(); + assert!(choice.selected().is_some_and(Selection::is_stopped)); + let mut samples = window(stopped, &[fresh_floor, 80 * GIB, fresh_floor, 70 * GIB]); + let settled = settle(&pool.candidate, &mut choice, &lease, &mut samples).unwrap(); + assert!(settled.selected().is_none()); + assert_eq!(settled.0.refusal(), None); + for observed in &samples { + assert_eq!(observed.minimum_disk_free_bytes, fresh_floor); + assert_eq!( + observed.disk_budget_basis, + "new-pool-storage-overlay-host-reserve" + ); + assert!(observed.admitted); + assert_eq!(observed.retained_reserve_refusal, None); + } + // From here the start is the unchanged fresh path: no restart-only gate, a fresh + // pre-boot sample and a start permit without release checks. + let owner = Owner::load(&pool.candidate).unwrap(); + let restart = restart_only(&pool.candidate, settled, &owner, &lease).unwrap(); + fresh_create(&restart).unwrap(); + let mut asked = None; + let gate = admit_boot_with(&pool.candidate, restart.0, Some(7), |owned| { + asked = Some(owned); + Ok(sample(owned, true, Some(7))) + }) + .unwrap(); + assert_eq!(asked, Some(admission::OwnedPool::Fresh)); + let BootGate::Admitted(boot) = gate else { + panic!("fresh pre-boot sample admitted") + }; + boot.release(&pool.candidate, &lease).unwrap(); + + // One sample below the fresh budget needs the reserve: its release checks run under the + // lease and refuse the start; nothing is rebased or dropped. + let mut choice = select(&pool.candidate, Profile::Development).unwrap(); + let mut samples = window(stopped, &[80 * GIB, fresh_floor - 1, 80 * GIB]); + assert_eq!( + settle(&pool.candidate, &mut choice, &lease, &mut samples) + .err() + .unwrap() + .code, + "retained_reserve_active_owner" + ); + assert!(choice.selected().is_some_and(Selection::is_stopped)); + assert!( + samples.iter().all( + |observed| observed.disk_budget_basis == "verified-stopped-owned-pool-host-reserve" + ) + ); + drop(held); + // Released, the reserve-band start keeps its selection on the reserve basis. + let settled = settle(&pool.candidate, &mut choice, &lease, &mut samples).unwrap(); + assert!(settled.selected().is_some_and(Selection::is_stopped)); + assert_eq!(samples[1].minimum_disk_free_bytes, 16 * GIB); + + // A drifted receipt is refused before any basis is settled, even with fresh headroom. + let mut choice = select(&pool.candidate, Profile::Development).unwrap(); + fs::write(root(&pool.candidate).join("owner.pending"), b"{}").unwrap(); + let mut samples = window(stopped, &[80 * GIB, 80 * GIB, 80 * GIB]); + assert_eq!( + settle(&pool.candidate, &mut choice, &lease, &mut samples) + .err() + .unwrap() + .code, + "retained_reserve_dirty_owner" + ); + assert!(choice.selected().is_some()); + fs::remove_file(root(&pool.candidate).join("owner.pending")).unwrap(); + drop(lease); + + // A live selection keeps its own basis whatever the headroom. + let running = Pool::new(); + let lease = state::Lock::acquire(&root(&running.candidate)).unwrap(); + let mut choice = select(&running.candidate, Profile::Development).unwrap(); + let mut samples = window(admission::OwnedPool::Running, &[80 * GIB, 80 * GIB]); + let settled = settle(&running.candidate, &mut choice, &lease, &mut samples).unwrap(); + assert!( + settled + .selected() + .is_some_and(|selected| !selected.is_stopped()) + ); + assert!( + samples + .iter() + .all(|observed| observed.disk_budget_basis == "verified-live-owned-pool-host-reserve") + ); +} diff --git a/packages/runtime-core/src/provider/lifecycle/pressure_tests.rs b/packages/runtime-core/src/provider/lifecycle/pressure_tests.rs index 1c1dbd337..d7280223e 100644 --- a/packages/runtime-core/src/provider/lifecycle/pressure_tests.rs +++ b/packages/runtime-core/src/provider/lifecycle/pressure_tests.rs @@ -26,6 +26,7 @@ fn report(level: u32) -> Admission { swapouts: Some(42), admitted: true, reasons: Vec::new(), + retained_reserve_refusal: None, } } diff --git a/packages/runtime-core/src/provider/state.rs b/packages/runtime-core/src/provider/state.rs index 2168dd31d..093f4ea02 100644 --- a/packages/runtime-core/src/provider/state.rs +++ b/packages/runtime-core/src/provider/state.rs @@ -241,7 +241,41 @@ pub struct Owner { pub profile: super::Profile, #[serde(default)] pub reclamation: Option, + /// The boot a `down` stopped cleanly. Recorded only by `down`, in the same receipt write + /// as its `stopped` phase, after the guest acknowledged flush and shutdown and the + /// provider's absence was proven. Any other phase write and every new boot clear it, so + /// a later rewrite of the phase to `stopped` never relabels an unclean stop as clean. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub clean_stop: Option, +} + +/// The engine identity of the boot whose `down` completed cleanly. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CleanStop { + pub guest_boot_id: String, + pub daemon_pid: u32, + pub daemon_start: u64, } + +impl CleanStop { + /// The marker for `owner`'s current boot, when its engine identity is complete. + pub fn of(owner: &Owner) -> Option { + Some(Self { + guest_boot_id: owner.guest_boot_id.clone()?, + daemon_pid: owner.daemon_pid?, + daemon_start: owner.daemon_start?, + }) + } + + /// Whether this marker names exactly this boot and engine. + pub fn names(&self, guest_boot_id: &str, daemon_pid: u32, daemon_start: u64) -> bool { + self.guest_boot_id == guest_boot_id + && self.daemon_pid == daemon_pid + && self.daemon_start == daemon_start + } +} + impl Owner { pub fn load(candidate: &Candidate) -> Result { Self::load_with_short_home(candidate, false) @@ -453,6 +487,7 @@ impl Owner { guest_boot_id: None, previous_guest_boot_id: None, reclamation: None, + clean_stop: None, daemon_pid: None, daemon_start: None, rootfs_digest: None, @@ -494,6 +529,8 @@ impl Owner { write(&candidate.state_root.join("run/smolvm/owner.json"), self) } pub fn begin_boot(&mut self) -> Option { + // A new boot supersedes any clean stop of the previous one. + self.clean_stop = None; self.previous_guest_boot_id = self .guest_boot_id .take()