diff --git a/Cargo.lock b/Cargo.lock index 862a2f4d67c..4a38b122d66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2105,6 +2105,13 @@ dependencies = [ "log", ] +[[package]] +name = "environment-test" +version = "0.0.0" +dependencies = [ + "spacetimedb 2.3.0", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -2893,6 +2900,13 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hosted-auth-test" +version = "0.0.0" +dependencies = [ + "spacetimedb 2.3.0", +] + [[package]] name = "hostname" version = "0.3.1" @@ -7806,6 +7820,7 @@ name = "spacetimedb-cli" version = "2.3.0" dependencies = [ "anyhow", + "axum", "base64 0.21.7", "bytes", "cargo_metadata", @@ -7843,10 +7858,12 @@ dependencies = [ "rolldown_common", "rolldown_error", "rolldown_utils", + "rustix 1.1.2", "rustyline", "serde", "serde_json", "serde_with", + "sha2", "slab", "spacetimedb-auth", "spacetimedb-client-api-messages", @@ -7855,6 +7872,7 @@ dependencies = [ "spacetimedb-fs-utils", "spacetimedb-jsonwebtoken", "spacetimedb-lib 2.3.0", + "spacetimedb-oci", "spacetimedb-paths", "spacetimedb-primitives 2.3.0", "spacetimedb-schema", @@ -7869,9 +7887,12 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-tungstenite 0.27.0", + "tokio-util", "toml 0.8.23", "toml_edit 0.22.27", "tracing", + "url", + "uuid", "walkdir", "wasmbin", "webbrowser", @@ -8369,6 +8390,20 @@ dependencies = [ "prometheus", ] +[[package]] +name = "spacetimedb-oci" +version = "2.3.0" +dependencies = [ + "anyhow", + "flate2", + "serde", + "serde_json", + "sha2", + "spacetimedb-lib 2.3.0", + "tar", + "zstd", +] + [[package]] name = "spacetimedb-paths" version = "2.3.0" @@ -8775,11 +8810,13 @@ dependencies = [ "serde", "serde_json", "serial_test", + "spacetimedb-auth", "spacetimedb-cli", "spacetimedb-client-api", "spacetimedb-client-api-messages", "spacetimedb-core", "spacetimedb-data-structures", + "spacetimedb-datastore", "spacetimedb-lib 2.3.0", "spacetimedb-paths", "spacetimedb-schema", @@ -10236,6 +10273,7 @@ checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.4", "js-sys", + "serde", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index 2968240b375..815bdc7f62c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "crates/guard", "crates/fs-utils", "crates/lib", + "crates/oci", "crates/metrics", "crates/paths", "crates/pg", @@ -42,6 +43,8 @@ members = [ "modules/keynote-benchmarks", "modules/perf-test", "modules/module-test", + "modules/hosted-auth-test", + "modules/environment-test", "templates/basic-rs/spacetimedb", "templates/chat-console-rs/spacetimedb", "modules/sdk-test", diff --git a/crates/auth/src/hosted.rs b/crates/auth/src/hosted.rs new file mode 100644 index 00000000000..2f860510801 --- /dev/null +++ b/crates/auth/src/hosted.rs @@ -0,0 +1,403 @@ +//! Target-bound credentials for a database's hosted container. +//! +//! Decoded claims are untrusted. Only signature verification against an explicitly +//! configured platform issuer and comparison with an authoritative instance/grant +//! binding can produce [`VerifiedHostedAuth`]. Receiving hosts must additionally +//! recheck their durable target fence at each transaction and subscription admission. + +use crate::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; +use anyhow::{bail, ensure, Context}; +use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, EncodingKey, Header, Validation}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::Identity; +use std::fmt; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +pub const HOSTED_TOKEN_KIND: &str = "spacetimedb_hosted_v1"; +pub const HOSTED_TOKEN_TYPE: &str = "spacetimedb-hosted+jwt"; +pub const MAX_HOSTED_TOKEN_LIFETIME: Duration = Duration::from_secs(30); +pub const MAX_HOSTED_TOKEN_BYTES: usize = 8192; + +/// Wire claims, deliberately distinct from ordinary issuer/subject-derived Identity claims. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostedTokenClaims { + pub kind: Box, + #[serde(rename = "iss")] + pub issuer: Box, + #[serde(rename = "sub")] + pub subject: Box, + #[serde(with = "identity_hex")] + pub source_database: Identity, + /// A scalar, canonical database Identity. Lists and database names are not accepted. + #[serde(rename = "aud", with = "identity_hex")] + pub target_database: Identity, + pub generation: u64, + pub grant_revision: u64, + #[serde(rename = "iat")] + pub issued_at: u64, + #[serde(rename = "exp")] + pub expires_at: u64, + #[serde(rename = "jti")] + pub token_id: Box, +} + +/// Trusted input obtained from current source registration, placement and target grant state. +/// Never construct this by copying the incoming token's claims. Admission must be open, +/// and the assigned node/incarnation and required module capability must already be checked. +#[derive(Clone, Copy, Debug)] +pub struct HostedTokenBinding { + pub source_database: Identity, + pub target_database: Identity, + pub generation: u64, + pub grant_revision: u64, + pub lease_expires_at: SystemTime, +} + +/// Authentication proof. It cannot be deserialized or constructed from decoded claims. +#[derive(Clone)] +pub struct VerifiedHostedAuth { + claims: HostedTokenClaims, + // Local lifetime state only. Never serialized into signed claims. + monotonic_deadline: Instant, +} + +impl fmt::Debug for VerifiedHostedAuth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VerifiedHostedAuth") + .field("source_database", &self.source_database()) + .field("target_database", &self.target_database()) + .field("generation", &self.generation()) + .field("grant_revision", &self.grant_revision()) + .field("expires_at", &self.expires_at()) + .finish_non_exhaustive() + } +} + +impl VerifiedHostedAuth { + pub fn source_database(&self) -> Identity { + self.claims.source_database + } + pub fn target_database(&self) -> Identity { + self.claims.target_database + } + pub fn generation(&self) -> u64 { + self.claims.generation + } + pub fn grant_revision(&self) -> u64 { + self.claims.grant_revision + } + pub fn issued_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.claims.issued_at) + } + pub fn expires_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.claims.expires_at) + } + pub fn token_id(&self) -> &str { + &self.claims.token_id + } + pub fn issuer(&self) -> &str { + &self.claims.issuer + } + pub fn is_internal(&self) -> bool { + self.source_database() == self.target_database() + } + + /// Expiry has no positive leeway. This does not replace generation/grant fencing. + pub fn check_at(&self, now: SystemTime) -> anyhow::Result<()> { + self.check_at_clocks(now, Instant::now()) + } + + fn check_at_clocks(&self, now: SystemTime, monotonic_now: Instant) -> anyhow::Result<()> { + ensure!(now >= self.issued_at(), "hosted credential is not yet valid"); + ensure!( + now < self.expires_at() && monotonic_now < self.monotonic_deadline, + "hosted credential expired" + ); + Ok(()) + } + + /// Cap the local lifetime using a fresh authority confirmation. The caller + /// records `confirmation_started` before sending the request and obtains + /// `confirmed_time` from the successful authoritative response. Charging the + /// entire round trip against the signed remaining lifetime is conservative. + /// Neither a receiving clock behind authority nor a later confirmation can + /// extend this proof. The original signed claims are preserved exactly. + pub fn constrain_expiration( + mut self, + confirmed_time: SystemTime, + confirmation_started: Instant, + ) -> anyhow::Result { + let now = Instant::now(); + ensure!(confirmation_started <= now, "invalid hosted confirmation clock"); + self.check_at_clocks(confirmed_time, now)?; + let remaining = self.expires_at().duration_since(confirmed_time)?; + let confirmed_deadline = confirmation_started + .checked_add(remaining) + .context("hosted confirmation deadline overflow")?; + self.monotonic_deadline = self.monotonic_deadline.min(confirmed_deadline); + self.check_at_clocks(confirmed_time, now)?; + Ok(self) + } + + /// Remaining connection lifetime, bounded by both signed wall-clock expiry + /// and the already captured local deadline. Repeated calls never reset it. + pub fn remaining_lifetime(&self, now: SystemTime) -> Duration { + self.remaining_lifetime_at_clocks(now, Instant::now()) + } + + fn remaining_lifetime_at_clocks(&self, now: SystemTime, monotonic_now: Instant) -> Duration { + if self.check_at_clocks(now, monotonic_now).is_err() { + return Duration::ZERO; + } + self.expires_at() + .duration_since(now) + .unwrap_or_default() + .min(self.monotonic_deadline.saturating_duration_since(monotonic_now)) + } + + pub fn into_connection_auth(self) -> anyhow::Result { + // Keep the actual claims, including the source/target/generation restrictions. + // Normalizing JSON whitespace does not alter any signed claim values. + let jwt_payload = serde_json::to_string(&self.claims)?.into_boxed_str(); + let mut extra = serde_json::to_value(&self.claims)?; + let extra = extra.as_object_mut().expect("hosted claims serialize as an object"); + for key in ["iss", "sub", "aud", "iat", "exp"] { + extra.remove(key); + } + let claims = SpacetimeIdentityClaims { + identity: self.source_database(), + subject: self.claims.subject.clone(), + issuer: self.claims.issuer.clone(), + audience: [self.target_database().to_hex().to_string().into_boxed_str()].into(), + iat: self.issued_at(), + exp: Some(self.expires_at()), + extra: Some( + extra + .iter() + .map(|(key, value)| (key.clone().into_boxed_str(), value.clone())) + .collect(), + ), + }; + Ok(ConnectionAuthCtx { + claims, + jwt_payload, + hosted: Some(self), + }) + } +} + +/// Classifies the reserved namespace only. A positive result grants no authority. +/// All reserved versions are rejected by ordinary OIDC validation and token exchange. +pub fn has_reserved_hosted_token_kind(token: &str) -> anyhow::Result { + classify_reserved_token(token, is_reserved_hosted_type, is_reserved_hosted_kind) +} + +/// Operational container proofs are never client Identity credentials. Reserve +/// their entire versioned namespace so a lease/registry proof cannot enter +/// OIDC discovery, ordinary JWT validation, or the Identity token exchange. +pub fn has_reserved_platform_token_kind(token: &str) -> anyhow::Result { + classify_reserved_token( + token, + |kind| is_reserved_hosted_type(kind) || kind.starts_with("spacetimedb-container-"), + |kind| is_reserved_hosted_kind(kind) || kind.starts_with("spacetimedb_container_"), + ) +} + +fn classify_reserved_token( + token: &str, + reserved_type: impl FnOnce(&str) -> bool, + reserved_kind: impl FnOnce(&str) -> bool, +) -> anyhow::Result { + let header = decode_header(token)?; + if header.typ.as_deref().is_some_and(reserved_type) { + return Ok(true); + } + let mut validation = Validation::new(Algorithm::ES256); + validation.required_spec_claims.clear(); + validation.validate_exp = false; + validation.validate_aud = false; + validation.insecure_disable_signature_validation(); + let data = decode::(token, &DecodingKey::from_secret(b"classification-only"), &validation)?; + Ok(data + .claims + .get("kind") + .and_then(serde_json::Value::as_str) + .is_some_and(reserved_kind)) +} + +pub fn is_reserved_hosted_kind(kind: &str) -> bool { + kind.starts_with("spacetimedb_hosted_") +} +fn is_reserved_hosted_type(kind: &str) -> bool { + kind.starts_with("spacetimedb-hosted") +} + +/// Decode routing hints only, never authentication. The caller must use these hints to +/// find trusted registration/grant state and then call [`verify_hosted_token`]. +pub fn unverified_hosted_token_claims(token: &str) -> anyhow::Result { + ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); + let mut validation = Validation::new(Algorithm::ES256); + validation.required_spec_claims.clear(); + validation.validate_exp = false; + validation.validate_aud = false; + validation.insecure_disable_signature_validation(); + Ok(decode::(token, &DecodingKey::from_secret(b"routing-only"), &validation)?.claims) +} + +/// Verify against a configured key and issuer, never a JWT-supplied key or JWKS URL. +/// `binding` must be authoritative state for that issuer's registered source database. +pub fn verify_hosted_token( + token: &str, + public_key: &DecodingKey, + trusted_issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, +) -> anyhow::Result { + let verification_started = Instant::now(); + ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); + let header = decode_header(token)?; + ensure!(header.alg == Algorithm::ES256, "hosted credential requires ES256"); + ensure!( + header.typ.as_deref() == Some(HOSTED_TOKEN_TYPE), + "invalid hosted credential type" + ); + let mut validation = Validation::new(Algorithm::ES256); + validation.set_required_spec_claims(&["iss", "sub", "aud", "exp"]); + validation.set_issuer(&[trusted_issuer]); + validation.set_audience(&[binding.target_database.to_hex().to_string()]); + validation.leeway = 0; + // Check time below against the caller's trusted clock, including exact expiry equality. + validation.validate_exp = false; + let claims = decode::(token, public_key, &validation)?.claims; + validate_claims(&claims, trusted_issuer, binding, now)?; + let remaining = (UNIX_EPOCH + Duration::from_secs(claims.expires_at)).duration_since(now)?; + let monotonic_deadline = verification_started + .checked_add(remaining) + .context("hosted credential deadline overflow")?; + let proof = VerifiedHostedAuth { + claims, + monotonic_deadline, + }; + proof.check_at(now)?; + Ok(proof) +} + +/// Mint from the broker's authoritative binding, with no guest-selected sender or generation. +pub fn sign_hosted_token( + private_key: &EncodingKey, + trusted_issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, + expires_at: SystemTime, + token_id: &str, +) -> anyhow::Result { + let claims = HostedTokenClaims { + kind: HOSTED_TOKEN_KIND.into(), + issuer: trusted_issuer.into(), + subject: binding.source_database.to_hex().to_string().into_boxed_str(), + source_database: binding.source_database, + target_database: binding.target_database, + generation: binding.generation, + grant_revision: binding.grant_revision, + issued_at: unix_seconds(now)?, + expires_at: unix_seconds(expires_at)?, + token_id: token_id.into(), + }; + validate_claims(&claims, trusted_issuer, binding, now)?; + let mut header = Header::new(Algorithm::ES256); + header.typ = Some(HOSTED_TOKEN_TYPE.into()); + Ok(jsonwebtoken::encode(&header, &claims, private_key)?) +} + +fn validate_claims( + claims: &HostedTokenClaims, + issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, +) -> anyhow::Result<()> { + ensure!( + !issuer.is_empty() && issuer.len() <= 128, + "invalid trusted hosted issuer" + ); + ensure!( + claims.kind.as_ref() == HOSTED_TOKEN_KIND, + "unsupported hosted credential kind" + ); + ensure!(claims.issuer.as_ref() == issuer, "untrusted hosted credential issuer"); + ensure!( + claims.source_database == binding.source_database, + "hosted credential source mismatch" + ); + ensure!( + claims.target_database == binding.target_database, + "hosted credential target mismatch" + ); + ensure!( + claims.subject.as_ref() == claims.source_database.to_hex().as_str(), + "hosted credential subject mismatch" + ); + ensure!( + claims.generation == binding.generation, + "hosted credential generation mismatch" + ); + ensure!( + claims.grant_revision == binding.grant_revision, + "hosted credential grant revision mismatch" + ); + ensure!( + !claims.token_id.is_empty() + && claims.token_id.len() <= 128 + && claims + .token_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'), + "invalid hosted credential token ID" + ); + let Some(lifetime) = claims.expires_at.checked_sub(claims.issued_at) else { + bail!("invalid hosted credential lifetime") + }; + ensure!( + lifetime > 0 && lifetime <= MAX_HOSTED_TOKEN_LIFETIME.as_secs(), + "hosted credential lifetime exceeds limit" + ); + let issued_at = UNIX_EPOCH + .checked_add(Duration::from_secs(claims.issued_at)) + .context("invalid hosted issue time")?; + let expires_at = UNIX_EPOCH + .checked_add(Duration::from_secs(claims.expires_at)) + .context("invalid hosted expiry")?; + ensure!( + issued_at <= now && now < expires_at, + "hosted credential outside validity interval" + ); + ensure!( + expires_at <= binding.lease_expires_at, + "hosted credential exceeds confirmed lease" + ); + Ok(()) +} + +fn unix_seconds(time: SystemTime) -> anyhow::Result { + Ok(time.duration_since(UNIX_EPOCH)?.as_secs()) +} + +mod identity_hex { + use super::*; + pub fn serialize(value: &Identity, serializer: S) -> Result { + serializer.serialize_str(value.to_hex().as_str()) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(serde::de::Error::custom( + "expected canonical 64-character lowercase database Identity", + )); + } + Identity::from_hex(value).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +#[path = "hosted/expiration_tests.rs"] +mod expiration_tests; diff --git a/crates/auth/src/hosted/expiration_tests.rs b/crates/auth/src/hosted/expiration_tests.rs new file mode 100644 index 00000000000..1624bfc474f --- /dev/null +++ b/crates/auth/src/hosted/expiration_tests.rs @@ -0,0 +1,136 @@ +use super::*; + +fn proof() -> (VerifiedHostedAuth, SystemTime, Instant) { + let wall = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let monotonic = Instant::now(); + ( + VerifiedHostedAuth { + claims: HostedTokenClaims { + kind: HOSTED_TOKEN_KIND.into(), + issuer: "platform.test".into(), + subject: Identity::ZERO.to_hex().to_string().into(), + source_database: Identity::ZERO, + target_database: Identity::ZERO, + generation: 1, + grant_revision: 2, + issued_at: 1_700_000_000, + expires_at: 1_700_000_030, + token_id: "expiration-test".into(), + }, + monotonic_deadline: monotonic + Duration::from_secs(30), + }, + wall, + monotonic, + ) +} + +#[test] +fn receiving_clock_behind_control_cannot_extend_confirmed_lifetime() { + let (proof, wall, monotonic) = proof(); + let request_started = monotonic - Duration::from_secs(2); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), request_started) + .unwrap(); + let deadline = request_started + Duration::from_secs(5); + assert_eq!(proof.monotonic_deadline, deadline); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(3), deadline - Duration::from_nanos(1)) + .is_ok()); + assert!(proof.check_at_clocks(wall + Duration::from_secs(3), deadline).is_err()); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(3), deadline), + Duration::ZERO + ); +} + +#[test] +fn confirmation_round_trip_consumes_remaining_signed_lifetime() { + let (proof, wall, monotonic) = proof(); + assert!(proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic - Duration::from_secs(6)) + .is_err()); +} + +#[test] +fn later_confirmation_cannot_extend_a_previous_deadline() { + let (proof, wall, monotonic) = proof(); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic) + .unwrap(); + let first_deadline = proof.monotonic_deadline; + // Even a second response with an earlier authority timestamp cannot extend + // the stricter deadline already held by this authentication proof. + let proof = proof + .constrain_expiration(wall + Duration::from_secs(20), Instant::now()) + .unwrap(); + assert_eq!(proof.monotonic_deadline, first_deadline); +} + +#[test] +fn backward_wall_clock_does_not_restart_the_local_lifetime() { + let (proof, wall, monotonic) = proof(); + let later = monotonic + Duration::from_secs(29); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(1), later), + Duration::from_secs(1) + ); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(1), monotonic + Duration::from_secs(30)) + .is_err()); +} + +#[test] +fn wall_clock_expiration_remains_an_independent_limit() { + let (proof, wall, monotonic) = proof(); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(29), monotonic), + Duration::from_secs(1) + ); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(30), monotonic) + .is_err()); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall - Duration::from_secs(1), monotonic), + Duration::ZERO + ); +} + +#[test] +fn invalid_confirmation_clocks_are_rejected() { + let (proof, wall, monotonic) = proof(); + assert!(proof + .clone() + .constrain_expiration(wall - Duration::from_secs(1), monotonic) + .is_err()); + assert!(proof + .clone() + .constrain_expiration(wall + Duration::from_secs(30), monotonic) + .is_err()); + assert!(proof + .constrain_expiration(wall, Instant::now() + Duration::from_secs(60)) + .is_err()); +} + +#[test] +fn cloning_and_connection_conversion_preserve_the_cap_and_signed_claims() { + let (proof, wall, monotonic) = proof(); + let original_claims = serde_json::to_value(&proof.claims).unwrap(); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic) + .unwrap(); + let deadline = proof.monotonic_deadline; + let cloned = proof.clone(); + let connection = proof.into_connection_auth().unwrap(); + let connection = connection.clone(); + let retained = connection.hosted.as_ref().unwrap(); + assert_eq!(cloned.monotonic_deadline, deadline); + assert_eq!(retained.monotonic_deadline, deadline); + assert_eq!(connection.claims.exp, Some(wall + Duration::from_secs(30))); + assert_eq!( + serde_json::from_str::(&connection.jwt_payload).unwrap(), + original_claims + ); + assert!(retained + .check_at_clocks(wall + Duration::from_secs(1), deadline) + .is_err()); +} diff --git a/crates/auth/src/identity.rs b/crates/auth/src/identity.rs index 5ecbe61dec7..960187f72b2 100644 --- a/crates/auth/src/identity.rs +++ b/crates/auth/src/identity.rs @@ -7,10 +7,20 @@ use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::Identity; use std::time::SystemTime; -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ConnectionAuthCtx { pub claims: SpacetimeIdentityClaims, pub jwt_payload: Box, + pub hosted: Option, +} + +impl std::fmt::Debug for ConnectionAuthCtx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConnectionAuthCtx") + .field("identity", &self.claims.identity) + .field("hosted", &self.hosted) + .finish_non_exhaustive() + } } impl TryFrom for ConnectionAuthCtx { @@ -20,6 +30,7 @@ impl TryFrom for ConnectionAuthCtx { Ok(ConnectionAuthCtx { claims, jwt_payload: payload.into(), + hosted: None, }) } } @@ -98,6 +109,15 @@ impl TryInto for IncomingClaims { type Error = anyhow::Error; fn try_into(self) -> anyhow::Result { + if self + .extra + .as_ref() + .and_then(|extra| extra.get("kind")) + .and_then(serde_json::Value::as_str) + .is_some_and(crate::hosted::is_reserved_hosted_kind) + { + anyhow::bail!("hosted credentials require dedicated target-bound validation"); + } // The issuer and subject must be less than 128 bytes. if self.issuer.len() > 128 { return Err(anyhow::anyhow!("Issuer too long: {:?}", self.issuer)); diff --git a/crates/auth/src/lib.rs b/crates/auth/src/lib.rs index db53a0c9064..b8b0976f40d 100644 --- a/crates/auth/src/lib.rs +++ b/crates/auth/src/lib.rs @@ -1 +1,2 @@ +pub mod hosted; pub mod identity; diff --git a/crates/bindings-cpp/README.md b/crates/bindings-cpp/README.md index ef31361c10b..4fbfc2a2d86 100644 --- a/crates/bindings-cpp/README.md +++ b/crates/bindings-cpp/README.md @@ -2,6 +2,30 @@ The SpacetimeDB C++ Module Library provides a modern C++20 API for building SpacetimeDB modules that run inside the database as WebAssembly. +## Function visibility and invocation authentication + +Apply `SPACETIMEDB_FUNCTION_VISIBILITY(name, Public)`, `Private`, or `Internal` +to a reducer or procedure after its definition: + +```cpp +SPACETIMEDB_REDUCER(process_jobs, ReducerContext ctx) { + return Ok(); +} +SPACETIMEDB_FUNCTION_VISIBILITY(process_jobs, Internal); +``` + +Omission means public for ordinary functions and private for scheduled functions. +An explicit choice is preserved when the function is scheduled. Lifecycle +reducers permit only omission or `Internal` and can only run for their host +lifecycle event. Internal functions require verified internal authority. Private +functions also admit the owner, and public functions admit any client. + +`ctx.sender_auth().is_internal()` captures the host's invocation authority. It is +independent of connection and JWT presence, so an internal call can have a JWT. +JWT identity is the verified sender supplied by the host. Procedures preserve +this authentication in `with_tx` and `try_with_tx`. Newly compiled modules emit +schema V10 and advertise `hosted_auth_v1`, requiring a compatible host. + ## Current State This library provides a production-ready C++ bindings for SpacetimeDB with complete type system support: @@ -274,4 +298,3 @@ See the `modules/*-cpp/src/` directory for example modules: ## Contributing This library is part of the SpacetimeDB project. Please see the main repository for contribution guidelines. - diff --git a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h index 23064a518f2..138cbdd6ce6 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h @@ -73,6 +73,8 @@ using ::identity; // ===== JWT ===== using ::get_jwt; +using ::get_call_auth_flags; +using ::env_get; // ===== Procedure Transactions ===== #ifdef SPACETIMEDB_UNSTABLE_FEATURES diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index e1aa12ac2de..136868a2c2c 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -39,6 +39,12 @@ #define STDB_IMPORT_10_5(name) \ __attribute__((import_module("spacetime_10.5"), import_name(#name))) extern +#define STDB_IMPORT_10_6(name) \ + __attribute__((import_module("spacetime_10.6"), import_name(#name))) extern + +#define STDB_IMPORT_10_7(name) \ + __attribute__((import_module("spacetime_10.7"), import_name(#name))) extern + // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; using SpacetimeDB::TableId; @@ -59,6 +65,13 @@ using SpacetimeDB::ConsoleTimerId; extern "C" { +STDB_IMPORT_10_7(env_get) +Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out); + +// Verified invocation authority. Bit 0 is INTERNAL; JWT presence is independent. +STDB_IMPORT_10_6(get_call_auth_flags) +uint32_t get_call_auth_flags(); + // ===== Table and Index Management ===== STDB_IMPORT(table_id_from_name) Status table_id_from_name(const uint8_t* name_ptr, size_t name_len, TableId* out); diff --git a/crates/bindings-cpp/include/spacetimedb/auth_ctx.h b/crates/bindings-cpp/include/spacetimedb/auth_ctx.h index 00a4898e020..a8ae7fc62c9 100644 --- a/crates/bindings-cpp/include/spacetimedb/auth_ctx.h +++ b/crates/bindings-cpp/include/spacetimedb/auth_ctx.h @@ -28,22 +28,25 @@ struct ConnectionId; class AuthCtx { private: bool is_internal_; + std::optional verified_sender_; mutable std::shared_ptr> jwt_; std::function()> jwt_loader_; // Private constructor used by factory methods - AuthCtx(bool is_internal, std::function()> loader); + AuthCtx(bool is_internal, std::function()> loader, + std::optional verified_sender = std::nullopt); + static AuthCtx from_connection_with_flags(ConnectionId connection_id, Identity sender, uint32_t flags); public: /** * @brief Creates an AuthCtx from an optional ConnectionId. * * If the connection_id is present, creates an AuthCtx that will load the JWT. - * If the connection_id is absent, creates an internal AuthCtx. + * Internal authority is captured from the host, independently of connection presence. * * @param connection_id Optional connection ID - * @param sender The identity of the caller (already derived from JWT claims by the host) - * @return An AuthCtx based on the connection_id + * @param sender The verified caller Identity supplied by the host + * @return An AuthCtx with captured invocation authority and lazy JWT loading */ static AuthCtx from_connection_id_opt(std::optional connection_id, Identity sender); @@ -63,11 +66,10 @@ class AuthCtx { * This is primarily used for testing purposes, allowing you to create * an AuthCtx with specific JWT claims without needing a real connection. * - * Note: The Identity must be computed by calling the host function, - * as we cannot compute Blake3 hashes in WASM. + * The Identity must be the verified sender supplied by the host. * * @param jwt_payload The raw JWT payload (JSON claims) - * @param identity The identity derived from the JWT's issuer and subject + * @param identity The verified sender Identity * @return An AuthCtx with the provided JWT */ static AuthCtx from_jwt_payload(std::string jwt_payload, Identity identity); @@ -76,11 +78,10 @@ class AuthCtx { * @brief Creates an AuthCtx that reads the JWT for the given connection ID. * * The JWT will be lazily loaded from the host when first accessed. - * The identity parameter is the sender's identity, already derived from - * JWT claims by the host (using Blake3 hashing). + * The identity parameter is the verified sender supplied by the host. * * @param connection_id The connection ID to load the JWT for - * @param sender The identity of the caller (already derived from JWT claims by the host) + * @param sender The verified sender Identity supplied by the host * @return An AuthCtx that will load the JWT on demand */ static AuthCtx from_connection_id(ConnectionId connection_id, Identity sender); @@ -93,9 +94,9 @@ class AuthCtx { bool is_internal() const { return is_internal_; } /** - * @brief Checks if there is a JWT without loading it. + * @brief Checks if there is a JWT, loading it lazily if necessary. * - * If is_internal() returns true, this will return false. + * Independent of is_internal(). Internal calls can also have a JWT. * * @return true if a JWT is available */ @@ -113,9 +114,8 @@ class AuthCtx { /** * @brief Gets the caller's identity. * - * For internal calls, this returns the database's identity. - * For external calls, this returns the identity derived from the JWT - * (based on the issuer and subject claims). + * Returns the verified sender captured when constructing the context, + * independently of JWT presence or token claims. * * @return The caller's Identity */ @@ -126,16 +126,16 @@ class AuthCtx { // INLINE IMPLEMENTATIONS // ============================================================================ -constexpr uint16_t ERROR_BUFFER_TOO_SMALL = 11; - -inline AuthCtx::AuthCtx(bool is_internal, std::function()> loader) - : is_internal_(is_internal), jwt_loader_(std::move(loader)) {} +inline AuthCtx::AuthCtx(bool is_internal, std::function()> loader, + std::optional verified_sender) + : is_internal_(is_internal), verified_sender_(std::move(verified_sender)), jwt_loader_(std::move(loader)) {} inline AuthCtx AuthCtx::from_connection_id_opt(std::optional connection_id, Identity sender) { + const auto flags = FFI::get_call_auth_flags(); if (connection_id.has_value()) { - return from_connection_id(*connection_id, std::move(sender)); + return from_connection_with_flags(*connection_id, std::move(sender), flags); } else { - return internal(); + return AuthCtx((flags & 1) != 0, []() -> std::optional { return std::nullopt; }, sender); } } @@ -144,13 +144,17 @@ inline AuthCtx AuthCtx::internal() { } inline AuthCtx AuthCtx::from_jwt_payload(std::string jwt_payload, Identity identity) { - return AuthCtx(false, [payload = std::move(jwt_payload), id = std::move(identity)]() mutable -> std::optional { + return AuthCtx(false, [payload = std::move(jwt_payload), id = identity]() mutable -> std::optional { return JwtClaims(std::move(payload), std::move(id)); - }); + }, identity); } inline AuthCtx AuthCtx::from_connection_id(ConnectionId connection_id, Identity sender) { - return AuthCtx(false, [connection_id, sender]() -> std::optional { + return from_connection_with_flags(connection_id, std::move(sender), FFI::get_call_auth_flags()); +} + +inline AuthCtx AuthCtx::from_connection_with_flags(ConnectionId connection_id, Identity sender, uint32_t flags) { + return AuthCtx((flags & 1) != 0, [connection_id, sender]() -> std::optional { // Call the host FFI to get the JWT BytesSource jwt_source; @@ -169,35 +173,24 @@ inline AuthCtx AuthCtx::from_connection_id(ConnectionId connection_id, Identity } // Read the JWT payload from the BytesSource - std::vector buffer; - buffer.resize(4096); // Start with 4KB buffer - - size_t buffer_len = buffer.size(); - int16_t result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); - - while (result == ERROR_BUFFER_TOO_SMALL) { - buffer.resize(buffer.size() * 2); - buffer_len = buffer.size(); - result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); + std::array buffer; + std::string jwt_payload; + for (;;) { + size_t buffer_len = buffer.size(); + const auto result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); + if (result != 0 && result != -1) return std::nullopt; + jwt_payload.append(reinterpret_cast(buffer.data()), buffer_len); + // -1 is successful exhaustion and may include the final payload bytes. + if (result == -1) break; + if (buffer_len == 0) return std::nullopt; } - - if (result < 0) { - return std::nullopt; - } - - // Convert bytes to string - std::string jwt_payload(buffer.begin(), buffer.begin() + buffer_len); - - // Use the provided sender identity (already computed by host from JWT claims) + if (jwt_payload.empty()) return std::nullopt; + // Token claims cannot override the verified sender, including hosted tokens. return JwtClaims(std::move(jwt_payload), sender); - }); + }, sender); } inline bool AuthCtx::has_jwt() const { - if (is_internal_) { - return false; - } - // Load the JWT if not already loaded, then check if it has a value // This ensures has_jwt() and get_jwt() are consistent return get_jwt().has_value(); @@ -211,6 +204,7 @@ inline const std::optional& AuthCtx::get_jwt() const { } inline Identity AuthCtx::get_caller_identity() const { + if (verified_sender_.has_value()) return *verified_sender_; if (is_internal_) { // Return database identity for internal calls std::array identity_bytes; diff --git a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h index 5fcba095c76..a61fc3bcc59 100644 --- a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h +++ b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h @@ -124,10 +124,10 @@ namespace SpacetimeDB::bsatn { template std::optional read_optional() { uint8_t tag = read_u8(); - if (tag == 0) { - return std::nullopt; - } else if (tag == 1) { + if (tag == 0) { // Some, matching the canonical BSATN option type. return SpacetimeDB::bsatn::deserialize(*this); + } else if (tag == 1) { // None. + return std::nullopt; } else { std::abort(); // Invalid optional tag in BSATN deserialization } diff --git a/crates/bindings-cpp/include/spacetimedb/environment.h b/crates/bindings-cpp/include/spacetimedb/environment.h new file mode 100644 index 00000000000..f2df08234d0 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment.h @@ -0,0 +1,34 @@ +#ifndef SPACETIMEDB_ENVIRONMENT_H +#define SPACETIMEDB_ENVIRONMENT_H +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { +/// Read-only database environment. Reads use the current transaction, or a +/// short snapshot in a procedure outside a transaction. Values are not cached. +class Environment { +public: + std::optional get(std::string_view key) const { + if (key.empty() || key.size() > 256) LOG_PANIC("invalid environment variable name"); + BytesSource source{0}; + if (FFI::env_get(reinterpret_cast(key.data()), static_cast(key.size()), &source) != Status(0)) + LOG_PANIC("environment read failed"); + if (source == BytesSource{0}) return std::nullopt; + std::array buffer; + std::string value; + for (;;) { + size_t len = buffer.size(); + const auto status = FFI::bytes_source_read(source, buffer.data(), &len); + if ((status != 0 && status != -1) || len > buffer.size()) LOG_PANIC("environment source read failed"); + value.append(reinterpret_cast(buffer.data()), len); + if (status == -1) return value; + if (len == 0) LOG_PANIC("environment source made no progress"); + } + } +}; +} +#endif diff --git a/crates/bindings-cpp/include/spacetimedb/function_visibility.h b/crates/bindings-cpp/include/spacetimedb/function_visibility.h new file mode 100644 index 00000000000..9bd36e19e48 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/function_visibility.h @@ -0,0 +1,6 @@ +#pragma once + +namespace SpacetimeDB { +// Omission preserves the host default: Public ordinarily, Private when scheduled. +enum class FunctionVisibility { Public, Private, Internal }; +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h index 423276de9b4..9795b3bd2d7 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h @@ -18,5 +18,7 @@ namespace SpacetimeDB::Internal { enum class FunctionVisibility : uint8_t { Private = 0, ClientCallable = 1, + Internal = 2, + ExplicitClientCallable = 3, }; } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index 1efcad29ed5..b435119ba5c 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -28,5 +28,5 @@ namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h index f746093c574..e585d7f787a 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h @@ -13,6 +13,7 @@ #include #include "../bsatn/bsatn.h" #include "../database.h" +#include "../function_visibility.h" #include "autogen/CaseConversionPolicy.g.h" #include "autogen/ExplicitNameEntry.g.h" #include "autogen/NameMapping.g.h" @@ -47,6 +48,7 @@ void fail_reducer(std::string message); namespace Internal { +// Builds the V10 module definition with explicit function visibility. class V10Builder { public: V10Builder() = default; @@ -435,7 +437,7 @@ class V10Builder { RawReducerDefV10 reducer_def{ reducer_name, ProductType{}, - FunctionVisibility::Private, + FunctionVisibility::Internal, MakeUnitAlgebraicType(), MakeStringAlgebraicType(), }; @@ -623,6 +625,7 @@ class V10Builder { void RegisterExplicitTableName(const std::string& source_name, const std::string& canonical_name); void RegisterExplicitFunctionName(const std::string& source_name, const std::string& canonical_name); + void SetFunctionVisibility(const std::string& source_name, ::SpacetimeDB::FunctionVisibility visibility); void RegisterExplicitIndexName(const std::string& source_name, const std::string& canonical_name); RawModuleDefV10 BuildModuleDef() const; diff --git a/crates/bindings-cpp/include/spacetimedb/jwt_claims.h b/crates/bindings-cpp/include/spacetimedb/jwt_claims.h index cdc9aef511d..6a72e973633 100644 --- a/crates/bindings-cpp/include/spacetimedb/jwt_claims.h +++ b/crates/bindings-cpp/include/spacetimedb/jwt_claims.h @@ -15,8 +15,8 @@ namespace SpacetimeDB { * This class provides lazy parsing of JWT claims, parsing specific fields * on demand. It follows the same pattern as the Rust and C# implementations. * - * The Identity is provided in the constructor because computing it requires - * Blake3 hashing, which is done on the host side. + * The Identity is the verified sender supplied by the host. Token claims + * cannot override it, including for hosted container credentials. */ class JwtClaims { private: @@ -36,11 +36,10 @@ class JwtClaims { /** * @brief Constructs a JwtClaims from a JWT payload and its associated Identity. * - * The Identity must be provided because computing it requires Blake3 hashing, - * which is performed on the host side. + * The Identity must be the verified sender supplied by the host. * * @param jwt_payload The raw JWT payload (JSON claims) - * @param identity The identity derived from the JWT's issuer and subject + * @param identity The verified sender Identity */ JwtClaims(std::string jwt_payload, Identity identity); @@ -71,8 +70,7 @@ class JwtClaims { /** * @brief Returns the identity for these credentials. * - * The identity is based on the 'iss' and 'sub' claims and is computed - * using Blake3 hashing on the host side. + * This is the verified sender supplied by the host, independently of claims. * * @return The identity */ diff --git a/crates/bindings-cpp/include/spacetimedb/macros.h b/crates/bindings-cpp/include/spacetimedb/macros.h index b0ca63b5336..2d7a23b8317 100644 --- a/crates/bindings-cpp/include/spacetimedb/macros.h +++ b/crates/bindings-cpp/include/spacetimedb/macros.h @@ -542,6 +542,15 @@ inline std::vector parseParameterNames(const std::string& param_lis // VISIBILITY FILTER MACRO // ============================================================================= +// Apply to a registered reducer or procedure. Runs after function registration; +// lifecycle reducers only accept Internal. Scheduling preserves this choice. +#define SPACETIMEDB_FUNCTION_VISIBILITY(function_name, visibility) \ + extern "C" __attribute__((export_name("__preinit__40_visibility_" #function_name))) \ + void CONCAT(__spacetimedb_function_visibility_, function_name)() { \ + ::SpacetimeDB::Internal::getV10Builder().SetFunctionVisibility( \ + #function_name, ::SpacetimeDB::FunctionVisibility::visibility); \ + } + /** * @brief Set module case conversion policy using a fixed preinit registration symbol. * @@ -782,4 +791,3 @@ inline std::vector parseParameterNames(const std::string& param_lis #endif // SPACETIMEDB_MACROS_H - diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index ea189f8ef12..b6270deb5af 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -17,6 +17,8 @@ #include #include +#include + namespace SpacetimeDB { /** @@ -57,8 +59,10 @@ struct ProcedureContext { private: // Caller's identity - who invoked this procedure Identity sender_; + AuthCtx sender_auth_ = AuthCtx::internal(); public: + Environment env; // Timestamp when the procedure was invoked Timestamp timestamp; @@ -84,7 +88,11 @@ struct ProcedureContext { ProcedureContext() = default; ProcedureContext(Identity s, Timestamp t, ConnectionId conn_id) - : sender_(s), timestamp(t), connection_id(conn_id) {} + : sender_(s), sender_auth_(AuthCtx::from_connection_id_opt( + conn_id.id.low == 0 && conn_id.id.high == 0 ? std::nullopt : std::optional(conn_id), s)), + timestamp(t), connection_id(conn_id) {} + + const AuthCtx& sender_auth() const { return sender_auth_; } Identity sender() const { return sender_; @@ -100,7 +108,7 @@ struct ProcedureContext { * @code * auto module_id = ctx.database_identity(); * std::string url = "http://localhost:3000/v1/database/" + - * module_id.to_hex() + "/schema?version=9"; + * module_id.to_hex_string() + "/schema?version=10"; * @endcode */ Identity database_identity() const { @@ -200,8 +208,9 @@ struct ProcedureContext { auto make_reducer_ctx = [this](Timestamp tx_timestamp) { return ReducerContext( sender(), - std::optional(connection_id), - tx_timestamp + connection_id.id.low == 0 && connection_id.id.high == 0 ? std::nullopt : std::optional(connection_id), + tx_timestamp, + sender_auth_ ); }; return Internal::with_tx(make_reducer_ctx, body); @@ -232,8 +241,9 @@ struct ProcedureContext { auto make_reducer_ctx = [this](Timestamp tx_timestamp) { return ReducerContext( sender(), - std::optional(connection_id), - tx_timestamp + connection_id.id.low == 0 && connection_id.id.high == 0 ? std::nullopt : std::optional(connection_id), + tx_timestamp, + sender_auth_ ); }; return Internal::try_with_tx(make_reducer_ctx, body); diff --git a/crates/bindings-cpp/include/spacetimedb/reducer_context.h b/crates/bindings-cpp/include/spacetimedb/reducer_context.h index 41865b14f3a..14a6028c606 100644 --- a/crates/bindings-cpp/include/spacetimedb/reducer_context.h +++ b/crates/bindings-cpp/include/spacetimedb/reducer_context.h @@ -13,6 +13,8 @@ // Include database for DatabaseContext #include +#include + namespace SpacetimeDB { // Enhanced ReducerContext with database access - matches Rust pattern @@ -21,6 +23,7 @@ struct ReducerContext { Identity sender_; public: + Environment env; // Core fields - sender is exposed via sender() like Rust, other fields remain directly accessible std::optional connection_id; Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/tx_context.h b/crates/bindings-cpp/include/spacetimedb/tx_context.h index 1a04ef027e1..c874a22b4ff 100644 --- a/crates/bindings-cpp/include/spacetimedb/tx_context.h +++ b/crates/bindings-cpp/include/spacetimedb/tx_context.h @@ -56,6 +56,7 @@ struct TxContext { // In C++, we explicitly expose references where possible and provide // accessors for fields exposed as methods on ReducerContext. DatabaseContext& db; + const Environment& env; const Timestamp& timestamp; const std::optional& connection_id; @@ -63,6 +64,7 @@ struct TxContext { explicit TxContext(ReducerContext& ctx) : ctx_(ctx), db(ctx.db), + env(ctx.env), timestamp(ctx.timestamp), connection_id(ctx.connection_id) {} diff --git a/crates/bindings-cpp/include/spacetimedb/view_context.h b/crates/bindings-cpp/include/spacetimedb/view_context.h index 3a7fdd577f0..12898664059 100644 --- a/crates/bindings-cpp/include/spacetimedb/view_context.h +++ b/crates/bindings-cpp/include/spacetimedb/view_context.h @@ -6,6 +6,8 @@ #include // For ReadOnlyDatabaseContext #include +#include + namespace SpacetimeDB { /** @@ -40,6 +42,7 @@ struct ViewContext { public: // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; // Constructors ViewContext() = default; @@ -74,6 +77,7 @@ struct ViewContext { struct AnonymousViewContext { // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; // Constructors AnonymousViewContext() = default; diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index eb22114e8b9..d7eadd210ae 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -209,6 +209,31 @@ RawConstraintDefV10 V10Builder::CreateUniqueConstraint(const std::string& table_ }; } +void V10Builder::SetFunctionVisibility(const std::string& name, ::SpacetimeDB::FunctionVisibility visibility) { + FunctionVisibility declared; + switch (visibility) { + case ::SpacetimeDB::FunctionVisibility::Public: declared = FunctionVisibility::ExplicitClientCallable; break; + case ::SpacetimeDB::FunctionVisibility::Private: declared = FunctionVisibility::Private; break; + case ::SpacetimeDB::FunctionVisibility::Internal: declared = FunctionVisibility::Internal; break; + default: + SetConstraintRegistrationError("INVALID_FUNCTION_VISIBILITY", "function='" + name + "'"); + return; + } + for (const auto& lifecycle : lifecycle_reducers_) { + if (lifecycle.function_name == name && declared != FunctionVisibility::Internal) { + SetConstraintRegistrationError("INVALID_LIFECYCLE_VISIBILITY", "function='" + name + "' must be Internal"); + return; + } + } + for (auto& reducer : reducers_) { + if (reducer.source_name == name) { reducer.visibility = declared; return; } + } + for (auto& procedure : procedures_) { + if (procedure.source_name == name) { procedure.visibility = declared; return; } + } + SetConstraintRegistrationError("UNKNOWN_FUNCTION_VISIBILITY", "function='" + name + "' is not a reducer or procedure"); +} + RawModuleDefV10 V10Builder::BuildModuleDef() const { RawModuleDefV10 v10_module; @@ -217,27 +242,12 @@ RawModuleDefV10 V10Builder::BuildModuleDef() const { std::vector reducers = reducers_; std::vector procedures = procedures_; - std::unordered_set internal_functions; - for (const auto& lifecycle : lifecycle_reducers_) { - internal_functions.insert(lifecycle.function_name); - } - for (const auto& schedule : schedules_) { - internal_functions.insert(schedule.function_name); - } - for (auto& reducer : reducers) { - if (internal_functions.find(reducer.source_name) != internal_functions.end()) { - reducer.visibility = FunctionVisibility::Private; - } - } - for (auto& procedure : procedures) { - if (internal_functions.find(procedure.source_name) != internal_functions.end()) { - procedure.visibility = FunctionVisibility::Private; - } - } - RawModuleDefV10Section section_typespace; section_typespace.set<0>(typespace_); v10_module.sections.push_back(section_typespace); + RawModuleDefV10Section capabilities; + capabilities.set<13>(std::vector{"hosted_auth_v1"}); + v10_module.sections.push_back(std::move(capabilities)); if (!types.empty()) { RawModuleDefV10Section section_types; diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt index 0ced4e0194c..2a540f96f31 100644 --- a/crates/bindings-cpp/tests/unit/CMakeLists.txt +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -11,6 +11,18 @@ endif() add_executable(bindings_cpp_unit_tests main.cpp http_unit_tests.cpp + hosted_auth_unit_tests.cpp + function_visibility_unit_tests.cpp +) + +# Exercise the real module builder without the standalone WASI shims, which +# replace the Node test runner's standard I/O and process lifecycle functions. +target_sources(bindings_cpp_unit_tests PRIVATE + ../../src/internal/Module.cpp + ../../src/internal/AlgebraicType.cpp + ../../src/internal/v9_builder.cpp + ../../src/internal/v10_builder.cpp + ../../src/internal/module_type_registration.cpp ) target_include_directories(bindings_cpp_unit_tests PRIVATE diff --git a/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp new file mode 100644 index 00000000000..23cff3a0482 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp @@ -0,0 +1,98 @@ +#include "test_harness.h" +#include "spacetimedb/reducer_error.h" +#include "spacetimedb/procedure_context.h" +#include "spacetimedb/internal/v10_builder.h" +#include "spacetimedb/internal/autogen/RawModuleDef.g.h" +#include "spacetimedb/macros.h" + +using namespace SpacetimeDB; +using namespace SpacetimeDB::Internal; + +namespace { +ReducerResult noop(ReducerContext) { return Ok(); } +uint32_t procedure(ProcedureContext) { return 7; } +} + +SPACETIMEDB_FUNCTION_VISIBILITY(visibility_macro_target, Internal); + +TEST_CASE(visibility_macro_applies_after_function_registration) { + auto& builder = getV10Builder(); + builder.RegisterReducer("visibility_macro_target", &noop, {}); + __spacetimedb_function_visibility_visibility_macro_target(); + bool found = false; + for (const auto& section : builder.BuildModuleDef().sections) { + if (section.get_tag() != 3) continue; + for (const auto& reducer : section.get<3>()) { + if (reducer.source_name != "visibility_macro_target") continue; + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, reducer.visibility); + found = true; + } + } + ASSERT_TRUE(found); +} + +TEST_CASE(v10_retains_explicit_visibility_and_schedule_default) { + V10Builder builder; + builder.RegisterReducer("omitted", &noop, {}); + builder.RegisterReducer("public", &noop, {}); + builder.RegisterReducer("private", &noop, {}); + builder.RegisterReducer("internal", &noop, {}); + builder.SetFunctionVisibility("public", SpacetimeDB::FunctionVisibility::Public); + builder.SetFunctionVisibility("private", SpacetimeDB::FunctionVisibility::Private); + builder.SetFunctionVisibility("internal", SpacetimeDB::FunctionVisibility::Internal); + builder.RegisterSchedule("jobs", 0, "public"); + builder.RegisterSchedule("other_jobs", 0, "omitted"); + builder.RegisterProcedure("procedure", &procedure); + builder.SetFunctionVisibility("procedure", SpacetimeDB::FunctionVisibility::Internal); + + RawModuleDef versioned; + versioned.set<2>(builder.BuildModuleDef()); + std::vector bytes; + bsatn::Writer writer(bytes); + bsatn::serialize(writer, versioned); + ASSERT_EQ(uint8_t{2}, bytes.at(0)); + ASSERT_EQ(uint8_t{2}, versioned.get_tag()); + bool saw_reducers = false, saw_procedure = false, saw_capability = false; + for (const auto& section : versioned.get<2>().sections) { + if (section.get_tag() == 3) { + const auto& reducers = section.get<3>(); + ASSERT_EQ(size_t{4}, reducers.size()); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::ClientCallable, reducers[0].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::ExplicitClientCallable, reducers[1].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Private, reducers[2].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, reducers[3].visibility); + saw_reducers = true; + } else if (section.get_tag() == 4) { + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, section.get<4>().at(0).visibility); + saw_procedure = true; + } else if (section.get_tag() == 13) { + ASSERT_EQ(std::vector{"hosted_auth_v1"}, section.get<13>()); + saw_capability = true; + } + } + ASSERT_TRUE(saw_reducers && saw_procedure && saw_capability); +} + +TEST_CASE(v10_visibility_extends_enum_without_changing_reducer_field_layout) { + V10Builder builder; + builder.RegisterReducer("r", &noop, {}); + auto reducer = builder.GetReducers().at(0); + for (uint8_t tag = 0; tag <= 3; ++tag) { + reducer.visibility = static_cast(tag); + std::vector bytes; + bsatn::Writer writer(bytes); + bsatn::serialize(writer, reducer); + const std::vector expected{1, 0, 0, 0, 'r', 0, 0, 0, 0, tag, 2, 0, 0, 0, 0, 4}; + ASSERT_EQ(expected, bytes); + const RawProcedureDefV10 procedure_def{ + "p", ProductType{}, reducer.ok_return_type, reducer.visibility, + }; + std::vector procedure_bytes; + bsatn::Writer procedure_writer(procedure_bytes); + bsatn::serialize(procedure_writer, procedure_def); + const std::vector expected_procedure{ + 1, 0, 0, 0, 'p', 0, 0, 0, 0, 2, 0, 0, 0, 0, tag, + }; + ASSERT_EQ(expected_procedure, procedure_bytes); + } +} diff --git a/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp b/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp new file mode 100644 index 00000000000..cfa432bf30b --- /dev/null +++ b/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp @@ -0,0 +1,137 @@ +#include "test_harness.h" +#include "spacetimedb/procedure_context.h" + +#include +#include + +using namespace SpacetimeDB; + +namespace { +uint32_t auth_flags; +size_t flag_reads; +size_t jwt_reads; +size_t payload_offset; +std::string jwt_payload; + +Identity verified_sender() { + std::array bytes{}; + bytes[0] = 42; + return Identity(bytes); +} + +void reset_host(uint32_t flags, std::string payload = {}) { + auth_flags = flags; + flag_reads = jwt_reads = payload_offset = 0; + jwt_payload = std::move(payload); +} +} + +extern "C" uint32_t get_call_auth_flags() { + ++flag_reads; + return auth_flags; +} + +extern "C" Status get_jwt(const uint8_t*, BytesSource* out) { + ++jwt_reads; + payload_offset = 0; + *out = BytesSource{jwt_payload.empty() ? 0u : 1u}; + return Status{0}; +} + +extern "C" Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out) { + payload_offset = 0; + const std::string name(reinterpret_cast(key), key_len); + if (name == "ERROR") return Status{1}; + *out = BytesSource{name == "MISSING" ? 0u : 1u}; + return Status{0}; +} + +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* len) { + *len = std::min(*len, jwt_payload.size() - payload_offset); + std::memcpy(out, jwt_payload.data() + payload_offset, *len); + payload_offset += *len; + // Successful exhaustion can return the last bytes together with -1. + return payload_offset == jwt_payload.size() ? -1 : 0; +} + +extern "C" void identity(uint8_t* out) { std::memset(out, 0, 32); } +extern "C" Status procedure_start_mut_tx(int64_t* out) { *out = 0; return Status{0}; } +extern "C" Status procedure_commit_mut_tx() { return Status{0}; } +extern "C" Status procedure_abort_mut_tx() { return Status{0}; } +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, + uint32_t, const uint8_t*, size_t) {} + +TEST_CASE(authority_without_connection_is_captured_from_host) { + for (uint32_t flags : {0u, 1u}) { + reset_host(flags); + auto ctx = AuthCtx::from_connection_id_opt(std::nullopt, verified_sender()); + auth_flags = flags ^ 1; + ASSERT_EQ(size_t{1}, flag_reads); + ASSERT_EQ(flags == 1, ctx.is_internal()); + ASSERT_TRUE(!ctx.has_jwt()); + ASSERT_EQ(verified_sender(), ctx.get_caller_identity()); + ASSERT_EQ(size_t{0}, jwt_reads); + } +} + +TEST_CASE(internal_call_retains_lazy_jwt_and_verified_identity) { + reset_host(1, R"({"iss":"other","sub":"other","identity":"untrusted"})"); + auto ctx = AuthCtx::from_connection_id(ConnectionId(5), verified_sender()); + auth_flags = 0; + ASSERT_TRUE(ctx.is_internal()); + ASSERT_EQ(size_t{0}, jwt_reads); + ASSERT_TRUE(ctx.has_jwt()); + ASSERT_EQ(verified_sender(), ctx.get_jwt()->get_identity()); + ASSERT_EQ(std::string("other"), ctx.get_jwt()->subject()); + ASSERT_EQ(size_t{1}, jwt_reads); +} + +TEST_CASE(jwt_source_reads_all_chunks_including_final_exhausted_bytes) { + reset_host(0, "{\"padding\":\"" + std::string(8192, 'x') + "\",\"sub\":\"last\"}"); + auto ctx = AuthCtx::from_connection_id(ConnectionId(5), verified_sender()); + ASSERT_TRUE(ctx.has_jwt()); + ASSERT_EQ(std::string("last"), ctx.get_jwt()->subject()); + ASSERT_EQ(jwt_payload.size(), payload_offset); + ASSERT_EQ(size_t{1}, jwt_reads); +} + +TEST_CASE(procedure_transactions_preserve_authority_connection_and_sender) { + for (uint64_t connection : {0u, 5u}) { + reset_host(1, R"({"sub":"worker"})"); + ProcedureContext ctx(verified_sender(), Timestamp::from_micros_since_epoch(0), ConnectionId(connection)); + auth_flags = 0; + ctx.with_tx([&](TxContext& tx) { + ASSERT_TRUE(tx.sender_auth().is_internal()); + ASSERT_EQ(verified_sender(), tx.sender()); + ASSERT_EQ(connection != 0, tx.connection_id.has_value()); + ASSERT_EQ(connection != 0, tx.sender_auth().has_jwt()); + if (connection) ASSERT_EQ(verified_sender(), tx.sender_auth().get_jwt()->get_identity()); + }); + ASSERT_EQ(size_t{1}, flag_reads); + } +} + +TEST_CASE(environment_preserves_missing_empty_and_all_chunks_without_caching) { + Environment env; + reset_host(0); + ASSERT_TRUE(!env.get("MISSING").has_value()); + ASSERT_EQ(std::string{}, env.get("EMPTY").value()); + jwt_payload = std::string(8192, 'x'); + ASSERT_EQ(jwt_payload, env.get("LARGE").value()); + jwt_payload = std::string("a\0b", 3); + ASSERT_EQ(jwt_payload, env.get("NUL").value()); + jwt_payload = "updated"; + ASSERT_EQ(jwt_payload, env.get("NUL").value()); + ProcedureContext procedure(verified_sender(), Timestamp::from_micros_since_epoch(0), ConnectionId(0)); + ASSERT_EQ(jwt_payload, procedure.env.get("VALUE").value()); + procedure.with_tx([&](TxContext& tx) { ASSERT_EQ(jwt_payload, tx.env.get("VALUE").value()); }); +} + +TEST_CASE(optional_reader_matches_canonical_bsatn_tags_and_preserves_following_bytes) { + const std::vector bytes{1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 'a', 0, 'b', 42}; + bsatn::Reader reader(bytes.data(), bytes.size()); + ASSERT_TRUE(!bsatn::deserialize>(reader).has_value()); + ASSERT_EQ(std::string{}, bsatn::deserialize>(reader).value()); + ASSERT_EQ(std::string("a\0b", 3), bsatn::deserialize>(reader).value()); + ASSERT_EQ(uint8_t{42}, reader.read_u8()); +} diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index f3933229c16..849f40b0b43 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -347,6 +347,65 @@ public static void @params(ProcedureContext ctx) Assert.Empty(GetCompilationErrors(compilationAfterGen)); } + [Fact] + public static async Task ExplicitFunctionVisibilityCompilesAndRejectsExternalLifecycle() + { + var fixture = await Fixture.Compile("server"); + const string source = """ + using SpacetimeDB; + public static partial class VisibilityFunctions + { + [Reducer(Visibility = FunctionVisibility.Public)] + public static void PublicJob(ReducerContext ctx) {} + [Reducer(Visibility = FunctionVisibility.Private)] + public static void PrivateJob(ReducerContext ctx) {} + [Reducer(Visibility = FunctionVisibility.Internal)] + public static void InternalJob(ReducerContext ctx) {} + [Procedure(Visibility = FunctionVisibility.Internal)] + public static int InternalProcedure(ProcedureContext ctx) => 1; + } + """; + var parseOptions = new CSharpParseOptions(fixture.SampleCompilation.LanguageVersion); + var tree = CSharpSyntaxTree.ParseText(source, parseOptions); + var compilation = fixture.SampleCompilation.AddSyntaxTrees(tree); + var driver = CSharpGeneratorDriver.Create( + [ + new SpacetimeDB.Codegen.Type().AsSourceGenerator(), + new SpacetimeDB.Codegen.Module().AsSourceGenerator(), + ], + parseOptions: parseOptions + ); + var result = driver.RunGenerators(compilation).GetRunResult(); + Assert.Empty(result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error)); + Assert.Empty(GetCompilationErrors(compilation.AddSyntaxTrees(result.GeneratedTrees))); + var generated = string.Join("\n", result.GeneratedTrees.Select(t => t.ToString())); + Assert.Contains( + "Visibility: SpacetimeDB.Internal.FunctionVisibility.ExplicitClientCallable", + generated + ); + Assert.Contains("Visibility: SpacetimeDB.Internal.FunctionVisibility.Private", generated); + Assert.Contains("Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal", generated); + + var invalid = CSharpSyntaxTree.ParseText( + """ + using SpacetimeDB; + public static partial class BadVisibility + { + [Reducer(ReducerKind.Init, Visibility = FunctionVisibility.Public)] + public static void InvalidLifecycle(ReducerContext ctx) {} + } + """, + parseOptions + ); + var rejected = driver + .RunGenerators(fixture.SampleCompilation.AddSyntaxTrees(invalid)) + .GetRunResult(); + Assert.Contains( + rejected.Diagnostics, + diagnostic => diagnostic.GetMessage().Contains("Lifecycle reducers only permit") + ); + } + [Fact] public static async Task TestDiagnostics() { diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 2504cc013bf..9762f360c78 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -649,6 +649,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -896,6 +897,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -909,6 +911,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) @@ -3097,7 +3100,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(TestDuplicateReducerKind1), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3118,7 +3121,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(TestDuplicateReducerKind2), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index b46de3189b4..8773f431bde 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -51,6 +51,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -279,6 +280,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -292,6 +294,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index edc7d5f2af4..8c0072cae5a 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -493,6 +493,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -730,6 +731,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -743,6 +745,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) @@ -2334,7 +2337,7 @@ SpacetimeDB.BSATN.ITypeRegistrar registrar new( SourceName: nameof(Init), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen/Diag.cs b/crates/bindings-csharp/Codegen/Diag.cs index 928b7a71635..fb70f672f85 100644 --- a/crates/bindings-csharp/Codegen/Diag.cs +++ b/crates/bindings-csharp/Codegen/Diag.cs @@ -333,4 +333,13 @@ string prefix $"HTTP handler method {method.Identifier} must be non-generic and take exactly two parameters.", method => method.ParameterList ); + + public static readonly ErrorDescriptor InvalidFunctionVisibility = + new( + group, + "Invalid function visibility", + _ => + $"Visibility must be Default, Public, Private, or Internal. Lifecycle reducers only permit Default or Internal.", + method => method.Identifier + ); } diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index 870cadce093..c6268371d59 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -1405,13 +1405,46 @@ public byte[] Invoke( } /// -/// Represents a reducer method declaration in a module. +/// Validates a declared function visibility and maps it to the V10 schema. /// +static class FunctionVisibilityDeclaration +{ + internal static string Resolve( + FunctionVisibility visibility, + bool lifecycle, + MethodDeclarationSyntax method, + DiagReporter diag + ) + { + if ( + ( + lifecycle + && visibility is not (FunctionVisibility.Default or FunctionVisibility.Internal) + ) || !Enum.IsDefined(typeof(FunctionVisibility), visibility) + ) + { + diag.Report(ErrorDescriptor.InvalidFunctionVisibility, method); + return "SpacetimeDB.Internal.FunctionVisibility.Internal"; + } + return visibility switch + { + FunctionVisibility.Public => + "SpacetimeDB.Internal.FunctionVisibility.ExplicitClientCallable", + FunctionVisibility.Private => "SpacetimeDB.Internal.FunctionVisibility.Private", + FunctionVisibility.Internal => "SpacetimeDB.Internal.FunctionVisibility.Internal", + _ => lifecycle + ? "SpacetimeDB.Internal.FunctionVisibility.Internal" + : "SpacetimeDB.Internal.FunctionVisibility.ClientCallable", + }; + } +} + record ReducerDeclaration { public readonly string Name; public readonly string? CanonicalName; public readonly ReducerKind Kind; + public readonly string Visibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1450,6 +1483,12 @@ public ReducerDeclaration(GeneratorAttributeSyntaxContext context, DiagReporter } Kind = attr.Kind; + Visibility = FunctionVisibilityDeclaration.Resolve( + attr.Visibility, + Kind != ReducerKind.UserDefined, + methodSyntax, + diag + ); CanonicalName = attr.Name; FullName = SymbolToName(method); Args = new( @@ -1478,7 +1517,7 @@ class {{Identifier}}: SpacetimeDB.Internal.IReducer { public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new ( SourceName: nameof({{Identifier}}), Params: [{{MemberDeclaration.GenerateDefs(Args)}}], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + Visibility: {{Visibility}}, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -1535,6 +1574,7 @@ record ProcedureDeclaration { public readonly string Name; public readonly string? CanonicalName; + public readonly string Visibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1551,6 +1591,12 @@ public ProcedureDeclaration(GeneratorAttributeSyntaxContext context, DiagReporte var methodSyntax = (MethodDeclarationSyntax)context.TargetNode; var method = (IMethodSymbol)context.TargetSymbol; var attr = context.Attributes.Single().ParseAs(); + Visibility = FunctionVisibilityDeclaration.Resolve( + attr.Visibility, + false, + methodSyntax, + diag + ); if ( method.Parameters.FirstOrDefault()?.Type @@ -1710,7 +1756,7 @@ class {{{Identifier}}} : SpacetimeDB.Internal.IProcedure { SourceName: nameof({{{Identifier}}}), Params: [{{{MemberDeclaration.GenerateDefs(Args)}}}], ReturnType: {{{returnTypeExpr}}}, - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable + Visibility: {{{Visibility}}} ); public byte[] Invoke(BinaryReader reader, SpacetimeDB.Internal.IProcedureContext ctx) { @@ -2463,6 +2509,7 @@ public static class Handlers { ))}} } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -2665,6 +2712,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -2676,6 +2724,7 @@ internal ViewContext(Identity sender, Internal.LocalReadOnly db) public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/README.md b/crates/bindings-csharp/README.md index 289bd570ff0..94158b66fb5 100644 --- a/crates/bindings-csharp/README.md +++ b/crates/bindings-csharp/README.md @@ -6,6 +6,27 @@ See the [C# module library reference](https://spacetimedb.com/docs/modules/c-sha ## Internal documentation +### Function visibility and invocation authentication + +Reducers and procedures can declare `Visibility = FunctionVisibility.Public`, +`Private`, or `Internal` in their attributes. Omission (`Default`) means public +for ordinary functions and private for scheduled functions. An explicit choice +is preserved when the function is scheduled. Lifecycle reducers permit only +omission or `Internal` and can only run for their host lifecycle event. + +Internal functions require verified internal authority. Private functions also +admit the owner, and public functions admit any client. For example: + +```csharp +[Reducer(Visibility = FunctionVisibility.Internal)] +public static void ProcessJobs(ReducerContext ctx) { } +``` + +`ctx.SenderAuth.IsInternal` comes from the host's invocation authority. It is +independent of connection and JWT presence, so an internal call can have a JWT. +JWT identity is the verified sender supplied by the host. Newly compiled modules +emit schema V10 and advertise `hosted_auth_v1`, requiring a compatible host. + These projects contain the SpacetimeDB SATS typesystem, codegen and runtime bindings for SpacetimeDB WebAssembly modules. It also contains serialization code for SpacetimeDB C# clients. @@ -19,4 +40,3 @@ The [`Codegen`](./Codegen/) and [`Runtime`](./Runtime/) libraries are used: - only by C# Modules. They provide all of the functionality needed to write SpacetimeDB modules in C#. See their READMEs for more information. - diff --git a/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs new file mode 100644 index 00000000000..d50e6cff7f4 --- /dev/null +++ b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs @@ -0,0 +1,68 @@ +namespace Runtime.Tests; + +using SpacetimeDB.BSATN; +using SpacetimeDB.Internal; + +public class FunctionVisibilityTests +{ + [Theory] + [InlineData(FunctionVisibility.Private, 0)] + [InlineData(FunctionVisibility.ClientCallable, 1)] + [InlineData(FunctionVisibility.Internal, 2)] + [InlineData(FunctionVisibility.ExplicitClientCallable, 3)] + public void V10RetainsVisibilityEnumEncoding(FunctionVisibility visibility, byte tag) + { + var bytes = IStructuralReadWrite.ToBytes( + new SpacetimeDB.BSATN.Enum(), + visibility + ); + Assert.Equal(new byte[] { tag }, bytes); + } + + [Theory] + [InlineData(FunctionVisibility.ExplicitClientCallable)] + [InlineData(FunctionVisibility.ClientCallable)] + [InlineData(FunctionVisibility.Private)] + [InlineData(FunctionVisibility.Internal)] + public void SchedulingPreservesVisibility(FunctionVisibility visibility) + { + var module = new RawModuleDefV10(); + var reducer = new RawReducerDefV10( + "run_job", + [], + visibility, + AlgebraicType.Unit, + new AlgebraicType.String(default) + ); + module.RegisterReducer(reducer, null); + module.RegisterTable( + new RawTableDefV10 { SourceName = "jobs" }, + new RawScheduleDefV10(null, "jobs", 0, "run_job") + ); + var raw = module.BuildModuleDefinition(); + var reducers = Assert.Single(raw.Sections.OfType()); + Assert.Equal(visibility, Assert.Single(reducers.Reducers_).Visibility); + var capabilities = Assert.Single( + raw.Sections.OfType() + ); + Assert.Contains("hosted_auth_v1", capabilities.Capabilities_); + } + + [Theory] + [InlineData(FunctionVisibility.ClientCallable)] + [InlineData(FunctionVisibility.ExplicitClientCallable)] + public void LifecycleRejectsExternalVisibility(FunctionVisibility visibility) + { + var module = new RawModuleDefV10(); + var reducer = new RawReducerDefV10( + "initialize", + [], + visibility, + AlgebraicType.Unit, + new AlgebraicType.String(default) + ); + Assert.Throws( + () => module.RegisterReducer(reducer, Lifecycle.Init) + ); + } +} diff --git a/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs b/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs new file mode 100644 index 00000000000..ada932fb631 --- /dev/null +++ b/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs @@ -0,0 +1,43 @@ +namespace Runtime.Tests; + +using SpacetimeDB; + +public class HostedAuthTests +{ + [Theory] + [InlineData(0u, false)] + [InlineData(1u, true)] + public void NoJwtCallsPreserveVerifiedInternalFlag(uint flags, bool expectedInternal) + { + var auth = AuthCtx.FromVerifiedCall(flags, () => null); + Assert.Equal(expectedInternal, auth.IsInternal); + Assert.False(auth.HasJwt); + Assert.Null(auth.Jwt); + } + + [Fact] + public void InternalCallCanRetainJwtAndVerifiedSenderIdentity() + { + var sender = Identity.FromHexString(new string('a', 64)); + var reads = 0; + var flags = 1u; + var auth = AuthCtx.FromVerifiedCall( + flags, + () => + { + reads++; + return new JwtClaims( + "{\"iss\":\"different-issuer\",\"sub\":\"different-subject\",\"identity\":\"untrusted\"}", + sender + ); + } + ); + flags = 0; + Assert.True(auth.IsInternal); + Assert.Equal(0, reads); + Assert.True(auth.HasJwt); + Assert.Equal(sender, auth.Jwt!.Identity); + Assert.Equal("different-subject", auth.Jwt.Subject); + Assert.Equal(1, reads); + } +} diff --git a/crates/bindings-csharp/Runtime/Attrs.cs b/crates/bindings-csharp/Runtime/Attrs.cs index 46daec5eec9..d321c6a6d9e 100644 --- a/crates/bindings-csharp/Runtime/Attrs.cs +++ b/crates/bindings-csharp/Runtime/Attrs.cs @@ -190,18 +190,30 @@ public enum ReducerKind ClientDisconnected, } + /// Invocation admission for reducers and procedures. + public enum FunctionVisibility + { + /// Public for ordinary functions, Private for scheduled functions. + Default, + Public, + Private, + Internal, + } + [AttributeUsage(AttributeTargets.Method, Inherited = false)] public sealed class ReducerAttribute(ReducerKind kind = ReducerKind.UserDefined) : Attribute { public ReducerKind Kind => kind; public string? Name { get; init; } + public FunctionVisibility Visibility { get; init; } } [AttributeUsage(AttributeTargets.Method, Inherited = false)] public sealed class ProcedureAttribute() : Attribute { public string? Name { get; init; } + public FunctionVisibility Visibility { get; init; } } [AttributeUsage(AttributeTargets.Method, Inherited = false)] diff --git a/crates/bindings-csharp/Runtime/AuthCtx.cs b/crates/bindings-csharp/Runtime/AuthCtx.cs index a99ebe7d21c..db7ba86eb23 100644 --- a/crates/bindings-csharp/Runtime/AuthCtx.cs +++ b/crates/bindings-csharp/Runtime/AuthCtx.cs @@ -14,12 +14,10 @@ private AuthCtx(bool isInternal, Func jwtFactory) } /// - /// Create an AuthCtx for an internal call, with no JWT. + /// Capture verified invocation authority independently from lazy JWT loading. /// - private static AuthCtx Internal() - { - return new AuthCtx(isInternal: true, jwtFactory: () => null); - } + internal static AuthCtx FromVerifiedCall(uint callAuthFlags, Func jwtFactory) => + new(isInternal: (callAuthFlags & 1) != 0, jwtFactory); /// /// Create an AuthCtx by looking up the credentials for a connection id in system tables. @@ -29,20 +27,27 @@ private static AuthCtx Internal() /// public static AuthCtx BuildFromSystemTables(ConnectionId? connectionId, Identity identity) { + // Read synchronously while this invocation is active. Neither connection + // presence nor token claims determine internal authority. + var callAuthFlags = SpacetimeDB.Internal.FFI.get_call_auth_flags(); if (connectionId == null) { - return Internal(); + return FromVerifiedCall(callAuthFlags, () => null); } - return FromConnectionId(connectionId.Value, identity); + return FromConnectionId(connectionId.Value, identity, callAuthFlags); } /// /// Create an AuthCtx that reads JWT for a given connection ID. /// - private static AuthCtx FromConnectionId(ConnectionId connectionId, Identity identity) + private static AuthCtx FromConnectionId( + ConnectionId connectionId, + Identity identity, + uint callAuthFlags + ) { - return new AuthCtx( - isInternal: false, + return FromVerifiedCall( + callAuthFlags, jwtFactory: () => { var result = SpacetimeDB.Internal.FFI.get_jwt(ref connectionId, out var source); @@ -59,23 +64,18 @@ private static AuthCtx FromConnectionId(ConnectionId connectionId, Identity iden } /// - /// True if this reducer was spawned from inside the database. + /// True if the host verified internal authority for this invocation. /// public bool IsInternal => _isInternal; /// /// Check if there is a JWT present. - /// If IsInternal is true, this will be false. + /// Independent of IsInternal. An internal call may also have a JWT. /// public bool HasJwt { get { - if (_isInternal) - { - return false; - } - // At this point we do load the bytes. return _jwtLazy.Value != null; } diff --git a/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs new file mode 100644 index 00000000000..59ca2c26ec2 --- /dev/null +++ b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs @@ -0,0 +1,26 @@ +namespace SpacetimeDB; + +/// +/// Read-only database environment. Values are plaintext and accessible to database +/// collaborators. Procedure reads outside a transaction use a short snapshot. +/// +public readonly struct DatabaseEnvironment +{ + internal static readonly DatabaseEnvironment Instance = new(); + + /// Return null for a missing key, or an empty string for a present empty value. + public unsafe string? Get(string key) + { + ArgumentNullException.ThrowIfNull(key); + var bytes = System.Text.Encoding.UTF8.GetBytes(key); + fixed (byte* ptr = bytes) + { + Internal.FFI.env_get(ptr, checked((uint)bytes.Length), out var source); + if (source == Internal.BytesSource.INVALID) + { + return null; + } + return System.Text.Encoding.UTF8.GetString(Internal.Module.Consume(source)); + } + } +} diff --git a/crates/bindings-csharp/Runtime/HandlerContext.cs b/crates/bindings-csharp/Runtime/HandlerContext.cs index 8ad7fe14239..861cd3f803f 100644 --- a/crates/bindings-csharp/Runtime/HandlerContext.cs +++ b/crates/bindings-csharp/Runtime/HandlerContext.cs @@ -7,6 +7,7 @@ namespace SpacetimeDB; public abstract class HandlerContextBase { public Random Rng => txState.Rng; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => txState.Timestamp; // NOTE: The host rejects procedure HTTP requests while a mut transaction is open @@ -90,6 +91,7 @@ public abstract class HandlerTxContextBase(Internal.TxContext inner) : IRefresha void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => Inner.Timestamp; public Random Rng => Inner.Rng; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs index 2f9772dd591..29adc856f78 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs @@ -12,5 +12,7 @@ public enum FunctionVisibility { Private, ClientCallable, + Internal, + ExplicitClientCallable, } } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 6706fb278f0..98301786227 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -21,6 +21,7 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( SpacetimeDB.CaseConversionPolicy CaseConversionPolicy, ExplicitNames ExplicitNames, System.Collections.Generic.List HttpHandlers, - System.Collections.Generic.List HttpRoutes + System.Collections.Generic.List HttpRoutes, + System.Collections.Generic.List Capabilities )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index 261233303dc..89871c2d706 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -98,6 +98,32 @@ internal static partial class FFI #endif ; + const string StdbNamespace10_6 = +#if EXPERIMENTAL_WASM_AOT + "spacetime_10.6" +#else + "bindings" +#endif + ; + + [LibraryImport(StdbNamespace10_6)] + public static partial uint get_call_auth_flags(); + + const string StdbNamespace10_7 = +#if EXPERIMENTAL_WASM_AOT + "spacetime_10.7" +#else + "bindings" +#endif + ; + + [LibraryImport(StdbNamespace10_7)] + public static unsafe partial CheckedStatus env_get( + byte* key, + uint keyLen, + out BytesSource source + ); + [NativeMarshalling(typeof(Marshaller))] public struct CheckedStatus { diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 530cc20b402..e89de0c0172 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -55,13 +55,23 @@ internal AlgebraicType.Ref RegisterType(Func l.FunctionName) - .Concat(scheduleDefs.Select(s => s.FunctionName)) - .ToHashSet(StringComparer.Ordinal); - - foreach (var reducer in reducerDefs) - { - if (internalFunctions.Contains(reducer.SourceName)) - { - reducer.Visibility = FunctionVisibility.Private; - } - } - - foreach (var procedure in procedureDefs) - { - if (internalFunctions.Contains(procedure.SourceName)) - { - procedure.Visibility = FunctionVisibility.Private; - } - } - var sections = new List { new RawModuleDefV10Section.Typespace(typespace), + new RawModuleDefV10Section.Capabilities(["hosted_auth_v1"]), }; if (typeDefs.Count > 0) diff --git a/crates/bindings-csharp/Runtime/JwtClaims.cs b/crates/bindings-csharp/Runtime/JwtClaims.cs index 6e656cca202..554ef4d02d0 100644 --- a/crates/bindings-csharp/Runtime/JwtClaims.cs +++ b/crates/bindings-csharp/Runtime/JwtClaims.cs @@ -16,8 +16,8 @@ public sealed class JwtClaims /// /// Create a JwtClaims from a raw JWT payload (JSON claims) and its associated Identity. /// - /// This only takes an Identity because the Blake3 hash package on nuget wraps rust code. - /// We should not expose this constructor publicly, but it is needed for AuthCtx. + /// Identity is the verified sender provided by the host. Claims cannot + /// override it, including for hosted database credentials. /// internal JwtClaims(string jwt, Identity identity) { diff --git a/crates/bindings-csharp/Runtime/ProcedureContext.cs b/crates/bindings-csharp/Runtime/ProcedureContext.cs index 9c5a197aa1e..bb9fa307992 100644 --- a/crates/bindings-csharp/Runtime/ProcedureContext.cs +++ b/crates/bindings-csharp/Runtime/ProcedureContext.cs @@ -5,6 +5,7 @@ namespace SpacetimeDB; #pragma warning disable STDB_UNSTABLE public abstract class ProcedureContextBase : Internal.IInternalProcedureContext { + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public static Identity Identity => Internal.IProcedureContext.GetIdentity(); public Identity Sender { get; } public ConnectionId? ConnectionId { get; } @@ -104,6 +105,7 @@ public abstract class ProcedureTxContextBase(Internal.TxContext inner) : IRefres void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Identity Sender => Inner.Sender; public ConnectionId? ConnectionId => Inner.ConnectionId; public Timestamp Timestamp => Inner.Timestamp; diff --git a/crates/bindings-csharp/Runtime/Runtime.csproj b/crates/bindings-csharp/Runtime/Runtime.csproj index d0f71c34de0..e7d1a35c149 100644 --- a/crates/bindings-csharp/Runtime/Runtime.csproj +++ b/crates/bindings-csharp/Runtime/Runtime.csproj @@ -45,6 +45,7 @@ + diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index 57ed816d939..60e376cab61 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -135,6 +135,14 @@ IMPORT(Status, datastore_clear, (table_id, count)); #undef SPACETIME_MODULE_VERSION +#define SPACETIME_MODULE_VERSION "spacetime_10.6" +IMPORT(uint32_t, get_call_auth_flags, (void), ()); +#undef SPACETIME_MODULE_VERSION + +#define SPACETIME_MODULE_VERSION "spacetime_10.7" +IMPORT(Status, env_get, (const uint8_t* key, uint32_t key_len, BytesSource* source), (key, key_len, source)); +#undef SPACETIME_MODULE_VERSION + #ifndef EXPERIMENTAL_WASM_AOT static MonoClass* ffi_class; diff --git a/crates/bindings-macro/src/procedure.rs b/crates/bindings-macro/src/procedure.rs index 9f76e5b547f..129b32cc2fe 100644 --- a/crates/bindings-macro/src/procedure.rs +++ b/crates/bindings-macro/src/procedure.rs @@ -1,4 +1,5 @@ use crate::reducer::{assert_only_lifetime_generics, extract_typed_args, generate_explicit_names_impl}; +use crate::reducer::{parse_visibility, DeclaredVisibility}; use crate::sym; use crate::util::{check_duplicate, ident_to_litstr, match_meta}; use proc_macro2::TokenStream; @@ -10,12 +11,16 @@ use syn::{ItemFn, LitStr}; pub(crate) struct ProcedureArgs { /// For consistency with reducers: allow specifying a different export name than the Rust function name. name: Option, + visibility: Option, } impl ProcedureArgs { pub(crate) fn parse(input: TokenStream) -> syn::Result { let mut args = Self::default(); syn::meta::parser(|meta| { + if parse_visibility(&meta, &mut args.visibility)? { + return Ok(()); + } match_meta!(match meta { sym::name => { check_duplicate(&args.name, &meta)?; @@ -29,10 +34,11 @@ impl ProcedureArgs { } } -pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) -> syn::Result { +pub(crate) fn procedure_impl(args: ProcedureArgs, original_function: &ItemFn) -> syn::Result { let func_name = &original_function.sig.ident; let vis = &original_function.vis; - let explicit_name = _args.name.as_ref(); + let explicit_name = args.name.as_ref(); + let visibility = args.visibility.map(DeclaredVisibility::tokens).into_iter(); let procedure_name = ident_to_litstr(func_name); @@ -117,6 +123,7 @@ pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) - /// The name of this function const NAME: &'static str = #procedure_name; + #(const DECLARED_VISIBILITY: Option = Some(#visibility);)* /// The parameter names of this function const ARG_NAMES: &'static [Option<&'static str>] = &[#(#opt_arg_names),*]; @@ -133,3 +140,29 @@ pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) - #generate_explicit_names }) } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn procedure_visibility_rejects_duplicates_and_emits_selection() { + assert!(ProcedureArgs::parse(quote!(private, public)).is_err()); + assert!(ProcedureArgs::parse(quote!(internal, internal)).is_err()); + let function: ItemFn = syn::parse_quote!( + fn example(ctx: &mut ProcedureContext) -> u64 { + 0 + } + ); + for (input, expected) in [ + (quote!(internal), "Internal"), + (quote!(private), "Private"), + (quote!(public), "ClientCallable"), + ] { + let tokens = procedure_impl(ProcedureArgs::parse(input).unwrap(), &function) + .unwrap() + .to_string(); + assert!(tokens.contains("DECLARED_VISIBILITY")); + assert!(tokens.contains(&format!("FunctionVisibility :: {expected}"))); + } + } +} diff --git a/crates/bindings-macro/src/reducer.rs b/crates/bindings-macro/src/reducer.rs index ac261ced35f..3093a51397a 100644 --- a/crates/bindings-macro/src/reducer.rs +++ b/crates/bindings-macro/src/reducer.rs @@ -10,6 +10,44 @@ use syn::{FnArg, Ident, ItemFn, LitStr, PatType}; pub(crate) struct ReducerArgs { name: Option, lifecycle: Option, + visibility: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DeclaredVisibility { + Internal, + Private, + Public, +} + +impl DeclaredVisibility { + pub(crate) fn tokens(self) -> TokenStream { + let variant = match self { + Self::Internal => "Internal", + Self::Private => "Private", + Self::Public => "ClientCallable", + }; + let variant = Ident::new(variant, Span::call_site()); + quote!(spacetimedb::rt::FunctionVisibility::#variant) + } +} + +pub(crate) fn parse_visibility( + meta: &syn::meta::ParseNestedMeta<'_>, + visibility: &mut Option, +) -> syn::Result { + let value = if meta.path.is_ident("internal") { + DeclaredVisibility::Internal + } else if meta.path.is_ident("private") { + DeclaredVisibility::Private + } else if meta.path.is_ident("public") { + DeclaredVisibility::Public + } else { + return Ok(false); + }; + check_duplicate_msg(visibility, meta, "already specified a function visibility")?; + *visibility = Some(value); + Ok(true) } enum LifecycleReducer { @@ -37,6 +75,9 @@ impl ReducerArgs { pub(crate) fn parse(input: TokenStream) -> syn::Result { let mut args = Self::default(); syn::meta::parser(|meta| { + if parse_visibility(&meta, &mut args.visibility)? { + return Ok(()); + } let mut set_lifecycle = |kind: fn(Span) -> _| -> syn::Result<()> { check_duplicate_msg(&args.lifecycle, &meta, "already specified a lifecycle reducer kind")?; args.lifecycle = Some(kind(meta.path.span())); @@ -55,6 +96,12 @@ impl ReducerArgs { Ok(()) }) .parse2(input)?; + if args.lifecycle.is_some() && args.visibility.is_some_and(|v| v != DeclaredVisibility::Internal) { + return Err(syn::Error::new( + Span::call_site(), + "lifecycle reducers must have internal visibility", + )); + } Ok(args) } } @@ -101,6 +148,7 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn assert_only_lifetime_generics(original_function, "reducers")?; let lifecycle = args.lifecycle.iter().filter_map(|lc| lc.to_lifecycle_value()); + let visibility = args.visibility.map(DeclaredVisibility::tokens).into_iter(); let typed_args = extract_typed_args(original_function)?; @@ -165,6 +213,7 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn /// The function kind, which will cause scheduled tables to accept reducers. type FnKind = spacetimedb::rt::FnKindReducer; const NAME: &'static str = #reducer_name; + #(const DECLARED_VISIBILITY: Option = Some(#visibility);)* #(const LIFECYCLE: Option = Some(#lifecycle);)* const ARG_NAMES: &'static [Option<&'static str>] = &[#(#opt_arg_names),*]; const INVOKE: Self::Invoke = #func_name::invoke; @@ -202,3 +251,43 @@ pub(crate) fn generate_explicit_names_impl( } } } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn visibility_declarations_are_unambiguous() { + for input in [ + quote!(private, public), + quote!(internal, internal), + quote!(init, private), + quote!(public, client_connected), + ] { + assert!(ReducerArgs::parse(input).is_err()); + } + for input in [ + quote!(), + quote!(public), + quote!(private), + quote!(internal), + quote!(init, internal), + ] { + assert!(ReducerArgs::parse(input).is_ok()); + } + } + #[test] + fn rust_item_visibility_does_not_select_database_visibility() { + let function: ItemFn = syn::parse_quote!( + pub fn example(ctx: &ReducerContext) {} + ); + let implicit = reducer_impl(ReducerArgs::parse(quote!()).unwrap(), &function) + .unwrap() + .to_string(); + assert!(!implicit.contains("DECLARED_VISIBILITY")); + let explicit = reducer_impl(ReducerArgs::parse(quote!(internal)).unwrap(), &function) + .unwrap() + .to_string(); + assert!(explicit.contains("DECLARED_VISIBILITY")); + assert!(explicit.contains("FunctionVisibility :: Internal")); + } +} diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index b31c4d0a451..36ee7f03f28 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -885,6 +885,25 @@ pub mod raw { pub fn datastore_clear(table_id: TableId, out: *mut u64) -> u16; } + #[link(wasm_import_module = "spacetime_10.6")] + unsafe extern "C" { + /// Authentication flags for the active invocation. Bit 0 is INTERNAL. + /// Read at context construction; neither a missing connection ID nor JWT + /// claims imply internal authority. Unknown bits must be ignored. + pub fn get_call_auth_flags() -> u32; + } + + #[link(wasm_import_module = "spacetime_10.7")] + unsafe extern "C" { + /// Read a UTF-8 environment value. Writes INVALID for a missing key; + /// present empty strings have a valid BytesSource. Returns ordinary errno. + /// Invalid keys return HOST_CALL_FAILURE. NO_SPACE means 256 byte + /// sources remain unconsumed; consume a source before retrying. + /// Calls outside a reducer/view + /// transaction or procedure return NOT_IN_TRANSACTION. + pub fn env_get(key: *const u8, key_len: usize, out: *mut BytesSource) -> u16; + } + /// What strategy does the database index use? /// /// See also: @@ -1496,6 +1515,14 @@ pub fn get_jwt(connection_id: [u8; 16]) -> Option { } } +/// Read a database environment value without exposing the system table. +#[inline] +pub fn env_get(key: &str) -> Option { + let source = unsafe { call(|out| raw::env_get(key.as_ptr(), key.len(), out)) } + .unwrap_or_else(|errno: Errno| panic!("Error reading environment: {errno}")); + (source != raw::BytesSource::INVALID).then_some(source) +} + pub struct RowIter { raw: raw::RowIter, } @@ -1661,3 +1688,10 @@ pub mod procedure { } } } + +/// Read host-verified authentication flags for the active invocation. +/// Bit 0 is INTERNAL; all other bits are reserved. +pub fn get_call_auth_flags() -> u32 { + // SAFETY: no pointers or guest-provided values are passed to the host. + unsafe { raw::get_call_auth_flags() } +} diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index 5eb0acc19a4..9a37a85f582 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -18,6 +18,26 @@ You can use the package in the browser, using a bundler like vite/parcel/rsbuild ### Usage +#### Module function visibility and invocation authentication + +Reducer and procedure options accept `visibility: 'public'`, `'private'`, or +`'internal'`. For example, `spacetime.reducer({ visibility: 'internal' }, ctx => {})` +declares an internal reducer. Omission means public for ordinary functions and +private for scheduled functions. An explicit choice is preserved when the +function is scheduled. Lifecycle reducers permit only omission or `'internal'` +and can only run for their host lifecycle event. + +Internal functions require verified internal authority. Private functions also +admit the owner, and public functions admit any client. `ctx.senderAuth.isInternal` +captures the host's invocation authority independently of connection and JWT +presence, so an internal call can have a JWT. `ctx.senderAuth.jwt.identity` is the +verified sender supplied by the host. Procedure transactions preserve this +authentication. Newly compiled modules retain schema V10 and advertise +`hosted_auth_v1`. The extended visibility values and capability section require +a compatible host; older V10 definitions retain their existing defaults. + +#### Client SDK + In order to connect to a database you have to generate module bindings for your database. ```ts diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index c0855af29bf..f4bea6d4a72 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -76,6 +76,8 @@ export type ExplicitNames = __Infer; export const FunctionVisibility = __t.enum('FunctionVisibility', { Private: __t.unit(), ClientCallable: __t.unit(), + Internal: __t.unit(), + ExplicitClientCallable: __t.unit(), }); export type FunctionVisibility = __Infer; @@ -387,6 +389,7 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { get HttpRoutes() { return __t.array(RawHttpRouteDefV10); }, + Capabilities: __t.array(__t.string()), }); export type RawModuleDefV10Section = __Infer; diff --git a/crates/bindings-typescript/src/lib/environment.ts b/crates/bindings-typescript/src/lib/environment.ts new file mode 100644 index 00000000000..3798cfb43d4 --- /dev/null +++ b/crates/bindings-typescript/src/lib/environment.ts @@ -0,0 +1,5 @@ +/** Read-only database environment access. Missing keys return null; empty values return "". */ +export interface Environment { + /** Reads the current transaction, or a short snapshot outside a procedure transaction. */ + get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index 0eae2adc2a9..6e3bee1c3f5 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -1,3 +1,4 @@ +import type { Environment } from './environment'; import type { DbView } from '../server/db_view'; import type { Random } from '../server/rng'; import type { ConnectionId } from './connection_id'; @@ -60,7 +61,7 @@ export type Reducer = ( * Authentication information for the caller of a reducer. */ export type AuthCtx = Readonly<{ - /** Whether the caller is an internal system process. */ + /** Whether the host verified internal invocation authority. Independent of JWT presence. */ isInternal: boolean; /** Whether the caller has authenticated with a JWT token. */ hasJWT: boolean; @@ -92,7 +93,7 @@ export interface JwtClaims { readonly issuer: string; /** The audience of the JWT token ('aud') */ readonly audience: readonly string[]; - /** The identity associated with the JWT token, which is based on the sub and iss */ + /** The verified sender Identity provided by the host, including hosted credentials. */ readonly identity: Identity; /** The full payload as a JsonObject */ readonly fullPayload: JsonObject; @@ -109,6 +110,7 @@ export type ReducerCtx = Readonly<{ timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + env: Environment; senderAuth: AuthCtx; newUuidV4(): Uuid; newUuidV7(): Uuid; diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index ab480c93db5..f5526b271d5 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -197,6 +197,7 @@ export class ModuleContext { lifeCycleReducers: [], httpHandlers: [], httpRoutes: [], + capabilities: ['hosted_auth_v1'], caseConversionPolicy: { tag: 'SnakeCase' }, explicitNames: { entries: [], @@ -217,6 +218,7 @@ export class ModuleContext { const module = this.#moduleDef; push(module.typespace && { tag: 'Typespace', value: module.typespace }); + push({ tag: 'Capabilities', value: module.capabilities }); push(module.types && { tag: 'Types', value: module.types }); push(module.tables && { tag: 'Tables', value: module.tables }); push(module.reducers && { tag: 'Reducers', value: module.reducers }); diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts new file mode 100644 index 00000000000..349e8ffed8c --- /dev/null +++ b/crates/bindings-typescript/src/server/environment.ts @@ -0,0 +1,6 @@ +import { env_get } from 'spacetime:sys@2.3'; +import type { Environment } from '../lib/environment'; + +/** Values are not cached: transaction and procedure reads retain host semantics. */ +export const environment: Environment = Object.freeze({ get: env_get }); +export type { Environment } from '../lib/environment'; diff --git a/crates/bindings-typescript/src/server/function_visibility.ts b/crates/bindings-typescript/src/server/function_visibility.ts new file mode 100644 index 00000000000..658fb1dad0d --- /dev/null +++ b/crates/bindings-typescript/src/server/function_visibility.ts @@ -0,0 +1,24 @@ +import { FunctionVisibility as RawFunctionVisibility } from '../lib/autogen/types'; + +/** Internal functions require verified internal authority. Private functions also + * admit the owner. Public functions admit any authenticated client. */ +export type FunctionVisibility = 'public' | 'private' | 'internal'; + +export function rawVisibility( + visibility: FunctionVisibility | undefined +): RawFunctionVisibility { + switch (visibility) { + case undefined: + // Preserve V10's existing context-dependent default, including scheduled + // private functions, without changing the raw definition's field layout. + return RawFunctionVisibility.ClientCallable; + case 'public': + return RawFunctionVisibility.ExplicitClientCallable; + case 'private': + return RawFunctionVisibility.Private; + case 'internal': + return RawFunctionVisibility.Internal; + default: + throw new TypeError('Invalid function visibility'); + } +} diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index a092f513611..0659461a74a 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -1,3 +1,4 @@ +import type { Environment } from '../lib/environment'; import type { Identity } from '../lib/identity'; import type { HttpMethod, @@ -214,6 +215,7 @@ export class Request { export interface HandlerContext { readonly timestamp: Timestamp; readonly http: HttpClient; + readonly env: Environment; readonly identity: Identity; readonly random: Random; withTx(body: (ctx: TransactionCtx) => T): T; diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index a840be4a59d..231f6b2f5ea 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -10,6 +10,7 @@ export { table } from '../lib/table'; export { SenderError, SpacetimeHostError, errors } from './errors'; export type { Reducer, ReducerCtx, JwtClaims, AuthCtx } from '../lib/reducers'; export type { ReducerExport } from './reducers'; +export type { FunctionVisibility } from './function_visibility'; export { type DbView } from './db_view'; export * from './query'; export type { @@ -35,3 +36,5 @@ export { export type { HandlerContext, HttpHandlerExport } from './http'; import './polyfills'; // Ensure polyfills are loaded + +export type { Environment } from './environment'; diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index d07b71f5185..ea932e6efcd 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -1,15 +1,16 @@ +import { environment, type Environment } from './environment'; import { AlgebraicType, ProductType, type Deserializer, type Serializer, } from '../lib/algebraic_type'; -import { FunctionVisibility } from '../lib/autogen/types'; +import { rawVisibility, type FunctionVisibility } from './function_visibility'; import BinaryReader from '../lib/binary_reader'; import BinaryWriter from '../lib/binary_writer'; import type { ConnectionId } from '../lib/connection_id'; import { Identity } from '../lib/identity'; -import type { ParamsObj, ReducerCtx } from '../lib/reducers'; +import type { AuthCtx, ParamsObj, ReducerCtx } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; import { Timestamp } from '../lib/timestamp'; import { @@ -22,7 +23,13 @@ import { Uuid } from '../lib/uuid'; import { httpClient, type HttpClient } from './http_internal'; import type { DbView } from './db_view'; import { makeRandom, type Random } from './rng'; -import { callUserFunction, ReducerCtxImpl, runWithTx, sys } from './runtime'; +import { + AuthCtxImpl, + callUserFunction, + ReducerCtxImpl, + runWithTx, + sys, +} from './runtime'; import { exportContext, registerExport, @@ -47,16 +54,14 @@ export function makeProcedureExport< ret: Ret, fn: ProcedureFn ): ProcedureExport { - const name = opts?.name; - const procedureExport: ProcedureExport = (...args) => fn(...args); procedureExport[exportContext] = ctx; procedureExport[registerExport] = (ctx, exportName) => { - registerProcedure(ctx, name ?? exportName, params, ret, fn); + registerProcedure(ctx, exportName, params, ret, fn, opts); ctx.functionExports.set( procedureExport as ProcedureExport, - name ?? exportName + exportName ); }; @@ -70,16 +75,20 @@ export type ProcedureFn< > = (ctx: ProcedureCtx, args: InferTypeOfRow) => Infer; export interface ProcedureOpts { - name: string; + name?: string; + /** Defaults to public, or private when scheduled. */ + visibility?: FunctionVisibility; } export interface ProcedureCtx { + readonly env: Environment; readonly sender: Identity; readonly databaseIdentity: Identity; /** @deprecated Use `databaseIdentity` instead. */ readonly identity: Identity; readonly timestamp: Timestamp; readonly connectionId: ConnectionId | null; + readonly senderAuth: AuthCtx; readonly http: HttpClient; readonly random: Random; withTx(body: (ctx: TransactionCtx) => T): T; @@ -91,12 +100,6 @@ export interface ProcedureCtx { export interface TransactionCtx extends ReducerCtx {} -type ITransactionCtx = TransactionCtx; - -const TransactionCtxImpl = class TransactionCtx - extends ReducerCtxImpl - implements ITransactionCtx {}; - function registerProcedure< S extends UntypedSchemaDef, Params extends ParamsObj, @@ -124,7 +127,7 @@ function registerProcedure< sourceName: exportName, params: paramsType, returnType, - visibility: FunctionVisibility.ClientCallable, + visibility: rawVisibility(opts?.visibility), }); if (opts?.name != null) { @@ -187,6 +190,8 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; + readonly senderAuth: AuthCtx; + readonly env = environment; constructor( readonly sender: Identity, @@ -195,6 +200,11 @@ const ProcedureCtxImpl = class ProcedureCtx dbView: () => DbView ) { this.#dbView = dbView; + this.senderAuth = AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); } get databaseIdentity() { @@ -216,11 +226,12 @@ const ProcedureCtxImpl = class ProcedureCtx withTx(body: (ctx: TransactionCtx) => T): T { return runWithTx( timestamp => - new TransactionCtxImpl( + new ReducerCtxImpl( this.sender, timestamp, this.connectionId, - this.#dbView() + this.#dbView(), + this.senderAuth ) as TransactionCtx, body ); diff --git a/crates/bindings-typescript/src/server/reducers.ts b/crates/bindings-typescript/src/server/reducers.ts index f8aa1c390bf..05001d9c476 100644 --- a/crates/bindings-typescript/src/server/reducers.ts +++ b/crates/bindings-typescript/src/server/reducers.ts @@ -1,5 +1,6 @@ import { AlgebraicType } from '../lib/algebraic_type'; -import { FunctionVisibility, type Lifecycle } from '../lib/autogen/types'; +import { type Lifecycle } from '../lib/autogen/types'; +import { rawVisibility, type FunctionVisibility } from './function_visibility'; import type { ParamsObj, Reducer } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; import { RowBuilder, type RowObj } from '../lib/type_builders'; @@ -18,7 +19,9 @@ export interface ReducerExport< ModuleExport {} export interface ReducerOpts { - name: string; + name?: string; + /** Defaults to public, or private when scheduled. Lifecycle hooks are internal. */ + visibility?: FunctionVisibility; } export function makeReducerExport< @@ -73,12 +76,19 @@ export function registerReducer( const ref = ctx.registerTypesRecursively(params); const paramsType = ctx.resolveType(ref).value; const isLifecycle = lifecycle != null; + if ( + isLifecycle && + opts?.visibility != null && + opts.visibility !== 'internal' + ) { + throw new TypeError('Lifecycle reducers only support internal visibility'); + } ctx.moduleDef.reducers.push({ sourceName: exportName, params: paramsType, - //ModuleDef validation code is responsible to mark private reducers - visibility: FunctionVisibility.ClientCallable, + // Keep the legacy default distinct from an explicit public declaration. + visibility: rawVisibility(opts?.visibility), //Hardcoded for now - reducers do not return values yet okReturnType: AlgebraicType.Product({ elements: [] }), errReturnType: AlgebraicType.String, diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index b3121dc2085..e4b1a28cf5f 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,5 +1,7 @@ +import { environment } from './environment'; import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; +import * as _syscalls2_2 from 'spacetime:sys@2.2'; import type { ModuleHooks, u128, u16, u256, u32 } from 'spacetime:sys@2.0'; import { @@ -59,7 +61,7 @@ import { HttpRequest, HttpResponse } from '../lib/autogen/types'; const { freeze } = Object; -export const sys = { ..._syscalls2_0, ..._syscalls2_1 }; +export const sys = { ..._syscalls2_0, ..._syscalls2_1, ..._syscalls2_2 }; function requestFromWire(request: HttpRequest, body: Uint8Array): Request { return Request[makeRequest](body, { @@ -104,7 +106,8 @@ class JwtClaimsImpl implements JwtClaims { /** * Creates a new JwtClaims instance. * @param rawPayload The JWT payload as a raw JSON string. - * @param identity The identity for this JWT. We are only taking this because we don't have a blake3 implementation (which we need to compute it). + * @param identity The verified sender Identity supplied by the host. Claims + * cannot override it, including for hosted database credentials. */ constructor( public readonly rawPayload: string, @@ -132,7 +135,7 @@ class JwtClaimsImpl implements JwtClaims { } } -class AuthCtxImpl implements AuthCtx { +export class AuthCtxImpl implements AuthCtx { public readonly isInternal: boolean; // Source of the JWT payload string, if there is one. @@ -178,29 +181,21 @@ class AuthCtxImpl implements AuthCtx { return this._jwtClaims!; } - /** Create a context representing internal (non-user) requests. */ - static internal(): AuthCtx { - return new AuthCtxImpl({ - isInternal: true, - jwtSource: () => null, - senderIdentity: Identity.zero(), - }); - } - /** If there is a connection id, look up the JWT payload from the system tables. */ static fromSystemTables( connectionId: ConnectionId | null, - sender: Identity + sender: Identity, + callAuthFlags: number ): AuthCtx { if (connectionId === null) { return new AuthCtxImpl({ - isInternal: false, + isInternal: (callAuthFlags & 1) !== 0, jwtSource: () => null, senderIdentity: sender, }); } return new AuthCtxImpl({ - isInternal: false, + isInternal: (callAuthFlags & 1) !== 0, jwtSource: () => { const payloadBuf = sys.get_jwt_payload(connectionId.__connection_id__); if (payloadBuf.length === 0) return null; @@ -219,25 +214,34 @@ export const ReducerCtxImpl = class ReducerCtx< > implements IReducerCtx { #identity: Identity | undefined; - #senderAuth: AuthCtx | undefined; + #senderAuth: AuthCtx; #uuidCounter: { value: number } | undefined; #random: Random | undefined; sender: Identity; timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + readonly env = environment; constructor( sender: Identity, timestamp: Timestamp, connectionId: ConnectionId | null, - dbView: DbView + dbView: DbView, + senderAuth?: AuthCtx ) { Object.seal(this); this.sender = sender; this.timestamp = timestamp; this.connectionId = connectionId; this.db = dbView; + this.#senderAuth = + senderAuth ?? + AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); } /** Reset the `ReducerCtx` to be used for a new transaction */ @@ -251,7 +255,11 @@ export const ReducerCtxImpl = class ReducerCtx< me.timestamp = timestamp; me.connectionId = connectionId; me.#uuidCounter = undefined; - me.#senderAuth = undefined; + me.#senderAuth = AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); } get databaseIdentity() { @@ -263,10 +271,7 @@ export const ReducerCtxImpl = class ReducerCtx< } get senderAuth() { - return (this.#senderAuth ??= AuthCtxImpl.fromSystemTables( - this.connectionId, - this.sender - )); + return this.#senderAuth; } get random() { @@ -422,6 +427,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = moduleCtx.views[id]; const ctx: ViewCtx = freeze({ + env: environment, sender: new Identity(sender), // this is the non-readonly DbView, but the typing for the user will be // the readonly one, and if they do call mutating functions it will fail @@ -447,6 +453,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = moduleCtx.anonViews[id]; const ctx: AnonymousViewCtx = freeze({ + env: environment, // this is the non-readonly DbView, but the typing for the user will be // the readonly one, and if they do call mutating functions it will fail // at runtime @@ -517,6 +524,7 @@ const BINARY_READER = new BinaryReader(new Uint8Array()); class HandlerContextImpl implements HandlerContext { + readonly env = environment; #identity: Identity | undefined; #uuidCounter: { value: number } | undefined; #random: Random | undefined; diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index d9f20be3025..97ddd4da26a 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -277,7 +277,11 @@ export class Schema implements ModuleDefaultExport { case 2: { let arg1; [arg1, fn] = args; - if (typeof arg1.name === 'string') opts = arg1 as ReducerOpts; + if ( + typeof arg1.name === 'string' || + typeof arg1.visibility === 'string' + ) + opts = arg1 as ReducerOpts; else params = arg1 as Params; break; } @@ -461,22 +465,22 @@ export class Schema implements ModuleDefaultExport { params: Params, ret: Ret, fn: ProcedureFn - ): ProcedureFn; + ): ProcedureExport; procedure>( ret: Ret, fn: ProcedureFn - ): ProcedureFn; + ): ProcedureExport; procedure>( opts: ProcedureOpts, params: Params, ret: Ret, fn: ProcedureFn - ): ProcedureFn; + ): ProcedureExport; procedure>( opts: ProcedureOpts, ret: Ret, fn: ProcedureFn - ): ProcedureFn; + ): ProcedureExport; procedure>( ...args: | [Params, Ret, ProcedureFn] @@ -495,7 +499,11 @@ export class Schema implements ModuleDefaultExport { case 3: { let arg1; [arg1, ret, fn] = args; - if (typeof arg1.name === 'string') opts = arg1 as ProcedureOpts; + if ( + typeof arg1.name === 'string' || + typeof arg1.visibility === 'string' + ) + opts = arg1 as ProcedureOpts; else params = arg1 as Params; break; } diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index f0315867cb3..32addabb9e7 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -123,3 +123,13 @@ declare module 'spacetime:sys@2.0' { declare module 'spacetime:sys@2.1' { export function datastore_clear(table_id: u32): u64; } + +declare module 'spacetime:sys@2.2' { + /** Verified invocation flags. Bit 0 is INTERNAL; JWT presence is independent. */ + export function get_call_auth_flags(): number; +} + +declare module 'spacetime:sys@2.3' { + /** Null means missing; an empty string is a present value. */ + export function env_get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index 6d2b475b177..fb0bd1ccde7 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -1,3 +1,4 @@ +import type { Environment } from '../lib/environment'; import { AlgebraicType, ProductType, @@ -75,11 +76,13 @@ export function makeAnonViewExport< export type ViewCtx = Readonly<{ sender: Identity; db: ReadonlyDbView; + env: Environment; from: QueryBuilder; }>; export type AnonymousViewCtx = Readonly<{ db: ReadonlyDbView; + env: Environment; from: QueryBuilder; }>; diff --git a/crates/bindings-typescript/tests/hosted_auth.test.ts b/crates/bindings-typescript/tests/hosted_auth.test.ts new file mode 100644 index 00000000000..c3aa52e3b53 --- /dev/null +++ b/crates/bindings-typescript/tests/hosted_auth.test.ts @@ -0,0 +1,305 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const host = vi.hoisted(() => ({ + flags: 0, + payload: '', + jwtReads: 0, + flagReads: 0, +})); +vi.mock('spacetime:sys@2.0', () => ({ + moduleHooks: Symbol('moduleHooks'), + identity: () => 1n, + row_iter_bsatn_close: () => {}, + procedure_start_mut_tx: () => 0n, + procedure_commit_mut_tx: () => {}, + procedure_abort_mut_tx: () => {}, + get_jwt_payload: () => { + host.jwtReads++; + return new TextEncoder().encode(host.payload); + }, +})); +vi.mock('spacetime:sys@2.1', () => ({})); +vi.mock('spacetime:sys@2.3', () => ({ env_get: () => null })); +vi.mock('spacetime:sys@2.2', () => ({ + get_call_auth_flags: () => { + host.flagReads++; + return host.flags; + }, +})); + +import { ReducerCtxImpl } from '../src/server/runtime'; +import { ConnectionId } from '../src/lib/connection_id'; +import { Identity } from '../src/lib/identity'; +import { Timestamp } from '../src/lib/timestamp'; +import { schema, exportContext, registerExport } from '../src/server/schema'; +import { callProcedure } from '../src/server/procedures'; +import { t } from '../src/lib/type_builders'; +import { + AlgebraicType, + FunctionVisibility, + ProductType, + RawModuleDef, + RawModuleDefV10Section, + RawReducerDefV10, +} from '../src/lib/autogen/types'; +import BinaryReader from '../src/lib/binary_reader'; +import BinaryWriter from '../src/lib/binary_writer'; + +beforeEach(() => { + Object.assign(host, { flags: 0, payload: '', jwtReads: 0, flagReads: 0 }); +}); + +describe('verified invocation authentication', () => { + it.each([0, 1])( + 'preserves flag %s for calls without a connection or JWT', + flags => { + host.flags = flags; + const ctx = new ReducerCtxImpl( + new Identity(1n), + Timestamp.UNIX_EPOCH, + null, + {} + ); + host.flags = flags ^ 1; + expect(host.flagReads).toBe(1); + expect(ctx.senderAuth.isInternal).toBe(Boolean(flags)); + expect(ctx.senderAuth.hasJWT).toBe(false); + expect(ctx.senderAuth.jwt).toBeNull(); + expect(host.jwtReads).toBe(0); + } + ); + + it('retains an internal connection and JWT independently, using the verified sender Identity', () => { + host.flags = 1; + host.payload = JSON.stringify({ + iss: 'unrelated-issuer', + sub: 'unrelated-subject', + identity: 'untrusted-claim', + }); + const sender = new Identity(123n); + const connection = new ConnectionId(7n); + const ctx = new ReducerCtxImpl( + sender, + Timestamp.UNIX_EPOCH, + connection, + {} + ); + host.flags = 0; + expect(ctx.connectionId).toBe(connection); + expect(ctx.senderAuth.isInternal).toBe(true); + expect(host.jwtReads).toBe(0); + expect(ctx.senderAuth.hasJWT).toBe(true); + expect(ctx.senderAuth.jwt?.identity).toBe(sender); + expect(ctx.senderAuth.jwt?.subject).toBe('unrelated-subject'); + expect(host.jwtReads).toBe(1); + }); + + it('refreshes captured flags and sender when a cached reducer context is reused', () => { + host.flags = 1; + const ctx = new ReducerCtxImpl( + new Identity(1n), + Timestamp.UNIX_EPOCH, + null, + {} + ); + const firstAuth = ctx.senderAuth; + host.flags = 0; + ReducerCtxImpl.reset( + ctx, + new Identity(2n), + Timestamp.UNIX_EPOCH, + new ConnectionId(8n) + ); + host.flags = 1; + expect(firstAuth.isInternal).toBe(true); + expect(ctx.senderAuth.isInternal).toBe(false); + expect(ctx.senderAuth.hasJWT).toBe(false); + }); + + it('preserves procedure auth inside a transaction after the host flags change', () => { + host.flags = 1; + const module = schema({}); + const proc = module.procedure(t.unit(), ctx => { + host.flags = 0; + ctx.withTx(tx => { + expect(tx.senderAuth).toBe(ctx.senderAuth); + expect(tx.senderAuth.isInternal).toBe(true); + expect(tx.connectionId).toBe(ctx.connectionId); + }); + return {}; + }); + const inner = proc[exportContext]!; + proc[registerExport](inner, 'procedure_auth'); + callProcedure( + inner, + 0, + new Identity(9n), + new ConnectionId(8n), + Timestamp.UNIX_EPOCH, + new Uint8Array(), + () => ({}) + ); + expect(host.flagReads).toBe(1); + }); +}); + +describe('V10 explicit function visibility', () => { + it('preserves existing visibility tags and appends the new variants and capability section', () => { + const legacyVisibility = t.enum('LegacyFunctionVisibility', { + Private: t.unit(), + ClientCallable: t.unit(), + }); + const variants = [ + FunctionVisibility.Private, + FunctionVisibility.ClientCallable, + FunctionVisibility.Internal, + FunctionVisibility.ExplicitClientCallable, + ]; + for (const [tag, visibility] of variants.entries()) { + const writer = new BinaryWriter(8); + FunctionVisibility.serialize(writer, visibility); + expect([...writer.getBuffer()]).toEqual([tag]); + const reader = new BinaryReader(writer.getBuffer()); + if (tag < 2) { + expect(legacyVisibility.deserialize(reader).tag).toBe(visibility.tag); + } + } + const writer = new BinaryWriter(8); + RawModuleDefV10Section.serialize(writer, { + tag: 'Capabilities', + value: [], + }); + expect([...writer.getBuffer()]).toEqual([13, 0, 0, 0, 0]); + }); + + it('retains the V10 reducer field layout without an optional visibility wrapper', () => { + const module = schema({}); + const reducer = module.reducer({ visibility: 'public' }, () => {}); + const inner = reducer[exportContext]!; + reducer[registerExport](inner, 'public_reducer'); + const definition = inner.moduleDef.reducers[0]; + const writer = new BinaryWriter(128); + RawReducerDefV10.serialize(writer, definition); + const expected = new BinaryWriter(128); + expected.writeString(definition.sourceName); + ProductType.serialize(expected, definition.params); + expected.writeByte(3); + AlgebraicType.serialize(expected, definition.okReturnType); + AlgebraicType.serialize(expected, definition.errReturnType); + expect(writer.getBuffer()).toEqual(expected.getBuffer()); + }); + + it('serializes omission separately from explicit visibility and advertises hosted auth', () => { + const module = schema({}); + const omitted = module.reducer(() => {}); + const explicitlyPublic = module.reducer({ visibility: 'public' }, () => {}); + const privateReducer = module.reducer({ visibility: 'private' }, () => {}); + const internalReducer = module.reducer( + { visibility: 'internal' }, + () => {} + ); + const inner = omitted[exportContext]!; + for (const [name, reducer] of Object.entries({ + omitted, + explicitlyPublic, + privateReducer, + internalReducer, + })) { + reducer[registerExport](inner, name); + } + // Being scheduled must not erase a public choice or manufacture an explicit + // choice for the default. The host resolves the latter to Private. + for (const name of [ + 'omitted', + 'explicitlyPublic', + 'privateReducer', + 'internalReducer', + ]) { + inner.moduleDef.schedules.push({ + sourceName: undefined, + tableName: `jobs_${name}`, + scheduleAtCol: 0, + functionName: name, + }); + } + const raw = RawModuleDef.V10(inner.rawModuleDefV10()); + const writer = new BinaryWriter(128); + RawModuleDef.serialize(writer, raw); + expect(writer.getBuffer()[0]).toBe(2); + const decoded = RawModuleDef.deserialize( + new BinaryReader(writer.getBuffer()) + ); + const roundTrip = new BinaryWriter(128); + RawModuleDef.serialize(roundTrip, decoded); + expect(roundTrip.getBuffer()).toEqual(writer.getBuffer()); + expect(decoded.tag).toBe('V10'); + if (decoded.tag !== 'V10') throw new Error('Expected V10'); + const reducers = decoded.value.sections.find( + section => section.tag === 'Reducers' + ); + expect(reducers?.value.map(reducer => reducer.visibility.tag)).toEqual([ + 'ClientCallable', + 'ExplicitClientCallable', + 'Private', + 'Internal', + ]); + expect( + inner.moduleDef.reducers.map(reducer => reducer.visibility.tag) + ).toEqual([ + 'ClientCallable', + 'ExplicitClientCallable', + 'Private', + 'Internal', + ]); + expect(inner.moduleDef.capabilities).toEqual(['hosted_auth_v1']); + }); + + it('retains procedure names and explicit visibility, including a visibility parameter', () => { + const module = schema({}); + const proc = module.procedure( + { name: 'public_name', visibility: 'internal' }, + t.unit(), + () => ({}) + ); + const reducer = module.reducer({ visibility: t.string() }, () => {}); + const inner = proc[exportContext]!; + proc[registerExport](inner, 'source_name'); + reducer[registerExport](inner, 'accept_visibility'); + expect(inner.moduleDef.procedures[0].sourceName).toBe('source_name'); + expect(inner.moduleDef.procedures[0].visibility.tag).toBe('Internal'); + expect(inner.moduleDef.explicitNames.entries).toContainEqual({ + tag: 'Function', + value: { sourceName: 'source_name', canonicalName: 'public_name' }, + }); + expect(inner.moduleDef.reducers[0].params.elements[0].name).toBe( + 'visibility' + ); + }); + + it.each(['private', 'public'] as const)( + 'rejects explicit %s lifecycle declarations', + visibility => { + const module = schema({}); + const invalid = module.init({ visibility }, () => {}); + expect(() => + invalid[registerExport](invalid[exportContext]!, 'invalid_init') + ).toThrow('Lifecycle reducers only support internal visibility'); + } + ); + + it.each([undefined, 'internal'] as const)( + 'preserves permitted lifecycle declaration %s for host event dispatch', + visibility => { + const module = schema({}); + const valid = module.init({ visibility }, () => {}); + valid[registerExport](valid[exportContext]!, 'valid_init'); + const inner = valid[exportContext]!; + expect(inner.moduleDef.reducers[0].visibility.tag).toBe( + visibility === undefined ? 'ClientCallable' : 'Internal' + ); + expect(inner.moduleDef.lifeCycleReducers).toEqual([ + { lifecycleSpec: { tag: 'Init' }, functionName: 'valid_init' }, + ]); + } + ); +}); diff --git a/crates/bindings/src/http.rs b/crates/bindings/src/http.rs index a0ee89819ba..f3fc86af45c 100644 --- a/crates/bindings/src/http.rs +++ b/crates/bindings/src/http.rs @@ -7,7 +7,7 @@ use crate::{ rt::{read_bytes_source_as, read_bytes_source_into}, - try_with_tx, with_tx, IterBuf, StdbRng, Timestamp, TxContext, + try_with_tx, with_tx, AuthCtx, IterBuf, StdbRng, Timestamp, TxContext, }; use bytes::Bytes; #[cfg(feature = "rand")] @@ -88,6 +88,7 @@ pub struct HandlerContext { /// Methods for performing HTTP requests. pub http: HttpClient, + sender_auth: AuthCtx, #[cfg(feature = "rand08")] pub(crate) rng: OnceCell, @@ -103,6 +104,7 @@ impl HandlerContext { Self { timestamp, http: HttpClient {}, + sender_auth: AuthCtx::from_invocation(Identity::ZERO, None), #[cfg(feature = "rand08")] rng: OnceCell::new(), #[cfg(feature = "rand")] @@ -117,12 +119,12 @@ impl HandlerContext { /// Acquire a mutable transaction and execute `body` with read-write access. pub fn with_tx(&mut self, body: impl Fn(&TxContext) -> T) -> T { - with_tx(body) + with_tx(Identity::ZERO, None, &self.sender_auth, body) } /// Acquire a mutable transaction and execute `body` with read-write access. pub fn try_with_tx(&mut self, body: impl Fn(&TxContext) -> Result) -> Result { - try_with_tx(body) + try_with_tx(Identity::ZERO, None, &self.sender_auth, body) } /// Create a new random [`Uuid`] `v4` using the built-in RNG. diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index abc972748dd..1f47dba6955 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -912,10 +912,29 @@ pub use spacetimedb_bindings_macro::view; pub struct QueryBuilder {} pub use query_builder::{Query, RawQuery}; +/// Read-only access to this database's environment store. +/// +/// Reads use the current transaction. In a procedure outside a transaction, +/// each read uses a short snapshot; use `with_tx` to read related keys together. +/// Values are stored in plaintext and may be read by database collaborators. +#[derive(Clone, Copy, Debug, Default)] +pub struct Environment { + _private: (), +} + +impl Environment { + /// Return None for a missing key and Some("") for a present empty value. + /// Keys must be POSIX environment names of at most 256 bytes. + pub fn get(&self, key: &str) -> Option { + rt::env_get(key) + } +} + /// One of two possible types that can be passed as the first argument to a `#[view]`. /// The other is [`ViewContext`]. /// Use this type if the view does not depend on the caller's identity. pub struct AnonymousViewContext { + pub env: Environment, pub db: LocalReadOnly, pub from: QueryBuilder, } @@ -923,6 +942,7 @@ pub struct AnonymousViewContext { impl Default for AnonymousViewContext { fn default() -> Self { Self { + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -932,6 +952,7 @@ impl Default for AnonymousViewContext { /// The other is [`AnonymousViewContext`]. /// Use this type if the view depends on the caller's identity. pub struct ViewContext { + pub env: Environment, sender: Identity, pub db: LocalReadOnly, pub from: QueryBuilder, @@ -941,6 +962,7 @@ impl ViewContext { pub fn new(sender: Identity) -> Self { Self { sender, + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -971,6 +993,8 @@ impl ViewContext { /// Implements the `DbContext` trait for accessing views into a database. #[non_exhaustive] pub struct ReducerContext { + /// Read-only access to the database environment in this transaction. + pub env: Environment, /// The `Identity` of the client that invoked the reducer. sender: Identity, @@ -1035,6 +1059,7 @@ impl ReducerContext { #[doc(hidden)] pub fn __dummy() -> Self { Self { + env: Environment::default(), db: Local {}, sender: Identity::__dummy(), timestamp: Timestamp::UNIX_EPOCH, @@ -1049,12 +1074,24 @@ impl ReducerContext { #[doc(hidden)] fn new(db: Local, sender: Identity, connection_id: Option, timestamp: Timestamp) -> Self { + let sender_auth = AuthCtx::from_invocation(sender, connection_id); + Self::new_with_auth(db, sender, connection_id, timestamp, sender_auth) + } + + fn new_with_auth( + db: Local, + sender: Identity, + connection_id: Option, + timestamp: Timestamp, + sender_auth: AuthCtx, + ) -> Self { Self { + env: Environment::default(), db, sender, timestamp, connection_id, - sender_auth: AuthCtx::from_connection_id_opt(connection_id), + sender_auth, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), #[cfg(feature = "rand")] @@ -1184,7 +1221,12 @@ impl Deref for TxContext { } #[cfg(feature = "unstable")] -fn try_with_tx(body: impl Fn(&TxContext) -> Result) -> Result { +fn try_with_tx( + sender: Identity, + connection_id: Option, + sender_auth: &AuthCtx, + body: impl Fn(&TxContext) -> Result, +) -> Result { let abort = || { crate::sys::procedure::procedure_abort_mut_tx() .expect("should have a pending mutable anon tx as `procedure_start_mut_tx` preceded") @@ -1195,8 +1237,8 @@ fn try_with_tx(body: impl Fn(&TxContext) -> Result) -> Result .expect("holding `&mut HandlerContext`, so should not be in a tx already; called manually elsewhere?"); let timestamp = Timestamp::from_micros_since_unix_epoch(timestamp); - // Use the internal auth context (no external caller identity). - let tx = ReducerContext::new(crate::Local {}, Identity::ZERO, None, timestamp); + // Every retry retains the original invocation's identity and authority. + let tx = ReducerContext::new_with_auth(crate::Local {}, sender, connection_id, timestamp, sender_auth.clone()); let tx = TxContext(tx); struct DoOnDrop(F); @@ -1230,9 +1272,14 @@ fn try_with_tx(body: impl Fn(&TxContext) -> Result) -> Result } #[cfg(feature = "unstable")] -fn with_tx(body: impl Fn(&TxContext) -> T) -> T { +fn with_tx( + sender: Identity, + connection_id: Option, + sender_auth: &AuthCtx, + body: impl Fn(&TxContext) -> T, +) -> T { use core::convert::Infallible; - match try_with_tx::(|tx| Ok(body(tx))) { + match try_with_tx::(sender, connection_id, sender_auth, |tx| Ok(body(tx))) { Ok(v) => v, Err(e) => match e {}, } @@ -1247,6 +1294,8 @@ fn with_tx(body: impl Fn(&TxContext) -> T) -> T { #[non_exhaustive] #[cfg(feature = "unstable")] pub struct ProcedureContext { + /// Read-only access to the database environment. + pub env: Environment, /// The `Identity` of the client that invoked the procedure. sender: Identity, @@ -1257,6 +1306,7 @@ pub struct ProcedureContext { /// /// Will be `None` for certain scheduled procedures. connection_id: Option, + sender_auth: AuthCtx, /// Methods for performing HTTP requests. pub http: crate::http::HttpClient, @@ -1279,6 +1329,8 @@ impl ProcedureContext { sender, timestamp, connection_id, + sender_auth: AuthCtx::from_invocation(sender, connection_id), + env: Environment::default(), http: http::HttpClient {}, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), @@ -1287,6 +1339,11 @@ impl ProcedureContext { } } + /// Host-verified authentication for this invocation, retained in transactions. + pub fn sender_auth(&self) -> &AuthCtx { + &self.sender_auth + } + /// The `Identity` of the client that invoked the procedure. pub fn sender(&self) -> Identity { self.sender @@ -1368,7 +1425,7 @@ impl ProcedureContext { /// This includes interior mutability through types like [`std::cell::Cell`]. #[cfg(feature = "unstable")] pub fn with_tx(&mut self, body: impl Fn(&TxContext) -> T) -> T { - with_tx(body) + with_tx(self.sender, self.connection_id, &self.sender_auth, body) } /// Acquire a mutable transaction @@ -1402,7 +1459,7 @@ impl ProcedureContext { /// This includes interior mutability through types like [`std::cell::Cell`]. #[cfg(feature = "unstable")] pub fn try_with_tx(&mut self, body: impl Fn(&TxContext) -> Result) -> Result { - try_with_tx(body) + try_with_tx(self.sender, self.connection_id, &self.sender_auth, body) } /// Create a new random [`Uuid`] `v4` using the built-in RNG. @@ -1547,6 +1604,7 @@ impl Local { /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token #[non_exhaustive] pub struct JwtClaims { + identity: Identity, payload: String, parsed: OnceCell, audience: OnceCell>, @@ -1562,10 +1620,16 @@ pub struct AuthCtx { } impl AuthCtx { - /// Creates an [`AuthCtx`] both for cases where there's a [`ConnectionId`] - /// and for when there isn't. - fn from_connection_id_opt(conn_id: Option) -> Self { - conn_id.map(Self::from_connection_id).unwrap_or_else(Self::internal) + /// Capture host authority immediately. JWT loading remains independent and lazy. + fn from_invocation(sender: Identity, connection_id: Option) -> Self { + let flags = spacetimedb_bindings_sys::get_call_auth_flags(); + Self::from_host_auth(sender, flags, move || connection_id.and_then(rt::get_jwt)) + } + + fn from_host_auth(sender: Identity, flags: u32, jwt_fn: impl FnOnce() -> Option + 'static) -> Self { + Self::new(flags & 1 != 0, move || { + jwt_fn().map(|payload| JwtClaims::new(payload, sender)) + }) } fn new(is_internal: bool, jwt_fn: impl FnOnce() -> Option + 'static) -> Self { @@ -1588,14 +1652,18 @@ impl AuthCtx { /// /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token pub fn from_jwt_payload(jwt_payload: String) -> AuthCtx { - Self::new(false, move || Some(JwtClaims::new(jwt_payload))) + let parsed: serde_json::Value = serde_json::from_str(&jwt_payload).expect("invalid test JWT payload"); + let sender = Identity::from_claims( + parsed["iss"].as_str().expect("missing test issuer"), + parsed["sub"].as_str().expect("missing test subject"), + ); + Self::from_jwt_payload_for_sender(jwt_payload, sender, false) } - /// Creates an [`AuthCtx`] that reads the [JWT] for the given connection id. - /// - /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token - fn from_connection_id(connection_id: ConnectionId) -> AuthCtx { - Self::new(false, move || rt::get_jwt(connection_id).map(JwtClaims::new)) + /// Create test authentication with an explicit effective sender and authority. + /// Production contexts receive these values from the host, not JWT claims. + pub fn from_jwt_payload_for_sender(jwt_payload: String, sender: Identity, is_internal: bool) -> AuthCtx { + Self::new(is_internal, move || Some(JwtClaims::new(jwt_payload, sender))) } /// Returns whether this reducer was spawned from inside the database. @@ -1603,8 +1671,8 @@ impl AuthCtx { self.is_internal } - /// Checks if there is a [JWT] without loading it. - /// If [`AuthCtx::is_internal`] returns true, this will return false. + /// Returns whether this invocation has a [JWT]. Internal invocations may + /// also carry a JWT; internal authority and credential presence are independent. /// /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token pub fn has_jwt(&self) -> bool { @@ -1620,8 +1688,9 @@ impl AuthCtx { } impl JwtClaims { - fn new(jwt: String) -> Self { + fn new(jwt: String, identity: Identity) -> Self { Self { + identity, payload: jwt, parsed: OnceCell::new(), audience: OnceCell::new(), @@ -1663,10 +1732,10 @@ impl JwtClaims { self.audience.get_or_init(|| self.extract_audience()) } - /// Returns the identity for these credentials, which is - /// based on the iss and sub claims. + /// The effective sender verified by the host for this invocation. + /// Hosted database credentials need not derive this Identity from iss/sub. pub fn identity(&self) -> Identity { - Identity::from_claims(self.issuer(), self.subject()) + self.identity } /// Get the whole JWT payload as a json string. @@ -1819,3 +1888,54 @@ mod tests { assert_eq!(audience, &["my-project-id".to_string()]); } } + +#[cfg(test)] +mod hosted_auth_tests { + use super::*; + + #[test] + fn host_authority_is_independent_of_jwt_presence() { + for internal in [false, true] { + for has_jwt in [false, true] { + let flags = u32::from(internal) | (1 << 31); + let auth = AuthCtx::from_host_auth(Identity::ONE, flags, move || { + has_jwt.then(|| r#"{"iss":"hosted","sub":"generation","hex_identity":"untrusted"}"#.to_string()) + }); + assert_eq!(auth.is_internal(), internal); + assert_eq!(auth.has_jwt(), has_jwt); + if let Some(jwt) = auth.jwt() { + assert_eq!(jwt.identity(), Identity::ONE); + assert_ne!(jwt.identity(), Identity::from_claims(jwt.issuer(), jwt.subject())); + } + } + } + } + + #[test] + fn verified_sender_wins_over_signed_payload_identity_claims() { + let payload = format!( + r#"{{"iss":"hosted","sub":"generation","hex_identity":"{}"}}"#, + Identity::ZERO.to_hex() + ); + let auth = AuthCtx::from_host_auth(Identity::ONE, 1, move || Some(payload)); + assert_eq!(auth.jwt().unwrap().identity(), Identity::ONE); + assert!(auth.is_internal()); + assert!(auth.has_jwt()); + } + + #[test] + fn transaction_context_preserves_identity_connection_and_authentication() { + let sender = Identity::ONE; + let connection = Some(ConnectionId::from_u128(123)); + let auth = AuthCtx::from_host_auth(sender, 1, || Some(r#"{"iss":"hosted","sub":"generation"}"#.into())); + // Procedure transactions/retries call this same constructor with the + // original captured AuthCtx rather than manufacturing internal authority. + for timestamp in [Timestamp::UNIX_EPOCH, Timestamp::from_micros_since_unix_epoch(1)] { + let context = ReducerContext::new_with_auth(Local {}, sender, connection, timestamp, auth.clone()); + assert_eq!(context.sender(), sender); + assert_eq!(context.connection_id(), connection); + assert!(context.sender_auth().is_internal()); + assert_eq!(context.sender_auth().jwt().unwrap().identity(), sender); + } + } +} diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 79543f4a925..93ac08983eb 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1,5 +1,7 @@ #![deny(unsafe_op_in_unsafe_fn)] +pub use spacetimedb_lib::db::raw_def::v10::FunctionVisibility; + use crate::query_builder::{FromWhere, HasCols, LeftSemiJoin, RawQuery, RightSemiJoin, Table as QbTable}; use crate::table::IndexAlgo; use crate::{sys, AnonymousViewContext, IterBuf, ReducerContext, ReducerResult, SpacetimeType, Table, ViewContext}; @@ -166,6 +168,9 @@ pub trait FnInfo: ExplicitNames { /// The lifecycle of the function, if there is one. const LIFECYCLE: Option = None; + /// Explicit SpacetimeDB visibility; Rust item visibility is independent. + const DECLARED_VISIBILITY: Option = None; + /// A description of the parameter names of the function. const ARG_NAMES: &'static [Option<&'static str>]; @@ -819,9 +824,13 @@ pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl register_describer(|module| { let params = A::schema::(&mut module.inner); if let Some(lifecycle) = I::LIFECYCLE { - module.inner.add_lifecycle_reducer(lifecycle, I::NAME, params); + module + .inner + .add_lifecycle_reducer_with_visibility(lifecycle, I::NAME, params, I::DECLARED_VISIBILITY); } else { - module.inner.add_reducer(I::NAME, params); + module + .inner + .add_reducer_with_visibility(I::NAME, params, I::DECLARED_VISIBILITY); } module.reducers.push(I::INVOKE); @@ -839,7 +848,9 @@ where register_describer(|module| { let params = A::schema::(&mut module.inner); let ret_ty = ::make_type(&mut module.inner); - module.inner.add_procedure(I::NAME, params, ret_ty); + module + .inner + .add_procedure_with_visibility(I::NAME, params, ret_ty, I::DECLARED_VISIBILITY); module.procedures.push(I::INVOKE); module.inner.add_explicit_names(I::explicit_names()); @@ -995,6 +1006,9 @@ extern "C" fn __describe_module__(description: BytesSink) { describer(&mut module) } + // These bindings capture host flags and preserve the verified sender in JWT claims. + module.inner.add_capability("hosted_auth_v1"); + // Serialize the module to bsatn. let module_def = module.inner.finish(); let module_def = RawModuleDef::V10(module_def); @@ -1334,6 +1348,13 @@ pub fn get_jwt(connection_id: ConnectionId) -> Option { Some(std::str::from_utf8(&buf).unwrap().to_string()) } +pub(crate) fn env_get(key: &str) -> Option { + let source = sys::env_get(key)?; + let mut buf = IterBuf::take(); + read_bytes_source_into(source, &mut buf); + Some(String::from_utf8(buf.to_vec()).expect("host environment values are UTF-8")) +} + /// Read `source` from the host fully into `buf`. pub(crate) fn read_bytes_source_into(source: BytesSource, buf: &mut Vec) { const INVALID: i16 = NO_SUCH_BYTES as i16; diff --git a/crates/bindings/tests/pass/function_visibility.rs b/crates/bindings/tests/pass/function_visibility.rs new file mode 100644 index 00000000000..7f53af5717f --- /dev/null +++ b/crates/bindings/tests/pass/function_visibility.rs @@ -0,0 +1,62 @@ +#![deny(warnings)] + +use spacetimedb::rt::{FnInfo, FunctionVisibility}; +use spacetimedb::{ProcedureContext, ReducerContext}; + +#[spacetimedb::reducer(internal)] +pub fn internal_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(private)] +fn private_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(public)] +fn public_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(init, internal)] +fn initialize(_ctx: &ReducerContext) {} + +#[spacetimedb::procedure(internal)] +fn internal_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +#[spacetimedb::procedure(private)] +fn private_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +#[spacetimedb::procedure(public)] +fn public_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +fn main() { + assert!(matches!( + internal_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + )); + assert!(matches!( + private_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::Private) + )); + assert!(matches!( + public_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::ClientCallable) + )); + assert!(matches!( + initialize::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + )); + assert!(matches!( + internal_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + )); + assert!(matches!( + private_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::Private) + )); + assert!(matches!( + public_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::ClientCallable) + )); +} diff --git a/crates/bindings/tests/ui/tables.stderr b/crates/bindings/tests/ui/tables.stderr index 7609d9ba378..18b61f49224 100644 --- a/crates/bindings/tests/ui/tables.stderr +++ b/crates/bindings/tests/ui/tables.stderr @@ -209,13 +209,13 @@ error[E0277]: `&'a Alpha` cannot appear as an argument to an index filtering ope = note: The allowed set of types are limited to integers, bool, strings, `Identity`, `Uuid`, `Timestamp`, `ConnectionId`, `Hash` and no-payload enums which derive `SpacetimeType`, = help: the following other types implement trait `FilterableValue`: &ConnectionId + &ContainerMode &FunctionVisibility &Identity &Lifecycle - &TableAccess - &TableType - &bool - ðnum::int::I256 + &PortExposure + &PortProtocol + &RestartPolicy and $N others note: required by a bound in `UniqueColumn::::ColType, Col>::find` --> src/table.rs @@ -241,13 +241,13 @@ help: the trait `FilterableValue` is not implemented for `Alpha` | ^^^^^^^^^^^^ = help: the following other types implement trait `FilterableValue`: &ConnectionId + &ContainerMode &FunctionVisibility &Identity &Lifecycle - &TableAccess - &TableType - &bool - ðnum::int::I256 + &PortExposure + &PortProtocol + &RestartPolicy and $N others = note: required for `Alpha` to implement `IndexScanRangeBounds<(Alpha,), SingleBound>` note: required by a bound in `RangedIndex::::filter` diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index bb71fb8b6e5..60e000ed4c6 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -27,6 +27,7 @@ spacetimedb-codegen.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-fs-utils.workspace = true spacetimedb-lib.workspace = true +spacetimedb-oci = { path = "../oci" } spacetimedb-paths.workspace = true spacetimedb-primitives.workspace = true spacetimedb-schema.workspace = true @@ -65,6 +66,8 @@ syntect.workspace = true tabled.workspace = true tar.workspace = true tempfile.workspace = true +sha2 = "0.10" +tokio-util.workspace = true termcolor.workspace = true termtree.workspace = true thiserror.workspace = true @@ -73,6 +76,8 @@ tokio-tungstenite.workspace = true toml.workspace = true toml_edit.workspace = true tracing = { workspace = true, features = ["release_max_level_off"] } +url.workspace = true +uuid = { workspace = true, features = ["std", "serde", "v7"] } walkdir.workspace = true wasmbin.workspace = true webbrowser.workspace = true @@ -91,6 +96,7 @@ notify.workspace = true path-clean = "1.0.1" [dev-dependencies] +axum.workspace = true pretty_assertions.workspace = true fs_extra.workspace = true @@ -98,6 +104,9 @@ fs_extra.workspace = true tikv-jemallocator = { workspace = true } tikv-jemalloc-ctl = { workspace = true } +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +rustix = { version = "1", features = ["fs", "process"] } + [target.'cfg(windows)'.dependencies] windows-sys = { workspace = true, features = ["Win32_System_Console"] } diff --git a/crates/cli/docs/container-build.md b/crates/cli/docs/container-build.md new file mode 100644 index 00000000000..71749eda520 --- /dev/null +++ b/crates/cli/docs/container-build.md @@ -0,0 +1,226 @@ +# Local container image preparation + +`spacetime container build` reads one database's `container` declaration from +`spacetime.json` and prepares a verified local OCI image. It does not resolve a +database through a server or publish anything. The optional `DATABASE` argument +selects an exact local configuration target. Omit it only when the configuration +contains one container target. Container declarations do not pass to children. +The command is dispatched before saved CLI server settings or credentials are +opened; only project configuration and explicitly selected build credentials +are read. + +The same verified preparation feeds managed `publish`, described below. +Container lifecycle commands remain separate integration work. + +## Configuration + +```json +{ + "database": "example", + "container": { + "image": { + "build": { + "builder": "dockerfile", + "context": ".", + "dockerfile": "Dockerfile" + } + }, + "env_keys": ["API_KEY"], + "resources": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824, + "scratch_bytes": 1073741824, + "pids_max": 256 + } + } +} +``` + +`image` accepts exactly one source: + +- `{"build":{"context":"."}}` selects Dockerfile by default. +- `{"build":{"builder":"railpack","context":"."}}` explicitly selects Railpack. +- `{"oci_ref":"registry.example/team/image:tag"}` imports a registry image. +- `{"oci_ref":"oci:./existing-layout"}` imports a local OCI image-layout directory. + +Build contexts and local layout paths are relative to the configuration +directory. The Dockerfile path is relative to its build context. A tag is +resolved once during import; the result records the selected immutable manifest +digest. Registry import requires Skopeo. A local directory import requires no +external image tools. + +The image's `Entrypoint` followed by `Cmd` supplies the command by default. +`command` replaces the complete argv. Image `User` and `WorkingDir` are preserved +unless `user` or `working_directory` overrides them. An omitted or empty image +`WorkingDir` normalizes to `/`. Image `Env` remains in the +immutable configuration blob. `env_keys` contains runtime store references, +never runtime values. Reserved `SPACETIMEDB_` keys are rejected. Mounts must be +empty in Stage 1. Shared resource, startup-string, port, and environment-key +limits apply to the normalized specification. + +## Tools and credentials + +Source builds require an explicitly selected local BuildKit Unix socket: + +```sh +spacetime container build example \ + --project-path ./project \ + --platform linux/amd64 \ + --out-dir ./prepared-image \ + --buildkit-host unix:///absolute/path/to/disposable-buildkit.sock +``` + +Use an endpoint you own and have verified. The command never selects a saved +Docker daemon or BuildKit endpoint. This implementation invokes `buildctl` +directly and currently runs external tools on Linux and macOS. An existing local +OCI directory can also be imported without those subprocesses on Windows. +`--platform` is required and accepts `linux/amd64` or `linux/arm64`; it never +silently uses the build computer's platform. `--out-dir` must not already exist, +and its parent must exist. + +`--buildctl`, `--skopeo`, and `--railpack` select absolute executable paths or +command names on PATH. Missing tools +produce installation/path errors. No executable is automatically downloaded. +Railpack is pinned to `0.35.0`, paired with +`ghcr.io/railwayapp/railpack-frontend:v0.35.0`. Detection or version failures end +the build without changing builders. Its `prepare` output is given to BuildKit's +matching gateway frontend, following the [Railpack production integration](https://railpack.com/platforms/running-railpack-in-production). + +Registry access is anonymous unless `--registry-auth-file FILE` explicitly names +an auth JSON file. The CLI creates an isolated auth directory and does not load +saved Docker credentials. Registry credentials are distinct from Spacetime +credentials and are never copied into prepared metadata. + +Pass build secrets as `--build-secret NAME=FILE`. Files enter BuildKit's secret +interface; Railpack receives only their names during plan generation. Builds +with secrets disable cached build results. Build secrets are separate from +`env_keys`, and prebuilt imports reject them. Credential and secret files are +bounded to 1 MiB each. Subprocess environments are cleared except for basic +executable/temporary-directory paths and the explicitly isolated tool paths. +Do not put secret values in a Dockerfile, image command, or image defaults. + +## Output and failure behavior + +The new output directory contains an OCI image layout plus `prepared.json`: + +- `container` is the normalized container specification. +- `manifest` is the selected immutable manifest descriptor. +- `objects` contains the executable manifest/config/layer closure, with digest, + media type, size, purpose, and a path relative to the layout. + +Prepared metadata omits image environment values and build credentials. The +image configuration blob necessarily retains its image defaults. Consumers must +verify descriptors when reopening output; the local metadata is not an +admission receipt from a server. + +`prepare_container` returns `PreparedContainer`, which owns its temporary +layout. Dropping it removes the temporary output. `persist` transfers the verified +layout with an atomic operation that does not replace an existing output. +Credentials, plans, and intermediate builder files remain in the temporary +workspace and are removed. Failed verification never returns a prepared object. +The shared `Runner` interface permits structured fake builders in tests and +reuse by a later managed publication path. + +Verification checks manifest/config digests, platform, exact compressed layer +digests, uncompressed diff IDs, and bounded tar structure. Layers are inspected +as streams and are never unpacked into the project. Imports reject archive +links, special entries, unknown paths, and duplicate object paths. Bounds include +256 layers, 64 GiB compressed image data, 128 GiB expanded tar data, and one +million expanded entries, with the shared per-object/decompression limits. +Two image tools and two verification workers may run concurrently. Tool calls +have a 30-minute deadline; verification has a 5-minute deadline. Captured stdout +and stderr each have a 4 MiB limit and are not echoed, avoiding accidental +secret disclosure in tool diagnostics. + +Cancellation retains the tool's workspace until its Unix process group has been +signalled and its leader reaped. The group leader remains unreaped until the +signal, preventing reuse of its numeric process-group identity. This cleanup is +for locally trusted tools; it is not a sandbox for a tool that deliberately +escapes its group. BuildKit daemon work is subject to the selected daemon's own +client-disconnect cancellation and retention policies. Blocking verification +retains its worker permit and workspace while checking cancellation between +bounded reads. + +Tests use generated local OCI fixtures and fake tool invocations, including an +owned shell fixture for process cleanup. They do not execute Docker, BuildKit, +Railpack, Skopeo, or any server operation. Actual supported-builder acceptance +remains a separate integration check. + +## Managed publication + +`spacetime publish` publishes a selected target's container declaration through +managed publication. Select the image platform explicitly. Server selection uses +the normal CLI URL, configured alias, or default: + +```sh +spacetime publish my-db --server https://your-test-server.example \ + --container-platform linux/amd64 \ + --artifact-endpoint https://your-test-artifacts.example +``` + +The URLs above are placeholders. Managed transport requires HTTPS for remote +servers and also supports HTTP loopback servers. The CLI does not send the publisher's Bearer credential to an +advertised artifact origin unless it is the same origin as the selected server +or the exact URL is explicitly approved with `--artifact-endpoint`. It does not +follow HTTP redirects or inherit HTTP proxy settings for managed publication. +Image registry credentials remain separate and require `--registry-auth-file`. + +A target with a container declaration and no module source preserves its existing +module. A new container-only database uses the immutable versioned empty module. +An explicit `module-path`, `bin-path`, or `js-path` replaces the module. An omitted +container preserves it; `--remove-container` removes it. `--remove-module` selects +the empty module and runs the existing authorized migration preflight. A selected +container declaration conflicts with `--remove-container`, and a configured +module source conflicts with `--remove-module`. Manual migrations and data-clear +publication are rejected. Precompiled NativeAOT modules can use `--bin-path`; +managed source compilation with `--native-aot` is not yet supported. + +Ordinary module-only publications use the legacy path. When deployment inspection +finds an existing managed revision, module-only updates use managed publication +with `ContainerAction::Keep`. `--managed` explicitly selects managed publication +for a new module-only deployment. A managed error never falls back to a raw +module publication. Existing databases use their exact current deployment +revision as a compare-and-set precondition. + +New managed databases first reserve a server-generated Identity under the +publisher and creation options. A caller cannot select an unreserved new Identity. +`--parent` resolves an accessible existing database; `--organization` currently +requires the organization's Identity. Requested database naming runs separately +after activation. If naming fails or its response is lost, the command reports +the successfully created Identity and does not repeat publication or overwrite +names on a later resume. + +Before staging artifacts, the CLI saves a private operation directory under +`.spacetime/publications/` beside the project configuration, or the explicit +`--publication-state-dir`. The directory retains exact request bytes, immutable +module/OCI artifacts and upload receipts. It contains no publisher credential or +resolved database environment values. Image blobs can contain environment +values baked into the image, so treat the directory as private build output. + +If a request fails or the command is interrupted, use the printed directory: + +```sh +spacetime publish --resume-publication .spacetime/publications/OPERATION_UUID \ + --server https://your-test-server.example \ + --artifact-endpoint https://your-test-artifacts.example +``` + +Resume authenticates the original publisher and server, observes accepted state, +and reuses the exact operation bytes. It does not read `spacetime.json`, rebuild, +or resolve a changed image tag. Ambiguous uploads query the same upload receipt +before continuing. If current database-read access was revoked, resume uses the +exact stored PUT to recover the original publisher's admitted result before +requiring local artifacts. A lost publication response never creates a second +operation. The coordinator PUT has a bounded 30-minute timeout covering its +separate schema, image, storage, and confirmation steps; interruption retains +the journal while the outcome is uncertain. Exact public PUT replay remains +limited to the operation's seven-day retry window; server-side recovery has its +own durable lifetime. +The default activation wait is 60 seconds; `--publication-wait 0` returns after +the first confirmed status, and a pending status prints its resume instruction. +Keep the directory while an outcome is uncertain; a confirmed terminal operation +can be removed locally when its artifacts are no longer needed. + +Container declarations remain local to their exact database target. They do not +inherit to nested targets. `spacetime dev` currently rejects selected container +targets because it does not yet supervise their local runtime. diff --git a/crates/cli/docs/container-operations.md b/crates/cli/docs/container-operations.md new file mode 100644 index 00000000000..3aa680d8167 --- /dev/null +++ b/crates/cli/docs/container-operations.md @@ -0,0 +1,39 @@ +# Container status and lifecycle requests + +`spacetime container status DATABASE` reads the current control state without +opening database storage. Viewer access is sufficient. `--json` returns the +typed status, including desired and observed state, deployment revision, +generation, fixed diagnostics, and public endpoint availability. An absent +current instance is distinct from an old instance's terminal report. Endpoint +allocation may be pending while the rest of the status is available. + +`spacetime container start DATABASE`, `stop DATABASE`, and `restart DATABASE` +require Admin access. Start requests execution, stop requests a stop, and restart +requests a new instance with a fresh environment snapshot. Acceptance records +the desired action; physical stop and readiness complete asynchronously. Use +status to inspect progress. None of these commands changes the published image +or its declaration. + +Each lifecycle request uses a UUIDv7. Before sending it, the CLI prints structured +retry parameters to stderr: the action, resolved database Identity, request ID, +and selected server URL. Keep these parameters if the response is lost. Retry +with that Identity, the same action, `--request-id UUID`, and `--server URL`. +`--request-id` requires an Identity so a changed database name cannot redirect a +retry. Never create a fresh request ID merely to resolve an unknown outcome. + +An accepted exact retry returns the original result generation, even if another +request has since advanced the database generation. Reusing an ID for another +action fails. Current Admin access is checked again on every retry. The retry +window is seven days from the UUID's timestamp. `--json` writes only a verified +receipt to stdout; its generation is a decimal string to preserve all 64 bits. + +Server selection and login use the normal CLI configuration, with an explicit +`--server` override supported on every command. Authenticated requests disable +redirects and inherited proxies. Public `container url` remains anonymous, and +local `container build` continues to run without loading server credentials. + +The HTTP counterparts are authenticated `GET +/v1/database/DATABASE/container/status` and `POST` to the `start`, `stop`, or +`restart` suffix with `{"request_id":"UUID"}`. Successful mutations return HTTP +202. Responses are not cacheable. The operational API rejects hosted container +credentials and uses current ordinary database roles. diff --git a/crates/cli/src/api.rs b/crates/cli/src/api.rs index d40b03ee87e..a608d220335 100644 --- a/crates/cli/src/api.rs +++ b/crates/cli/src/api.rs @@ -4,7 +4,7 @@ use std::ops::Add; use reqwest::{header, Client, RequestBuilder}; use serde::Deserialize; -use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::de::serde::DeserializeWrapper; use spacetimedb_lib::Identity; @@ -61,11 +61,11 @@ impl ClientApi { } /// Reads the `ModuleDef` from the `schema` endpoint. - pub async fn module_def(&self) -> anyhow::Result { + pub async fn module_def(&self) -> anyhow::Result { let res = self .client .get(self.con.db_uri("schema")) - .query(&[("version", "9")]) + .query(&[("version", "10")]) .send() .await?; let DeserializeWrapper(module_def) = res.json_or_error().await?; diff --git a/crates/cli/src/container/config.rs b/crates/cli/src/container/config.rs new file mode 100644 index 00000000000..b9f12a6cf66 --- /dev/null +++ b/crates/cli/src/container/config.rs @@ -0,0 +1,122 @@ +//! Per-database declarations. Values and build credentials are never part of this configuration. +use anyhow::{ensure, Result}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::container::{ + ContainerMode, ContainerMount, ContainerPort, ContainerResources, ContainerSpec, ContainerSpecLimits, + ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_oci::ContainerConfig as ImageConfig; +use std::path::PathBuf; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerConfig { + pub image: ImageSource, + pub command: Option>, + pub user: Option, + pub working_directory: Option, + #[serde(default = "default_mode")] + pub mode: ContainerMode, + #[serde(default = "default_restart")] + pub restart: RestartPolicy, + #[serde(default)] + pub env_keys: Vec, + pub resources: ContainerResources, + #[serde(default)] + pub ports: Vec, + #[serde(default)] + pub mounts: Vec, + #[serde(default = "default_grace")] + pub stop_grace_ms: u32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ImageSource { + Build(BuildImage), + Prebuilt(PrebuiltImage), +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BuildImage { + #[serde(deserialize_with = "source_build")] + pub build: SourceBuild, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PrebuiltImage { + pub oci_ref: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "builder", rename_all = "lowercase", deny_unknown_fields)] +pub enum SourceBuild { + Dockerfile { + #[serde(default = "default_context")] + context: PathBuf, + #[serde(default = "default_dockerfile")] + dockerfile: PathBuf, + }, + Railpack { + #[serde(default = "default_context")] + context: PathBuf, + }, +} + +// A missing builder is Dockerfile. Keep deserialization strict after applying +// that one default, including rejection of Dockerfile-only fields on Railpack. +fn source_build<'de, D: serde::Deserializer<'de>>(deserializer: D) -> std::result::Result { + let mut value = serde_json::Value::deserialize(deserializer)?; + if let Some(object) = value.as_object_mut() { + object.entry("builder").or_insert_with(|| "dockerfile".into()); + } + serde_json::from_value(value).map_err(serde::de::Error::custom) +} +fn default_context() -> PathBuf { + PathBuf::from(".") +} +fn default_dockerfile() -> PathBuf { + PathBuf::from("Dockerfile") +} +fn default_mode() -> ContainerMode { + ContainerMode::Service +} +fn default_restart() -> RestartPolicy { + RestartPolicy::OnFailure +} +fn default_grace() -> u32 { + 30_000 +} + +impl ContainerConfig { + pub fn normalize( + &self, + manifest: OciDigest, + platform: ImagePlatform, + image: &ImageConfig, + ) -> Result { + ensure!(self.mounts.is_empty(), "container mounts are not supported in Stage 1"); + let argv = image.argv(self.command.as_deref())?; + spacetimedb_lib::container::validate_exec_size(&argv, image.env.as_deref().unwrap_or_default())?; + Ok(ContainerSpec { + image_manifest: manifest, + image_platform: platform, + argv, + user: self.user.clone().unwrap_or_else(|| image.user.clone()), + working_directory: self.working_directory.clone().unwrap_or_else(|| { + if image.working_directory.is_empty() { + "/".into() + } else { + image.working_directory.clone() + } + }), + mode: self.mode, + restart: self.restart, + env_keys: self.env_keys.clone(), + resources: self.resources, + ports: self.ports.clone(), + mounts: vec![], + stop_grace_ms: self.stop_grace_ms, + } + .normalize(&ContainerSpecLimits::default())?) + } +} diff --git a/crates/cli/src/container/mod.rs b/crates/cli/src/container/mod.rs new file mode 100644 index 00000000000..02af7005921 --- /dev/null +++ b/crates/cli/src/container/mod.rs @@ -0,0 +1,395 @@ +//! Local image preparation shared by build-only commands and managed publication. +pub mod config; +pub mod oci; +pub mod process; +pub mod publish; + +#[cfg(test)] +pub(crate) mod tests; + +use anyhow::{ensure, Context, Result}; +use config::{ContainerConfig, ImageSource, SourceBuild}; +use oci::{LocalArtifact, VerifiedImage}; +use process::{Invocation, Runner}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::container::{ContainerSpec, ImagePlatform}; +use spacetimedb_oci::Descriptor; +use std::{ + ffi::OsString, + fs, + io::Read, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +pub const RAILPACK_VERSION: &str = "0.35.0"; +pub const RAILPACK_FRONTEND: &str = "ghcr.io/railwayapp/railpack-frontend:v0.35.0"; +const BUILD_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const VERIFY_TIMEOUT: Duration = Duration::from_secs(5 * 60); +static VERIFIERS: std::sync::LazyLock> = + std::sync::LazyLock::new(|| Arc::new(tokio::sync::Semaphore::new(2))); + +pub struct BuildSecret { + pub name: String, + pub file: PathBuf, +} +pub struct BuildTools { + pub buildctl: PathBuf, + pub buildkit_host: Option, + pub railpack: PathBuf, + pub skopeo: PathBuf, + pub registry_auth_file: Option, + pub secrets: Vec, +} +impl Default for BuildTools { + fn default() -> Self { + Self { + buildctl: "buildctl".into(), + buildkit_host: None, + railpack: "railpack".into(), + skopeo: "skopeo".into(), + registry_auth_file: None, + secrets: vec![], + } + } +} + +/// Public metadata contains immutable descriptors and relative paths, not image +/// environment values or credentials. Reopening output must verify the bytes. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PreparedMetadata { + pub version: u32, + pub container: ContainerSpec, + pub manifest: Descriptor, + pub objects: Vec, +} +/// Owns all temporary output. Drop removes it; persist transfers only verified +/// artifacts. A failed/cancelled prepare never yields this type. +pub struct PreparedContainer { + workspace: Arc, + pub metadata: PreparedMetadata, +} +impl PreparedContainer { + pub fn layout(&self) -> PathBuf { + self.workspace.path().join("verified") + } + pub fn persist(self, output: &Path) -> Result<()> { + ensure!( + !output.exists(), + "output directory already exists: {}", + output.display() + ); + // The final transition must not replace a directory created after the + // initial check. The workspace is placed alongside the requested output. + #[cfg(any(target_os = "linux", target_os = "macos"))] + rustix::fs::renameat_with( + rustix::fs::CWD, + self.layout(), + rustix::fs::CWD, + output, + rustix::fs::RenameFlags::NOREPLACE, + ) + .context("could not publish local OCI output without replacing an existing path")?; + #[cfg(windows)] + fs::rename(self.layout(), output) + .context("could not publish local OCI output without replacing an existing path")?; + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + anyhow::bail!("atomic container output is supported on Linux, macOS, and Windows"); + Ok(()) + } +} +struct CancelOnDrop(CancellationToken); +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel(); + } +} +fn path_argument(key: &str, path: &Path) -> Result { + let path = path.to_str().context("image tools require UTF-8 paths")?; + Ok(format!("{key}={path}").into()) +} +fn csv_argument(key: &str, path: &Path) -> Result { + let path = path.to_str().context("image tools require UTF-8 paths")?; + Ok(format!("\"{key}={}\"", path.replace('"', "\"\""))) +} +fn read_credential(path: &Path) -> Result> { + let mut bytes = vec![]; + fs::File::open(path)?.take(1024 * 1024 + 1).read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= 1024 * 1024, + "explicit build credential file exceeds 1 MiB" + ); + Ok(bytes) +} +fn local_buildkit_host(value: Option<&str>) -> Result<&str> { + let value = value.context("source builds require --buildkit-host unix:///absolute/path/to/buildkitd.sock")?; + let path = value + .strip_prefix("unix://") + .context("local source builds require an explicit BuildKit Unix socket")?; + ensure!( + Path::new(path).is_absolute() && !path.contains(['\0', '\n', '\r']), + "invalid local BuildKit socket path" + ); + Ok(value) +} + +pub async fn prepare_container( + declaration: &ContainerConfig, + config_dir: &Path, + platform: ImagePlatform, + tools: &BuildTools, + workspace_parent: &Path, + runner: &impl Runner, + cancel: CancellationToken, +) -> Result { + ensure!( + platform.os == "linux" && matches!(platform.architecture.as_str(), "amd64" | "arm64"), + "choose linux/amd64 or linux/arm64 explicitly" + ); + ensure!( + declaration.mounts.is_empty(), + "container mounts are not supported in Stage 1" + ); + let cancel = cancel.child_token(); + let _cancel = CancelOnDrop(cancel.clone()); + let workspace = Arc::new( + tempfile::Builder::new() + .prefix(".spacetime-image-") + .tempdir_in(workspace_parent)?, + ); + let base = workspace.path(); + fs::create_dir(base.join("auth"))?; + let auth_file = base.join("auth/config.json"); + let auth = tools + .registry_auth_file + .as_deref() + .map(read_credential) + .transpose()? + .unwrap_or_else(|| br#"{"auths":{}}"#.to_vec()); + fs::write(&auth_file, auth)?; + let environment = vec![ + ("DOCKER_CONFIG".into(), base.join("auth").into_os_string()), + ("REGISTRY_AUTH_FILE".into(), auth_file.clone().into_os_string()), + ("XDG_CONFIG_HOME".into(), base.join("tool-config").into_os_string()), + ("XDG_CACHE_HOME".into(), base.join("tool-cache").into_os_string()), + ]; + let invoke = |tool: PathBuf, label, args, cwd: PathBuf| Invocation { + tool, + label, + args, + env: environment.clone(), + cwd, + workspace: workspace.clone(), + timeout: BUILD_TIMEOUT, + cancel: cancel.clone(), + }; + let input = base.join("input"); + let mut archive = None; + match &declaration.image { + ImageSource::Prebuilt(image) => { + ensure!( + tools.secrets.is_empty(), + "build secrets cannot be supplied with a prebuilt image" + ); + if let Some(path) = image.oci_ref.strip_prefix("oci:") { + let path = config_dir + .join(path) + .canonicalize() + .context("prebuilt OCI layout does not exist")?; + ensure!(path.is_dir(), "oci: must name an OCI layout directory"); + return verify(declaration.clone(), platform, workspace.clone(), path, None, cancel).await; + } + ensure!( + !image.oci_ref.is_empty() + && !image.oci_ref.starts_with('-') + && !image.oci_ref.contains(['\0', '\n', '\r']), + "invalid OCI registry reference" + ); + let reference = image.oci_ref.strip_prefix("docker://").unwrap_or(&image.oci_ref); + ensure!( + !reference.contains("://") + && (!reference.contains('@') + || reference.rsplit_once('@').is_some_and(|(_, digest)| digest + .parse::() + .is_ok())), + "invalid OCI registry reference" + ); + let args = vec![ + "--override-os".into(), + platform.os.clone().into(), + "--override-arch".into(), + platform.architecture.clone().into(), + "copy".into(), + "--preserve-digests".into(), + "--authfile".into(), + auth_file.into_os_string(), + format!("docker://{reference}").into(), + format!("oci:{}:prepared", input.display()).into(), + ]; + runner + .run(invoke( + tools.skopeo.clone(), + "Skopeo image import", + args, + config_dir.to_path_buf(), + )) + .await?; + } + ImageSource::Build(image) => { + let endpoint = local_buildkit_host(tools.buildkit_host.as_deref())?; + let (context, dockerfile, railpack) = match &image.build { + SourceBuild::Dockerfile { context, dockerfile } => (context, Some(dockerfile), false), + SourceBuild::Railpack { context } => (context, None, true), + }; + let context = config_dir + .join(context) + .canonicalize() + .context("build context does not exist")?; + ensure!(context.is_dir(), "build context must be a directory"); + let mut secret_arguments = vec![]; + let mut secret_names = std::collections::BTreeSet::new(); + for (index, secret) in tools.secrets.iter().enumerate() { + spacetimedb_lib::container::validate_env_key(&secret.name)?; + ensure!(secret_names.insert(&secret.name), "duplicate build secret name"); + let path = base.join(format!("secret-{index}")); + fs::write(&path, read_credential(&secret.file)?)?; + secret_arguments.extend([ + OsString::from("--secret"), + format!("id={},{}", secret.name, csv_argument("src", &path)?).into(), + ]); + } + let dockerfile = if railpack { + let version = runner + .run(invoke( + tools.railpack.clone(), + "Railpack version check", + vec!["--version".into()], + context.clone(), + )) + .await?; + let version = std::str::from_utf8(&version.stdout).context("invalid Railpack version response")?; + ensure!( + version + .split_whitespace() + .any(|word| word.trim_start_matches('v') == RAILPACK_VERSION), + "install Railpack {RAILPACK_VERSION} to match the pinned frontend" + ); + let plan = base.join("railpack-plan.json"); + let mut args = vec![ + "prepare".into(), + context.clone().into_os_string(), + "--plan-out".into(), + plan.clone().into_os_string(), + "--info-out".into(), + base.join("railpack-info.json").into_os_string(), + ]; + // Only names enter the plan; BuildKit receives the actual files. + for name in secret_names { + args.extend(["--env".into(), format!("{name}=").into()]); + } + runner + .run(invoke( + tools.railpack.clone(), + "Railpack detection", + args, + context.clone(), + )) + .await?; + ensure!( + plan.is_file() && plan.metadata()?.len() <= 4 * 1024 * 1024, + "Railpack did not produce a bounded build plan" + ); + plan + } else { + context + .join(dockerfile.unwrap()) + .canonicalize() + .context("Dockerfile does not exist")? + }; + ensure!( + dockerfile.is_file(), + "Dockerfile or Railpack plan must be a regular file" + ); + let output = base.join("image.tar"); + let mut args = vec![ + "--addr".into(), + endpoint.into(), + "build".into(), + "--frontend".into(), + if railpack { + "gateway.v0".into() + } else { + "dockerfile.v0".into() + }, + "--local".into(), + path_argument("context", &context)?, + "--local".into(), + path_argument("dockerfile", dockerfile.parent().unwrap())?, + "--opt".into(), + path_argument("filename", Path::new(dockerfile.file_name().unwrap()))?, + "--opt".into(), + format!("platform={}/{}", platform.os, platform.architecture).into(), + "--output".into(), + format!("type=oci,{}", csv_argument("dest", &output)?).into(), + ]; + if railpack { + args.extend(["--opt".into(), format!("source={RAILPACK_FRONTEND}").into()]); + } + if !secret_arguments.is_empty() { + args.push("--no-cache".into()); + args.extend(secret_arguments); + } + runner + .run(invoke(tools.buildctl.clone(), "BuildKit OCI build", args, context)) + .await?; + archive = Some(output); + } + } + verify(declaration.clone(), platform, workspace.clone(), input, archive, cancel).await +} + +async fn verify( + declaration: ContainerConfig, + platform: ImagePlatform, + workspace: Arc, + input: PathBuf, + archive: Option, + cancel: CancellationToken, +) -> Result { + let permit = VERIFIERS + .clone() + .try_acquire_owned() + .context("two OCI images are already being verified")?; + let owner = workspace.clone(); + let metadata = tokio::task::spawn_blocking(move || { + let _permit = permit; + let deadline = Instant::now() + VERIFY_TIMEOUT; + if let Some(archive) = archive { + oci::extract_archive(&archive, &input, &cancel, deadline)?; + } + let VerifiedImage { + manifest, + config, + objects, + } = oci::verify_layout(&input, &owner.path().join("verified"), &platform, &cancel, deadline)?; + let container = declaration.normalize(manifest.digest, platform, &config.config)?; + let metadata = PreparedMetadata { + version: 1, + container, + manifest, + objects, + }; + fs::write( + owner.path().join("verified/prepared.json"), + serde_json::to_vec_pretty(&metadata)?, + )?; + Ok::<_, anyhow::Error>(metadata) + }) + .await + .context("OCI verification worker stopped")??; + Ok(PreparedContainer { workspace, metadata }) +} diff --git a/crates/cli/src/container/oci.rs b/crates/cli/src/container/oci.rs new file mode 100644 index 00000000000..3e5ec888a57 --- /dev/null +++ b/crates/cli/src/container/oci.rs @@ -0,0 +1,262 @@ +//! Bounded OCI import. Layers are inspected as streams, never unpacked into the project. +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::container::{ImagePlatform, OciDigest}; +use spacetimedb_oci::{ + self as oci, + layers::{verify_layer_with_check, LayerLimits}, + Descriptor, ImageConfig, +}; +use std::{ + collections::BTreeSet, + fs::{self, File}, + io::{Read, Seek, Write}, + path::{Path, PathBuf}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; + +pub const MAX_EXPANDED_IMAGE_BYTES: u64 = 128 * 1024 * 1024 * 1024; +const MAX_ARCHIVE_ENTRIES: usize = 1024; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactKind { + Manifest, + Config, + Layer, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LocalArtifact { + pub kind: ArtifactKind, + pub descriptor: Descriptor, + /// Relative to the owned OCI layout directory. + pub path: PathBuf, +} +pub(crate) struct VerifiedImage { + pub manifest: Descriptor, + pub config: ImageConfig, + pub objects: Vec, +} + +pub(crate) fn check(cancel: &CancellationToken, deadline: Instant) -> std::io::Result<()> { + if cancel.is_cancelled() || Instant::now() >= deadline { + Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "container preparation cancelled or timed out", + )) + } else { + Ok(()) + } +} +fn blob_path(digest: OciDigest) -> PathBuf { + PathBuf::from("blobs/sha256").join(digest.to_string().strip_prefix("sha256:").expect("SHA-256 digest")) +} +fn bounded_file(path: &Path, limit: u64) -> Result { + ensure!( + fs::symlink_metadata(path)?.is_file(), + "OCI object must be a regular file" + ); + let file = File::open(path)?; + ensure!(file.metadata()?.len() <= limit, "OCI object exceeds its size bound"); + Ok(file) +} +fn read_small(path: &Path, limit: usize) -> Result> { + let mut bytes = vec![]; + bounded_file(path, limit as u64)? + .take(limit as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!(bytes.len() <= limit, "OCI metadata exceeds its size bound"); + Ok(bytes) +} + +pub(crate) fn extract_archive( + archive: &Path, + output: &Path, + cancel: &CancellationToken, + deadline: Instant, +) -> Result<()> { + fs::create_dir_all(output.join("blobs/sha256"))?; + let mut seen = BTreeSet::new(); + let mut total = 0u64; + let archive = bounded_file(archive, oci::MAX_IMAGE_BYTES + 16 * 1024 * 1024)?; + for (index, entry) in tar::Archive::new(archive).entries()?.enumerate() { + check(cancel, deadline)?; + ensure!(index < MAX_ARCHIVE_ENTRIES, "too many OCI archive entries"); + let mut entry = entry?; + let path = entry.path()?.into_owned(); + let name = path.to_str().context("OCI archive path is not UTF-8")?; + if entry.header().entry_type().is_dir() { + ensure!( + matches!(name.trim_end_matches('/'), "." | "blobs" | "blobs/sha256"), + "unexpected OCI archive directory" + ); + continue; + } + ensure!( + entry.header().entry_type().is_file(), + "OCI archive links and special files are unsupported" + ); + let blob = name.strip_prefix("blobs/sha256/"); + ensure!( + matches!(name, "index.json" | "oci-layout") + || blob.is_some_and( + |v| v.len() == 64 && v.bytes().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()) + ), + "unexpected OCI archive path" + ); + ensure!(seen.insert(path.clone()), "duplicate OCI archive path"); + let size = entry.size(); + total = total.checked_add(size).context("OCI archive size overflow")?; + ensure!( + total <= oci::MAX_IMAGE_BYTES + 16 * 1024 * 1024, + "OCI archive exceeds image size bound" + ); + if blob.is_none() { + ensure!(size <= oci::MAX_MANIFEST_BYTES as u64, "OCI archive metadata too large"); + } + let mut file = File::options().write(true).create_new(true).open(output.join(&path))?; + let mut buffer = [0u8; 64 * 1024]; + loop { + check(cancel, deadline)?; + let n = entry.read(&mut buffer)?; + if n == 0 { + break; + } + file.write_all(&buffer[..n])?; + } + } + Ok(()) +} + +pub(crate) fn verify_layout( + input: &Path, + output: &Path, + platform: &ImagePlatform, + cancel: &CancellationToken, + deadline: Instant, +) -> Result { + check(cancel, deadline)?; + let layout: serde_json::Value = serde_json::from_slice(&read_small(&input.join("oci-layout"), 1024)?)?; + ensure!( + layout.get("imageLayoutVersion").and_then(|v| v.as_str()) == Some("1.0.0"), + "unsupported OCI layout version" + ); + let index = read_small(&input.join("index.json"), oci::MAX_MANIFEST_BYTES)?; + // Layout metadata is not a hashed image object and may omit mediaType. + // Defaults apply only here, never to immutable registry index bytes. + let mut index: serde_json::Value = serde_json::from_slice(&index)?; + index + .as_object_mut() + .context("OCI layout index must be an object")? + .entry("mediaType") + .or_insert_with(|| oci::OCI_INDEX.into()); + let index = serde_json::to_vec(&index)?; + let parsed: oci::ImageIndex = serde_json::from_slice(&index)?; + ensure!( + parsed.media_type == oci::OCI_INDEX, + "unsupported OCI layout index media type" + ); + ensure!( + parsed.schema_version == 2 && parsed.manifests.len() <= oci::MAX_INDEX_ENTRIES, + "invalid OCI layout index" + ); + // A local layout index commonly names one manifest without platform metadata. + // The immutable image config below must still match the explicit platform. + let descriptor = if parsed.manifests.len() == 1 && parsed.manifests[0].platform.is_none() { + parsed.manifests.into_iter().next().unwrap() + } else { + oci::select_platform(&index, platform)? + }; + // A layout can name a multi-platform index; select exactly one executable manifest. + let source = read_small(&input.join(blob_path(descriptor.digest)), oci::MAX_MANIFEST_BYTES)?; + oci::verify_object(&descriptor, &source)?; + let (descriptor, bytes) = if matches!(descriptor.media_type.as_str(), oci::OCI_INDEX | oci::DOCKER_INDEX) { + let selected = oci::select_platform(&source, platform)?; + let bytes = read_small(&input.join(blob_path(selected.digest)), oci::MAX_MANIFEST_BYTES)?; + oci::verify_object(&selected, &bytes)?; + (selected, bytes) + } else { + (descriptor, source) + }; + ensure!( + matches!(descriptor.media_type.as_str(), oci::OCI_MANIFEST | oci::DOCKER_MANIFEST), + "OCI layout does not select an executable image" + ); + let manifest = oci::parse_manifest(&bytes)?; + let config_bytes = read_small(&input.join(blob_path(manifest.config.digest)), oci::MAX_CONFIG_BYTES)?; + let config = oci::parse_config(&config_bytes, &manifest, platform)?; + let mut compressed = 0u64; + let mut expanded = 0u64; + let mut entries = 0u64; + for (layer, diff_id) in manifest.layers.iter().zip(&config.rootfs.diff_ids) { + check(cancel, deadline)?; + compressed = compressed.checked_add(layer.size).context("image size overflow")?; + ensure!(compressed <= oci::MAX_IMAGE_BYTES, "compressed image exceeds bound"); + let file = bounded_file(&input.join(blob_path(layer.digest)), layer.size)?; + let size = verify_layer_with_check(file, layer, *diff_id, LayerLimits::default(), || { + check(cancel, deadline) + })?; + expanded = expanded + .checked_add(size.uncompressed_tar_bytes) + .context("expanded image size overflow")?; + entries = entries + .checked_add(size.entries) + .context("image entry count overflow")?; + ensure!( + expanded <= MAX_EXPANDED_IMAGE_BYTES && entries <= 1_000_000, + "expanded image exceeds bound" + ); + } + fs::create_dir_all(output.join("blobs/sha256"))?; + let mut objects = vec![]; + for object in oci::object_closure(descriptor.clone(), &manifest)? { + check(cancel, deadline)?; + let path = blob_path(object.digest); + let mut source = bounded_file(&input.join(&path), object.size)?; + source.rewind()?; + let mut target = File::options().write(true).create_new(true).open(output.join(&path))?; + let mut hash = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + check(cancel, deadline)?; + let n = source.read(&mut buffer)?; + if n == 0 { + break; + } + total = total.checked_add(n as u64).context("object size overflow")?; + ensure!(total <= object.size, "OCI object changed while copying"); + hash.update(&buffer[..n]); + target.write_all(&buffer[..n])?; + } + ensure!( + total == object.size && OciDigest::sha256(hash.finalize().into()) == object.digest, + "OCI object changed while copying" + ); + objects.push(LocalArtifact { + kind: if object.digest == descriptor.digest { + ArtifactKind::Manifest + } else if object.digest == manifest.config.digest { + ArtifactKind::Config + } else { + ArtifactKind::Layer + }, + descriptor: object, + path, + }); + } + fs::write(output.join("oci-layout"), br#"{"imageLayoutVersion":"1.0.0"}"#)?; + fs::write( + output.join("index.json"), + serde_json::to_vec( + &serde_json::json!({"schemaVersion":2,"mediaType":oci::OCI_INDEX,"manifests":[descriptor]}), + )?, + )?; + Ok(VerifiedImage { + manifest: descriptor, + config, + objects, + }) +} diff --git a/crates/cli/src/container/process.rs b/crates/cli/src/container/process.rs new file mode 100644 index 00000000000..3d548833178 --- /dev/null +++ b/crates/cli/src/container/process.rs @@ -0,0 +1,212 @@ +//! Local trusted tools run in a dedicated Unix process group. Cancellation +//! retains the workspace until that group is signalled and its leader reaped. +//! This is process cleanup, not containment of tools that deliberately escape +//! their process group. +use anyhow::Result; +use std::{ffi::OsString, future::Future, path::PathBuf, sync::Arc, time::Duration}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +pub struct Invocation { + pub tool: PathBuf, + pub label: &'static str, + pub args: Vec, + pub env: Vec<(OsString, OsString)>, + pub cwd: PathBuf, + pub workspace: Arc, + pub timeout: Duration, + pub cancel: CancellationToken, +} +pub struct Output { + pub stdout: Vec, +} +pub trait Runner: Sync { + fn run(&self, invocation: Invocation) -> impl Future> + Send; +} +pub struct LocalRunner; + +impl Runner for LocalRunner { + async fn run(&self, invocation: Invocation) -> Result { + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + local::run(invocation).await + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = invocation; + anyhow::bail!( + "local image tools currently require Linux or macOS; import an existing oci: directory instead" + ) + } + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod local { + use super::*; + use anyhow::{bail, ensure, Context}; + use rustix::process::{Pid, WaitIdOptions}; + use std::process::Stdio; + use tokio::{ + io::AsyncReadExt, + process::Command, + sync::{oneshot, Semaphore}, + }; + + static PROCESSES: std::sync::LazyLock> = std::sync::LazyLock::new(|| Arc::new(Semaphore::new(2))); + const MAX_OUTPUT: usize = 4 * 1024 * 1024; + + async fn read_bounded(mut reader: impl tokio::io::AsyncRead + Unpin) -> Result> { + let mut bytes = vec![]; + (&mut reader) + .take(MAX_OUTPUT as u64 + 1) + .read_to_end(&mut bytes) + .await?; + ensure!(bytes.len() <= MAX_OUTPUT, "builder diagnostic output exceeded 4 MiB"); + Ok(bytes) + } + + // Keep the group leader unreaped until after killpg. Its waitable PID pins + // the numeric process-group identity even when it exits before descendants. + struct ProcessGroup(Option); + impl ProcessGroup { + fn kill(&mut self) -> std::io::Result<()> { + if let Some(pid) = self.0.take() { + match rustix::process::kill_process_group(pid, rustix::process::Signal::KILL) { + Ok(()) | Err(rustix::io::Errno::SRCH) => (), + #[cfg(target_os = "macos")] + Err(rustix::io::Errno::PERM) if exited_leader_is_sole_member(pid) => (), + Err(error) => return Err(error.into()), + } + } + Ok(()) + } + async fn observe_exit(&mut self) -> Result<()> { + let pid = self.0.context("local image tool ownership lost")?; + loop { + match rustix::process::waitid( + rustix::process::WaitId::Pid(pid), + WaitIdOptions::EXITED | WaitIdOptions::NOWAIT | WaitIdOptions::NOHANG, + ) { + Ok(Some(_)) => return Ok(()), + Ok(None) | Err(rustix::io::Errno::INTR) => (), + Err(error) => { + // An external reaper invalidates numeric PID ownership. + // Never signal that group after this boundary. + if error == rustix::io::Errno::CHILD { + self.0 = None; + } + return Err(error).context("could not observe local image tool exit"); + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + } + + #[cfg(target_os = "macos")] + fn exited_leader_is_sole_member(pid: Pid) -> bool { + // XNU excludes zombies from killpg's eligible processes, and reports + // EPERM when only an unreaped leader remains. Do not generalize EPERM: + // prove this exact child has exited and its group contains only it. + // WNOWAIT keeps the PID/PGID pinned across the bounded inventory. + if !matches!( + rustix::process::waitid( + rustix::process::WaitId::Pid(pid), + WaitIdOptions::EXITED | WaitIdOptions::NOWAIT | WaitIdOptions::NOHANG, + ), + Ok(Some(_)) + ) { + return false; + } + #[link(name = "proc")] + unsafe extern "C" { + fn proc_listpgrppids( + pgrpid: std::ffi::c_int, + buffer: *mut std::ffi::c_void, + buffersize: std::ffi::c_int, + ) -> std::ffi::c_int; + } + // libproc returns the number of PIDs copied, including zombies. Two + // slots distinguish the sole leader from any extra/truncated members. + let mut members = [0i32; 2]; + // SAFETY: writable aligned storage and its exact byte size are passed; + // the group ID belongs to the still-unreaped child observed above. + let count = unsafe { + proc_listpgrppids( + pid.as_raw_pid(), + members.as_mut_ptr().cast(), + std::mem::size_of_val(&members) as std::ffi::c_int, + ) + }; + count == 1 && members[0] == pid.as_raw_pid() + } + impl Drop for ProcessGroup { + fn drop(&mut self) { + let _ = self.kill(); + } + } + + pub(super) async fn run(invocation: Invocation) -> Result { + let permit = PROCESSES + .clone() + .try_acquire_owned() + .context("two local image tools are already running")?; + let (mut send, receive) = oneshot::channel(); + tokio::spawn(async move { + let _permit = permit; + let result = async { + ensure!(!send.is_closed() && !invocation.cancel.is_cancelled(), "container build cancelled"); + let _workspace = invocation.workspace; + let mut command = Command::new(&invocation.tool); + command.args(&invocation.args).current_dir(&invocation.cwd).env_clear(); + // No implicit registry, Spacetime, proxy or builder credentials. + for key in ["PATH", "SystemRoot", "TMPDIR", "TEMP", "TMP"] { + if let Some(value) = std::env::var_os(key) { command.env(key, value); } + } + command.envs(invocation.env).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped()).process_group(0); + let mut child = command.spawn().with_context(|| format!("could not run {}; install it or provide its executable path", invocation.tool.display()))?; + let mut group = ProcessGroup(Some(Pid::from_raw(child.id().context("builder PID unavailable")? as i32).context("invalid builder PID")?)); + let stdout = child.stdout.take().context("builder stdout unavailable")?; + let stderr = child.stderr.take().context("builder stderr unavailable")?; + let outcome = { + let completion = async { + let (status, stdout, _) = tokio::try_join!( + async { + group.observe_exit().await?; + group.kill().context("failed to stop builder descendants")?; + Ok::<_, anyhow::Error>(child.wait().await?) + }, + read_bounded(stdout), + read_bounded(stderr), + )?; + ensure!(status.success(), "{} failed ({status}); no prepared output was accepted", invocation.label); + Ok(Output { stdout }) + }; + tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("container build caller closed")), + _ = invocation.cancel.cancelled() => Err(anyhow::anyhow!("container build cancelled")), + _ = tokio::time::sleep(invocation.timeout) => Err(anyhow::anyhow!("{} exceeded its build deadline", invocation.label)), + result = completion => result, + } + }; + // Disarmed before every reap, including completion above. Child + // wait is cached if completion already reaped it. + group.kill().context("failed to stop local image tool group")?; + match tokio::time::timeout(Duration::from_secs(5), child.wait()).await { + Ok(result) => { result.context("local image tool could not be reaped")?; }, + Err(_) => { + // The owner retains the workspace and permit throughout + // delayed physical cleanup, even if the caller is gone. + child.wait().await.context("local image tool could not be reaped")?; + bail!("local image tool required delayed physical cleanup"); + } + } + outcome + }.await; + let _ = send.send(result); + }); + receive.await.context("local image tool owner stopped")? + } +} diff --git a/crates/cli/src/container/publish/client.rs b/crates/cli/src/container/publish/client.rs new file mode 100644 index 00000000000..ebea3d20545 --- /dev/null +++ b/crates/cli/src/container/publish/client.rs @@ -0,0 +1,434 @@ +//! Ordinary publisher transport. Credentials are held in memory and never +//! redirected, inherited from a proxy, or sent to an unapproved artifact URL. +use anyhow::{bail, ensure, Context, Result}; +use reqwest::{header::HeaderValue, Client, Method, StatusCode, Url}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use spacetimedb_client_api_messages::name::{DomainName, PrePublishResult, SetDomainsResult}; +use spacetimedb_lib::{container::OciDigest, deployment::api::*, Identity, Uuid}; +use std::time::Duration; + +pub const MAX_RESPONSE_BYTES: usize = 1024 * 1024; +// A submission composes the 120s schema worker, 300s image verification, +// two independently bounded 300s artifact writes, and read/pin/control calls. +// This caller deadline does not extend those server resource or credential +// bounds. Cancellation/timeout leaves the exact operation in its journal. +const COORDINATOR_REQUEST_TIMEOUT: Duration = Duration::from_secs(30 * 60); +pub const UPLOAD_CHUNK_BYTES: usize = 8 * 1024 * 1024; + +#[derive(Debug, thiserror::Error)] +#[error("{action} returned HTTP {status}")] +pub struct HttpFailure { + pub action: &'static str, + pub status: StatusCode, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UploadKind { + Manifest, + Config, + Layer, + Module, +} +impl UploadKind { + fn header(self) -> &'static str { + match self { + Self::Manifest => "manifest", + Self::Config => "config", + Self::Layer => "layer", + Self::Module => "module", + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObjectRef { + pub digest: OciDigest, + pub size: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UploadStatus { + pub id: uuid::Uuid, + pub object: ObjectRef, + pub offset: u64, + pub expires_at: u64, + pub complete: bool, +} +impl UploadStatus { + pub fn validate(&self, object: ObjectRef, id: Option) -> Result<()> { + ensure!( + self.id.get_version_num() == 4 && id.is_none_or(|id| self.id == id), + "artifact upload session changed" + ); + ensure!( + self.object == object && self.offset <= object.size && (!self.complete || self.offset == object.size), + "artifact upload descriptor or offset changed" + ); + Ok(()) + } +} + +pub struct ArtifactEndpoint(Url); +impl ArtifactEndpoint { + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +pub struct PublisherClient { + http: Client, + server: Url, + authorization: HeaderValue, +} +impl PublisherClient { + pub fn new(server: &str, mut authorization: HeaderValue) -> Result { + let server = endpoint(server)?; + ensure!( + authorization.as_bytes().starts_with(b"Bearer "), + "managed publication requires ordinary Bearer authentication" + ); + authorization.set_sensitive(true); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { + http, + server, + authorization, + }) + } + pub fn server(&self) -> &Url { + &self.server + } + fn request(&self, method: Method, url: Url) -> reqwest::RequestBuilder { + self.http + .request(method, url) + .header(reqwest::header::AUTHORIZATION, self.authorization.clone()) + } + /// A response cannot grant itself permission to receive this credential. + /// Same-origin paths are accepted; another origin requires an explicit URL. + pub fn artifact_endpoint(&self, advertised: &str, approved: Option<&str>) -> Result { + let artifact = endpoint(advertised)?; + if artifact.origin() != self.server.origin() { + let approved=approved.context("artifact service uses another origin; pass --artifact-endpoint with its exact trusted URL before sending publisher credentials")?; + ensure!( + endpoint(approved)? == artifact, + "advertised artifact endpoint differs from the explicitly approved endpoint" + ); + } else if let Some(approved) = approved { + ensure!( + endpoint(approved)? == artifact, + "advertised artifact endpoint differs from the explicitly approved endpoint" + ); + } + Ok(ArtifactEndpoint(artifact)) + } + pub async fn capabilities(&self) -> Result> { + let response = self + .http + .get(route(&self.server, &["v1", "containers", "capabilities"])?) + .send() + .await?; + optional_json(response, "publication capabilities").await + } + pub async fn permission(&self) -> Result { + json( + self.request( + Method::GET, + route(&self.server, &["v1", "containers", "publish-permission"])?, + ) + .send() + .await?, + "publication permission", + ) + .await + } + pub async fn deployment(&self, name: &str) -> Result> { + optional_json( + self.request( + Method::GET, + route(&self.server, &["v1", "database", name, "deployment"])?, + ) + .send() + .await?, + "deployment inspection", + ) + .await + } + pub async fn reserve(&self, request: &ReserveDatabaseRequest) -> Result { + json( + self.request( + Method::POST, + route(&self.server, &["v1", "containers", "reservations"])?, + ) + .json(request) + .send() + .await?, + "database reservation", + ) + .await + } + pub async fn submit(&self, database: Identity, request: &PublishRequest) -> Result { + let bytes = serde_json::to_vec(request)?; + self.submit_bytes(database, &bytes).await + } + /// Resume sends these same immutable bytes, without reconstructing a request + /// from a changed tag, project configuration, or current deployment. + pub async fn submit_bytes(&self, database: Identity, bytes: &[u8]) -> Result { + ensure!(bytes.len() <= 256 * 1024, "publication HTTP body exceeds 256 KiB"); + json( + self.request( + Method::PUT, + route(&self.server, &["v1", "database", &database.to_hex(), "deployment"])?, + ) + .timeout(COORDINATOR_REQUEST_TIMEOUT) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(bytes.to_vec()) + .send() + .await?, + "publication submission", + ) + .await + } + pub async fn status(&self, database: Identity, operation: Uuid) -> Result> { + optional_json( + self.request( + Method::GET, + route( + &self.server, + &[ + "v1", + "database", + &database.to_hex(), + "deployment", + "operations", + &operation.to_string(), + ], + )?, + ) + .send() + .await?, + "publication status", + ) + .await + } + pub async fn preflight(&self, database: Identity, kind: &str, bytes: &[u8]) -> Result { + json( + self.request( + Method::POST, + route(&self.server, &["v1", "database", &database.to_hex(), "pre_publish"])?, + ) + .query(&[("host_type", kind), ("pretty_print_style", "NoColor")]) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(bytes.to_vec()) + .send() + .await?, + "module migration preflight", + ) + .await + } + pub async fn set_name(&self, database: Identity, name: &str) -> Result<()> { + let name: DomainName = name.parse()?; + let result: SetDomainsResult = json( + self.request( + Method::PUT, + route(&self.server, &["v1", "database", &database.to_hex(), "names"])?, + ) + .json(&[name]) + .send() + .await?, + "database naming", + ) + .await?; + ensure!( + matches!(result, SetDomainsResult::Success), + "database was created, but assigning its requested name was not confirmed" + ); + Ok(()) + } + pub async fn begin_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + kind: UploadKind, + object: ObjectRef, + ) -> Result { + let status: UploadStatus = json( + self.request( + Method::POST, + route(&artifact.0, &["v1", "databases", &database.to_hex(), "uploads"])?, + ) + .header("x-spacetimedb-artifact-kind", kind.header()) + .json(&serde_json::json!({"kind":kind,"object":object})) + .send() + .await?, + "artifact upload creation", + ) + .await?; + status.validate(object, None)?; + Ok(status) + } + pub async fn upload_status( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + id: uuid::Uuid, + object: ObjectRef, + ) -> Result { + let status: UploadStatus = json( + self.request( + Method::GET, + route( + &artifact.0, + &["v1", "databases", &database.to_hex(), "uploads", &id.to_string()], + )?, + ) + .send() + .await?, + "artifact upload status", + ) + .await?; + status.validate(object, Some(id))?; + Ok(status) + } + pub async fn append_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + status: &UploadStatus, + chunk: Vec, + ) -> Result { + ensure!( + !chunk.is_empty() + && chunk.len() <= UPLOAD_CHUNK_BYTES + && status + .offset + .checked_add(chunk.len() as u64) + .is_some_and(|end| end <= status.object.size), + "invalid artifact chunk" + ); + let next: UploadStatus = json( + self.request( + Method::PATCH, + route( + &artifact.0, + &["v1", "databases", &database.to_hex(), "uploads", &status.id.to_string()], + )?, + ) + .query(&[("offset", status.offset)]) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(chunk) + .send() + .await?, + "artifact upload append", + ) + .await?; + next.validate(status.object, Some(status.id))?; + Ok(next) + } + pub async fn complete_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + status: &UploadStatus, + ) -> Result { + let completed: ObjectRef = json( + self.request( + Method::POST, + route( + &artifact.0, + &[ + "v1", + "databases", + &database.to_hex(), + "uploads", + &status.id.to_string(), + "complete", + ], + )?, + ) + .send() + .await?, + "artifact upload completion", + ) + .await?; + // The completion endpoint confirms the immutable object, rather than + // returning session status. The route already binds the original UUID; + // retain that session only after checking the exact digest and size. + ensure!(completed == status.object, "artifact completion descriptor changed"); + let mut next = status.clone(); + next.offset = completed.size; + next.complete = true; + next.validate(status.object, Some(status.id))?; + Ok(next) + } +} + +pub(crate) fn is_loopback(url: &Url) -> bool { + match url.host() { + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + Some(url::Host::Domain(domain)) => domain.eq_ignore_ascii_case("localhost"), + None => false, + } +} +pub fn endpoint(value: &str) -> Result { + let mut url = Url::parse(value).context("resolved server URL must use HTTP(S)")?; + ensure!( + url.username().is_empty() && url.password().is_none() && url.query().is_none() && url.fragment().is_none(), + "endpoint must not contain credentials, query, or fragment" + ); + let local = is_loopback(&url); + ensure!( + url.scheme() == "https" || (url.scheme() == "http" && local), + "use HTTPS, or HTTP for a local loopback server" + ); + ensure!(url.host_str().is_some(), "endpoint host is missing"); + if !url.path().ends_with('/') { + let path = format!("{}/", url.path()); + url.set_path(&path); + } + Ok(url) +} +fn route(base: &Url, segments: &[&str]) -> Result { + let mut url = base.clone(); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid base URL"))? + .pop_if_empty() + .extend(segments); + Ok(url) +} +async fn optional_json(response: reqwest::Response, action: &'static str) -> Result> { + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + json(response, action).await.map(Some) +} +async fn json(mut response: reqwest::Response, action: &'static str) -> Result { + if !response.status().is_success() { + return Err(HttpFailure { + action, + status: response.status(), + } + .into()); + } + if response + .content_length() + .is_some_and(|length| length > MAX_RESPONSE_BYTES as u64) + { + bail!("{action} response exceeds its bound"); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + chunk.len() <= MAX_RESPONSE_BYTES.saturating_sub(bytes.len()), + "{action} response exceeds its bound" + ); + bytes.extend_from_slice(&chunk); + } + serde_json::from_slice(&bytes).with_context(|| format!("invalid {action} response")) +} diff --git a/crates/cli/src/container/publish/journal.rs b/crates/cli/src/container/publish/journal.rs new file mode 100644 index 00000000000..adb8f9521d8 --- /dev/null +++ b/crates/cli/src/container/publish/journal.rs @@ -0,0 +1,336 @@ +//! A publication resume record contains exact request bytes and object +//! descriptors, never credentials or resolved runtime environment values. +use super::client::{ObjectRef, UploadKind, UploadStatus}; +use crate::container::{oci, PreparedContainer}; +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::{ + container::OciDigest, + deployment::{api::*, PUBLISH_PROTOCOL_VERSION}, + Identity, Uuid, +}; +use std::{ + fs::{self, File}, + io::{Read, Write}, + path::{Path, PathBuf}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; + +const MAX_RECORD_BYTES: usize = 1024 * 1024; +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UploadRecord { + pub kind: UploadKind, + pub object: ObjectRef, + pub session: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Record { + pub version: u32, + pub server: String, + pub artifact_endpoint: String, + pub publisher: Identity, + pub database: Option, + pub reservation: Option, + pub requested_name: Option, + /// UTF-8 JSON sent verbatim on every submission attempt. + pub request_json: String, + pub request_digest: OciDigest, + pub uploads: Vec, + pub submitted: bool, + pub status: Option, + pub name_confirmed: bool, + pub naming_attempted: bool, +} +impl Record { + pub fn request(&self) -> Result { + ensure!( + self.version == 1 && self.request_json.len() <= 256 * 1024, + "unsupported or oversized publication resume record" + ); + ensure!( + spacetimedb_oci::sha256(self.request_json.as_bytes()) == self.request_digest, + "publication request bytes changed" + ); + let request: PublishRequest = serde_json::from_str(&self.request_json)?; + let envelope = &request.manifest.current().envelope; + ensure!( + envelope.version == PUBLISH_PROTOCOL_VERSION, + "unsupported publication protocol" + ); + ensure!( + envelope.operation_id.get_version() == Some(spacetimedb_lib::sats::uuid::Version::V7), + "publication operation must be UUIDv7" + ); + ensure!(self.uploads.len() <= 259, "too many retained publication objects"); + if let Some(reservation) = &self.reservation { + ensure!( + reservation.operation_id == envelope.operation_id + && reservation.version == PUBLISH_PROTOCOL_VERSION + && request.creation.as_ref() == Some(&reservation.options), + "reservation differs from the immutable publication request" + ); + } else { + ensure!( + self.database.is_some() && request.creation.is_none(), + "publication database binding is missing" + ); + } + for upload in &self.uploads { + ensure!(upload.object.size > 0, "empty publication artifact"); + if let Some(status) = &upload.session { + status.validate(upload.object, None)?; + } + } + if let Some(status) = &self.status { + self.check_status(&request, status)?; + } + Ok(request) + } + pub fn check_status(&self, request: &PublishRequest, status: &PublicationStatus) -> Result<()> { + let envelope = &request.manifest.current().envelope; + ensure!( + Some(status.database_identity) == self.database + && status.operation_id == envelope.operation_id + && status.expected_revision == envelope.expected_revision + && status.proposed_revision == request.manifest.current().deployment.revision()?, + "publication response belongs to another operation or deployment" + ); + Ok(()) + } +} + +pub struct Journal { + directory: PathBuf, + _lock: File, + pub record: Record, +} +struct IncompleteDirectory(Option); +impl Drop for IncompleteDirectory { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + let _ = fs::remove_dir_all(path); + } + } +} +impl Journal { + pub fn create( + base: &Path, + record: Record, + image: Option, + module: Option<&[u8]>, + ) -> Result { + let request = record.request()?; + let has_image = image.is_some(); + let directory = base.join(request.manifest.current().envelope.operation_id.to_string()); + fs::create_dir_all(base)?; + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder + .create(&directory) + .context("publication resume directory already exists; resume it rather than reusing its operation")?; + let mut incomplete = IncompleteDirectory(Some(directory.clone())); + let lock = Self::lock(&directory, true)?; + if let Some(image) = image { + image.persist(&directory.join("image"))?; + } + if let Some(module) = module { + let mut file = File::options() + .write(true) + .create_new(true) + .open(directory.join("module.blob"))?; + file.write_all(module)?; + file.sync_all()?; + } + let journal = Self { + directory, + _lock: lock, + record, + }; + journal.sync_retained_artifacts(has_image)?; + journal.save()?; + sync_directory_chain(&journal.directory)?; + incomplete.0 = None; + Ok(journal) + } + pub fn open(directory: &Path) -> Result { + let directory = directory + .canonicalize() + .context("publication resume directory not found")?; + let lock = Self::lock(&directory, false)?; + let path = directory.join("publication.json"); + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "publication resume record must be a regular file" + ); + let mut bytes = vec![]; + File::open(path)? + .take(MAX_RECORD_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= MAX_RECORD_BYTES, + "publication resume record is too large" + ); + let record: Record = serde_json::from_slice(&bytes)?; + record.request()?; + // Creation saves JSON only after artifact flush. Reconfirm the parent + // link if its creator stopped before finishing that last barrier. + sync_directory_chain(&directory)?; + Ok(Self { + directory, + _lock: lock, + record, + }) + } + fn sync_retained_artifacts(&self, has_image: bool) -> Result<()> { + let mut files = self + .record + .uploads + .iter() + .map(|upload| self.object_path(upload)) + .collect::>(); + if has_image { + files.extend( + ["oci-layout", "index.json", "prepared.json"].map(|name| self.directory.join("image").join(name)), + ); + } + let mut directories = std::collections::BTreeSet::new(); + for path in files { + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "retained publication artifact must be a regular file" + ); + // Write access also lets Windows flush an owned immutable artifact. + File::options().read(true).write(true).open(&path)?.sync_all()?; + let mut parent = path.parent(); + while let Some(path) = parent { + directories.insert(path.to_owned()); + if path == self.directory { + break; + } + parent = path.parent(); + } + } + // Flush children before the directories that link them. + for directory in directories.into_iter().rev() { + sync_directory(&directory)?; + } + Ok(()) + } + fn lock(directory: &Path, create: bool) -> Result { + let file = File::options() + .read(true) + .write(true) + .create_new(create) + .open(directory.join("publication.lock"))?; + file.try_lock() + .context("another process is using this publication resume directory")?; + Ok(file) + } + pub fn directory(&self) -> &Path { + &self.directory + } + pub fn operation(&self) -> Result { + Ok(self.record.request()?.manifest.current().envelope.operation_id) + } + pub fn save(&self) -> Result<()> { + self.record.request()?; + let bytes = serde_json::to_vec_pretty(&self.record)?; + ensure!( + bytes.len() <= MAX_RECORD_BYTES, + "publication resume record is too large" + ); + let mut file = tempfile::NamedTempFile::new_in(&self.directory)?; + file.write_all(&bytes)?; + file.as_file().sync_all()?; + file.persist(self.directory.join("publication.json"))?; + #[cfg(unix)] + File::open(&self.directory)?.sync_all()?; + Ok(()) + } + pub fn object_path(&self, upload: &UploadRecord) -> PathBuf { + match upload.kind { + UploadKind::Module => self.directory.join("module.blob"), + _ => self.directory.join("image/blobs/sha256").join( + upload + .object + .digest + .to_string() + .strip_prefix("sha256:") + .expect("SHA256"), + ), + } + } + /// Reopening state never trusts stale local object bytes. This repeats only + /// their digest check; server preparation remains the admission authority. + pub async fn verify_objects(&self, cancel: CancellationToken) -> Result<()> { + let permit = crate::container::VERIFIERS + .clone() + .try_acquire_owned() + .context("two OCI verifications are already running")?; + let objects = self + .record + .uploads + .iter() + .map(|upload| (self.object_path(upload), upload.object)) + .collect::>(); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let deadline = Instant::now() + crate::container::VERIFY_TIMEOUT; + for (path, object) in objects { + oci::check(&cancel, deadline)?; + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "retained publication artifact must be a regular file" + ); + let mut file = File::open(path)?; + ensure!(file.metadata()?.len() == object.size, "retained artifact size changed"); + let mut hash = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + oci::check(&cancel, deadline)?; + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + total = total.checked_add(read as u64).context("artifact size overflow")?; + ensure!(total <= object.size, "retained artifact grew"); + hash.update(&buffer[..read]); + } + ensure!( + total == object.size && OciDigest::sha256(hash.finalize().into()) == object.digest, + "retained publication artifact digest changed" + ); + } + Ok::<_, anyhow::Error>(()) + }) + .await + .context("publication verification worker stopped")??; + Ok(()) + } +} + +fn sync_directory(directory: &Path) -> Result<()> { + #[cfg(unix)] + File::open(directory)?.sync_all()?; + // As in paths::utils::write_atomic, Windows directory handles cannot be + // synced through std. File flushes still precede publishing the journal. + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} +fn sync_directory_chain(directory: &Path) -> Result<()> { + let canonical = directory.canonicalize()?; + for ancestor in canonical.ancestors() { + sync_directory(ancestor)?; + } + Ok(()) +} diff --git a/crates/cli/src/container/publish/mod.rs b/crates/cli/src/container/publish/mod.rs new file mode 100644 index 00000000000..35ac356322e --- /dev/null +++ b/crates/cli/src/container/publish/mod.rs @@ -0,0 +1,315 @@ +//! Managed publication uses one immutable operation and a durable local resume +//! directory. Ambiguous HTTP responses never select a new operation or fall +//! back to the legacy module publication endpoint. +pub mod client; +pub mod journal; + +#[cfg(test)] +pub(crate) mod tests; + +use anyhow::{bail, ensure, Context, Result}; +use client::{ArtifactEndpoint, PublisherClient, UploadStatus, UPLOAD_CHUNK_BYTES}; +use journal::Journal; +use spacetimedb_lib::deployment::{ + api::{PublicationPhase, PublicationStatus}, + operation_expiry_ms, +}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio_util::sync::CancellationToken; + +#[derive(Debug)] +pub enum Outcome { + Complete(PublicationStatus), + Pending(PublicationStatus), + Aborted(PublicationStatus), + /// Database creation succeeded. Never describe a naming failure as a + /// rollback or send another publication operation to compensate for it. + NamingUnconfirmed(PublicationStatus), +} + +pub async fn run( + client: &PublisherClient, + journal: &mut Journal, + approved_artifact: Option<&str>, + wait: Duration, + cancel: CancellationToken, +) -> Result { + let cancel = cancel.child_token(); + let _cancel = crate::container::CancelOnDrop(cancel.clone()); + ensure!( + client.server().as_str() == journal.record.server, + "resume server differs from the original publication endpoint" + ); + let artifact = client.artifact_endpoint(&journal.record.artifact_endpoint, approved_artifact)?; + let request = journal.record.request()?; + let operation = request.manifest.current().envelope.operation_id; + let permission = client.permission().await?; + ensure!( + permission.identity == journal.record.publisher, + "resume publisher differs from the original publication identity" + ); + + // Observe an already admitted operation before requiring original local + // object files or applying today's new-publication permission/limits. + if journal.record.submitted + && let Some(database) = journal.record.database + { + match observe_or_replay(client, journal, database, operation).await { + Ok(Some(status)) => { + journal.record.check_status(&request, &status)?; + journal.record.status = Some(status); + journal.save()?; + return wait_and_name(client, journal, wait, &cancel).await; + } + Ok(None) => (), + Err(error) => { + return Err(error).context("publication observation was not confirmed; resume the same directory") + } + } + } + let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; + operation_expiry_ms(operation, now)?; + ensure!( + !cancel.is_cancelled(), + "publication cancelled; resume directory retained" + ); + if journal.record.database.is_none() { + let reservation = client + .reserve( + journal + .record + .reservation + .as_ref() + .context("creation reservation is missing")?, + ) + .await?; + ensure!( + reservation.operation_id == operation && reservation.staging_open, + "reservation was not confirmed for this operation" + ); + let returned = client.artifact_endpoint(&reservation.artifact_endpoint, approved_artifact)?; + ensure!( + returned.as_str() == artifact.as_str(), + "reservation changed the selected artifact endpoint" + ); + journal.record.database = Some(reservation.database_identity); + journal.save()?; + } + journal.verify_objects(cancel.clone()).await?; + for index in 0..journal.record.uploads.len() { + upload(client, &artifact, journal, index, &cancel).await?; + } + ensure!( + !cancel.is_cancelled(), + "publication cancelled; resume directory retained" + ); + // This marker must be durable before sending the admission request. A crash + // after the server commits but before the response can then observe/replay. + journal.record.submitted = true; + journal.save()?; + let status = client + .submit_bytes(journal.record.database.unwrap(), journal.record.request_json.as_bytes()) + .await + .context("publication outcome is not confirmed; resume the same directory")?; + journal.record.check_status(&request, &status)?; + journal.record.status = Some(status); + journal.save()?; + wait_and_name(client, journal, wait, &cancel).await +} + +/// Operation inspection has current database-read authorization. Exact PUT +/// replay instead authenticates the original admitted publisher and immutable +/// request. It must precede local artifact/expiry work after read revocation. +async fn observe_or_replay( + client: &PublisherClient, + journal: &Journal, + database: spacetimedb_lib::Identity, + operation: spacetimedb_lib::Uuid, +) -> Result> { + match client.status(database, operation).await { + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.status == reqwest::StatusCode::FORBIDDEN) => + { + client + .submit_bytes(database, journal.record.request_json.as_bytes()) + .await + .map(Some) + .context("exact publication replay was not confirmed; keep the same resume directory") + } + result => result, + } +} + +async fn upload( + client: &PublisherClient, + artifact: &ArtifactEndpoint, + journal: &mut Journal, + index: usize, + cancel: &CancellationToken, +) -> Result<()> { + let database = journal.record.database.context("reservation missing")?; + let item = journal.record.uploads[index].clone(); + let mut status = if let Some(prior) = item.session { + match client.upload_status(artifact, database, prior.id, item.object).await { + Ok(status) => status, + Err(error) + if error.downcast_ref::().is_some_and(|error| { + matches!(error.status, reqwest::StatusCode::NOT_FOUND | reqwest::StatusCode::GONE) + }) => + { + client.begin_upload(artifact, database, item.kind, item.object).await? + } + Err(error) => return Err(error), + } + } else { + client.begin_upload(artifact, database, item.kind, item.object).await? + }; + store_upload(journal, index, &status)?; + if status.complete { + return Ok(()); + } + let mut file = tokio::fs::File::open(journal.object_path(&journal.record.uploads[index])).await?; + let mut ambiguous_attempts = 0; + while status.offset < status.object.size { + ensure!( + !cancel.is_cancelled(), + "artifact upload cancelled; resume directory retained" + ); + file.seek(std::io::SeekFrom::Start(status.offset)).await?; + let len = usize::try_from((status.object.size - status.offset).min(UPLOAD_CHUNK_BYTES as u64))?; + let mut bytes = vec![0; len]; + file.read_exact(&mut bytes) + .await + .context("retained artifact was truncated")?; + let expected = status.offset + len as u64; + match client.append_upload(artifact, database, &status, bytes).await { + Ok(next) => { + ensure!( + next.offset == expected, + "artifact append acknowledged an unexpected offset" + ); + status = next; + ambiguous_attempts = 0; + } + Err(error) => { + if !retryable(&error) { + return Err(error); + } + ambiguous_attempts += 1; + ensure!( + ambiguous_attempts <= 3, + "artifact append outcome remains unknown; resume the same directory" + ); + let next = client + .upload_status(artifact, database, status.id, status.object) + .await?; + ensure!( + next.offset == status.offset || next.offset == expected, + "artifact offset changed outside this append" + ); + status = next; + } + } + store_upload(journal, index, &status)?; + } + status = match client.complete_upload(artifact, database, &status).await { + Ok(status) => status, + Err(error) if retryable(&error) => { + let observed = client + .upload_status(artifact, database, status.id, status.object) + .await?; + if observed.complete { + observed + } else { + return Err(error).context("artifact completion is not confirmed; resume the same directory"); + } + } + Err(error) => return Err(error), + }; + store_upload(journal, index, &status) +} +fn store_upload(journal: &mut Journal, index: usize, status: &UploadStatus) -> Result<()> { + journal.record.uploads[index].session = Some(status.clone()); + journal.save() +} +fn retryable(error: &anyhow::Error) -> bool { + error + .downcast_ref::() + .is_some_and(|error| error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()) + || error.downcast_ref::().is_some_and(|error| { + error.status.is_server_error() + || matches!( + error.status, + reqwest::StatusCode::CONFLICT | reqwest::StatusCode::TOO_MANY_REQUESTS + ) + }) +} +async fn wait_and_name( + client: &PublisherClient, + journal: &mut Journal, + wait: Duration, + cancel: &CancellationToken, +) -> Result { + let deadline = tokio::time::Instant::now() + wait; + let request = journal.record.request()?; + loop { + let status = journal + .record + .status + .clone() + .context("confirmed publication status missing")?; + match status.phase { + PublicationPhase::AbortedBeforeCommit => return Ok(Outcome::Aborted(status)), + PublicationPhase::Complete => { + if let Some(name) = journal + .record + .requested_name + .clone() + .filter(|_| !journal.record.name_confirmed) + { + // Existing name replacement has no CAS token. Do not retry an + // ambiguous name update and overwrite aliases added later. + if journal.record.naming_attempted { + return Ok(Outcome::NamingUnconfirmed(status)); + } + journal.record.naming_attempted = true; + journal.save()?; + if client.set_name(status.database_identity, &name).await.is_err() { + return Ok(Outcome::NamingUnconfirmed(status)); + } + journal.record.name_confirmed = true; + journal.save()?; + } + return Ok(Outcome::Complete(status)); + } + _ => (), + } + if tokio::time::Instant::now() >= deadline { + return Ok(Outcome::Pending(status)); + } + tokio::select! { + _=cancel.cancelled()=>bail!("publication wait cancelled; its durable operation continues, resume the same directory"), + _=tokio::time::sleep_until((tokio::time::Instant::now()+Duration::from_secs(1)).min(deadline))=>(), + } + let next = tokio::time::timeout_at( + deadline, + observe_or_replay(client, journal, status.database_identity, status.operation_id), + ) + .await; + match next { + Ok(Ok(Some(next))) => { + journal.record.check_status(&request, &next)?; + journal.record.status = Some(next); + journal.save()?; + } + Ok(Ok(None)) => bail!("confirmed publication disappeared; keep the same operation and resume directory"), + Ok(Err(error)) => { + return Err(error).context("publication status is not confirmed; resume the same directory") + } + Err(_) => return Ok(Outcome::Pending(status)), + } + } +} diff --git a/crates/cli/src/container/publish/tests.rs b/crates/cli/src/container/publish/tests.rs new file mode 100644 index 00000000000..6c9d04ce73a --- /dev/null +++ b/crates/cli/src/container/publish/tests.rs @@ -0,0 +1,628 @@ +//! These fixtures own numeric-loopback sockets and in-memory credentials. They +//! never load CLI configuration, environment endpoints, or external builders. +use super::*; +use axum::{ + body::Bytes, + extract::State, + http::{HeaderMap, Method, StatusCode, Uri}, + response::{IntoResponse, Response}, + Json, Router, +}; +use client::{ObjectRef, UploadKind, UploadStatus}; +use journal::{Record, UploadRecord}; +use serde_json::json; +use spacetimedb_lib::{ + deployment::{api::*, manifest::*, *}, + Identity, Uuid, +}; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +const TOKEN: &str = "Bearer isolated-fixture-credential"; +pub(crate) fn publisher() -> Identity { + Identity::from_be_byte_array([42; 32]) +} +pub(crate) fn database() -> Identity { + Identity::from_be_byte_array([43; 32]) +} +#[derive(Default)] +pub(crate) struct Behavior { + pub permission: bool, + pub lose_append: bool, + pub lose_completion: bool, + pub lose_submit_after_commit: bool, + pub lose_submit_before_commit: bool, + pub lose_reservation: bool, + pub stale: bool, + pub bad_receipt: bool, + pub bad_completion: bool, + pub bad_status: bool, + pub wrong_reservation: bool, + pub deny_preflight: bool, + pub deny_upload: bool, + pub deny_status: bool, + pub fail_naming: bool, + pub redirect: Option, + pub prior: Option, + pub uploads: BTreeMap)>, + pub submits: Vec>, + pub reservations: Vec, + pub names: usize, + pub preflights: usize, + pub begin_count: usize, + pub status: Option, + pub authenticated: usize, +} +pub(crate) struct Fixture { + pub endpoint: String, + pub state: Arc>, + stop: CancellationToken, + task: tokio::task::JoinHandle<()>, +} +impl Fixture { + pub async fn new() -> Self { + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("http://{}/", listener.local_addr().unwrap()); + let state = Arc::new(Mutex::new(Behavior { + permission: true, + ..Default::default() + })); + let stop = CancellationToken::new(); + let cancellation = stop.clone(); + let app = Router::new() + .fallback(handler) + .layer(axum::extract::DefaultBodyLimit::max(32 * 1024 * 1024)) + .with_state((state.clone(), endpoint.clone())); + let task = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(cancellation.cancelled_owned()) + .await + .unwrap(); + }); + Self { + endpoint, + state, + stop, + task, + } + } + pub fn client(&self) -> PublisherClient { + PublisherClient::new(&self.endpoint, TOKEN.parse().unwrap()).unwrap() + } + pub fn record(&self, new: bool, name: bool) -> Record { + let bytes = SYSTEM_EMPTY_MODULE_V1_BYTES; + let envelope = PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), + expected_revision: None, + module_action: ModuleAction::Set(UserModule { + kind: UserModuleKind::Wasm, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }), + container_action: Default::default(), + }; + let module_artifact = SYSTEM_EMPTY_MODULE_V1_ARTIFACT; + let request = PublishRequest { + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + deployment: envelope.resolve(None, &Default::default()).unwrap(), + envelope, + module_artifact, + migration_policy: PreparedMigrationPolicy::Compatible, + }), + creation: new.then_some(CreationOptions { + parent: None, + organization: None, + num_replicas: None, + enforce_anti_affinity: true, + }), + image_source: None, + }; + let request_json = serde_json::to_string(&request).unwrap(); + Record { + version: 1, + server: self.endpoint.clone(), + artifact_endpoint: self.endpoint.clone(), + publisher: publisher(), + database: (!new).then_some(database()), + reservation: request.creation.clone().map(|options| ReserveDatabaseRequest { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: request.manifest.current().envelope.operation_id, + options, + }), + requested_name: name.then_some("fixture-name".into()), + request_digest: spacetimedb_oci::sha256(request_json.as_bytes()), + request_json, + uploads: vec![UploadRecord { + kind: UploadKind::Module, + object: ObjectRef { + digest: module_artifact.digest, + size: module_artifact.size_bytes, + }, + session: None, + }], + submitted: false, + status: None, + name_confirmed: false, + naming_attempted: false, + } + } + pub async fn close(self) { + self.stop.cancel(); + self.task.await.unwrap(); + } +} +async fn handler( + State((shared, endpoint)): State<(Arc>, String)>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let mut state = shared.lock().unwrap(); + let path = uri.path(); + if path == "/v1/containers/capabilities" { + assert!(headers.get("authorization").is_none()); + if let Some(location) = &state.redirect { + return (StatusCode::TEMPORARY_REDIRECT, [("location", location.clone())]).into_response(); + } + return Json(PublicationCapabilities { + version: 1, + enabled: true, + artifact_endpoint: Some(endpoint), + }) + .into_response(); + } + if headers.get("authorization").and_then(|v| v.to_str().ok()) != Some(TOKEN) { + return StatusCode::UNAUTHORIZED.into_response(); + } + state.authenticated += 1; + if path == "/v1/containers/publish-permission" { + return Json(PublishPermission { + identity: publisher(), + can_publish: state.permission, + source_revision: None, + }) + .into_response(); + } + if path == "/v1/containers/reservations" { + let request: ReserveDatabaseRequest = serde_json::from_slice(&body).unwrap(); + if let Some(prior) = state.reservations.first() { + assert_eq!(prior, &request); + } + state.reservations.push(request.clone()); + if std::mem::take(&mut state.lose_reservation) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(DatabaseReservation { + database_identity: database(), + operation_id: if state.wrong_reservation { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) + } else { + request.operation_id + }, + expires_at: "fixture-only".into(), + staging_open: true, + artifact_endpoint: endpoint, + }) + .into_response(); + } + if path.ends_with("/pre_publish") { + state.preflights += 1; + if state.deny_preflight { + return StatusCode::FORBIDDEN.into_response(); + } + assert_eq!(body.as_ref(), SYSTEM_EMPTY_MODULE_V1_BYTES); + return Json(spacetimedb_client_api_messages::name::PrePublishResult::AutoMigrate( + spacetimedb_client_api_messages::name::PrePublishAutoMigrateResult { + migrate_plan: "fixture migration".into(), + break_clients: false, + token: spacetimedb_lib::Hash::ZERO, + major_version_upgrade: false, + }, + )) + .into_response(); + } + if path.ends_with("/names") { + state.names += 1; + if state.fail_naming { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(spacetimedb_client_api_messages::name::SetDomainsResult::Success).into_response(); + } + if path.contains("/deployment/operations/") { + if state.deny_status { + return StatusCode::FORBIDDEN.into_response(); + } + return state + .status + .clone() + .map(|status| Json(status).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + if path.ends_with("/deployment") { + if method == Method::GET { + return state + .prior + .clone() + .map(|p| Json(p).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + assert_eq!(method, Method::PUT); + state.submits.push(body.to_vec()); + if let Some(status) = &state.status { + assert_eq!(state.submits.first().unwrap().as_slice(), body.as_ref()); + return Json(status.clone()).into_response(); + } + if state.stale { + return (StatusCode::CONFLICT, "private-server-error-must-not-be-printed").into_response(); + } + if std::mem::take(&mut state.lose_submit_before_commit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let request: PublishRequest = serde_json::from_slice(&body).unwrap(); + request.manifest.validate(&Default::default()).unwrap(); + let status = PublicationStatus { + database_identity: database(), + operation_id: request.manifest.current().envelope.operation_id, + phase: PublicationPhase::Complete, + expected_revision: request.manifest.current().envelope.expected_revision, + proposed_revision: if state.bad_status { + spacetimedb_lib::Hash::ZERO + } else { + request.manifest.current().deployment.revision().unwrap() + }, + error: None, + }; + state.status = Some(status.clone()); + if std::mem::take(&mut state.lose_submit_after_commit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(status).into_response(); + } + if path.ends_with("/uploads") { + state.begin_count += 1; + if state.deny_upload { + return StatusCode::FORBIDDEN.into_response(); + } + let request: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + headers["x-spacetimedb-artifact-kind"], + request["kind"].as_str().unwrap() + ); + let object: ObjectRef = serde_json::from_value(request["object"].clone()).unwrap(); + let status = UploadStatus { + id: uuid::Uuid::new_v4(), + object, + offset: 0, + expires_at: u64::MAX, + complete: false, + }; + state.uploads.insert(status.id, (status.clone(), vec![])); + return Json(status).into_response(); + } + if let Some((_, tail)) = path.split_once("/uploads/") { + let id: uuid::Uuid = tail.split('/').next().unwrap().parse().unwrap(); + let Some((status, bytes)) = state.uploads.get_mut(&id) else { + return StatusCode::NOT_FOUND.into_response(); + }; + if method == Method::PATCH { + assert!(body.len() <= client::UPLOAD_CHUNK_BYTES); + let offset: u64 = uri.query().unwrap().strip_prefix("offset=").unwrap().parse().unwrap(); + assert_eq!(offset, status.offset); + bytes.extend_from_slice(&body); + status.offset += body.len() as u64; + } else if path.ends_with("/complete") { + assert_eq!(bytes.len() as u64, status.object.size); + assert_eq!(spacetimedb_oci::sha256(bytes), status.object.digest); + status.complete = true; + } + let mut response = status.clone(); + if path.ends_with("/complete") { + if std::mem::take(&mut state.lose_completion) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let mut object = response.object; + if state.bad_completion { + object.size += 1; + } + return Json(object).into_response(); + } + if method == Method::PATCH && std::mem::take(&mut state.lose_append) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + if state.bad_receipt { + response.object.size += 1; + } + return Json(response).into_response(); + } + panic!("unexpected fixture request: {method} {uri}"); +} +async fn run_now(client: &PublisherClient, journal: &mut Journal) -> Result { + run(client, journal, None, Duration::ZERO, CancellationToken::new()).await +} +fn journal(base: &Path, record: Record) -> Journal { + Journal::create(base, record, None, Some(SYSTEM_EMPTY_MODULE_V1_BYTES)).unwrap() +} +use std::path::Path; + +#[tokio::test] +async fn lost_append_and_committed_response_resume_without_reupload_or_current_permission() { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.lose_append = true; + s.lose_submit_after_commit = true; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.submitted); + let path = journal.directory().to_owned(); + drop(journal); + std::fs::remove_file(path.join("module.blob")).unwrap(); + fixture.state.lock().unwrap().permission = false; + let mut journal = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.submits, [exact.into_bytes()]); + assert_eq!(state.begin_count, 1); + } + drop(journal); + fixture.close().await; +} +#[tokio::test] +async fn lost_reservation_and_unadmitted_put_replay_exact_request_and_generated_identity() { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.lose_reservation = true; + s.lose_submit_before_commit = true; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(true, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.database.is_none()); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert_eq!(journal.record.database, Some(database())); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.reservations.len(), 2); + assert_eq!(state.submits, [exact.as_bytes(), exact.as_bytes()]); + assert_eq!(state.begin_count, 1); + } + fixture.close().await; +} +#[tokio::test] +async fn stale_revision_preserves_operation_and_redacts_response_without_fallback() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().stale = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); + assert!(!format!("{error:#}").contains("private-server-error")); + assert_eq!(journal.record.request_json, exact); + assert!(journal.record.submitted); + assert_eq!(fixture.state.lock().unwrap().submits.len(), 1); + fixture.close().await; +} +#[tokio::test] +async fn mismatched_reservation_or_upload_receipt_never_reaches_admission() { + for reservation in [true, false] { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.wrong_reservation = reservation; + s.bad_receipt = !reservation; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(reservation, false)); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(!journal.record.submitted); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + fixture.close().await; + } +} +#[tokio::test] +async fn completion_descriptor_must_match_before_admission() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().bad_completion = true; + let directory = tempfile::tempdir().unwrap(); + let mut journal = journal(directory.path(), fixture.record(true, false)); + let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); + assert!(error.to_string().contains("completion descriptor changed")); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + assert!(!journal.record.submitted); + fixture.state.lock().unwrap().bad_completion = false; + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + let status = journal.record.uploads[0].session.as_ref().unwrap(); + assert!(status.complete); + assert_eq!(status.offset, status.object.size); + fixture.close().await; +} + +#[tokio::test] +async fn lost_completion_response_observes_the_same_complete_session() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_completion = true; + let directory = tempfile::tempdir().unwrap(); + let mut journal = journal(directory.path(), fixture.record(true, false)); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + let session = journal.record.uploads[0].session.as_ref().unwrap(); + assert!(session.complete); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.begin_count, 1); + assert_eq!(state.uploads.len(), 1); + assert!(state.uploads[&session.id].0.complete); + } + fixture.close().await; +} + +#[tokio::test] +async fn wrong_publication_scope_and_denied_upload_are_not_accepted() { + for denied in [true, false] { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.deny_upload = denied; + s.bad_status = !denied; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.status.is_none()); + assert_eq!(fixture.state.lock().unwrap().submits.len(), usize::from(!denied)); + fixture.close().await; + } +} +#[tokio::test] +async fn naming_failure_reports_active_identity_without_republishing_or_overwriting_later_names() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().fail_naming = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(true, true)); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::NamingUnconfirmed(_) + )); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::NamingUnconfirmed(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.names, 1); + assert_eq!(state.submits.len(), 1); + } + fixture.close().await; +} +#[tokio::test] +async fn redirect_and_cross_origin_never_forward_credentials_without_exact_approval() { + let first = Fixture::new().await; + let foreign = Fixture::new().await; + first.state.lock().unwrap().redirect = Some(format!("{}v1/containers/publish-permission", foreign.endpoint)); + assert!(first.client().capabilities().await.is_err()); + assert!(first.client().artifact_endpoint(&foreign.endpoint, None).is_err()); + assert!(first + .client() + .artifact_endpoint(&foreign.endpoint, Some(&first.endpoint)) + .is_err()); + assert!(first + .client() + .artifact_endpoint(&foreign.endpoint, Some(&foreign.endpoint)) + .is_ok()); + assert_eq!(foreign.state.lock().unwrap().authenticated, 0); + first.close().await; + foreign.close().await; +} +#[tokio::test] +async fn journal_locks_and_reverifies_local_bytes_without_persisting_credentials() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let path = journal.directory().to_owned(); + assert!(Journal::open(&path).is_err()); + let bytes = std::fs::read_to_string(path.join("publication.json")).unwrap(); + assert!(!bytes.contains("isolated-fixture-credential")); + std::fs::write(path.join("module.blob"), vec![0; SYSTEM_EMPTY_MODULE_V1_BYTES.len()]).unwrap(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert_eq!(fixture.state.lock().unwrap().begin_count, 0); + drop(journal); + let mut value: serde_json::Value = serde_json::from_str(&bytes).unwrap(); + value["request_json"] = json!("{}"); + std::fs::write(path.join("publication.json"), serde_json::to_vec(&value).unwrap()).unwrap(); + assert!(Journal::open(&path).is_err()); + fixture.close().await; +} +#[test] +fn endpoints_and_receipts_are_bounded_and_unambiguous() { + for endpoint in [ + "Maincloud", + "http://10.0.0.1", + "https://user:password@example.invalid", + "https://example.invalid/?token=secret", + ] { + assert!(client::endpoint(endpoint).is_err()); + } + for endpoint in ["http://localhost:3000", "http://[::1]:3000"] { + assert!(client::endpoint(endpoint).is_ok()); + } + let object = ObjectRef { + digest: SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size: 1, + }; + let status = UploadStatus { + id: uuid::Uuid::now_v7(), + object, + offset: 0, + expires_at: 0, + complete: false, + }; + assert!(status.validate(object, None).is_err()); +} + +#[tokio::test] +async fn incomplete_artifact_retention_never_publishes_a_resume_record_or_mutates_server() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let base = temporary.path().join("new").join("nested").join("state"); + let record = fixture.record(false, false); + let id = record.request().unwrap().manifest.current().envelope.operation_id; + // A missing promised module fails the artifact flush before publication.json + // can make this operation available for reservation or admission. + assert!(Journal::create(&base, record, None, None).is_err()); + assert!(!base.join(id.to_string()).exists()); + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} + +#[tokio::test] +async fn revoked_status_access_replays_original_put_without_missing_local_artifacts_or_new_uploads() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_after_commit = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + let path = journal.directory().to_owned(); + drop(journal); + std::fs::remove_file(path.join("module.blob")).unwrap(); + { + let mut state = fixture.state.lock().unwrap(); + state.permission = false; + state.deny_status = true; + state.deny_upload = true; + } + let mut journal = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.submits, [exact.as_bytes(), exact.as_bytes()]); + assert_eq!(state.begin_count, 1); + } + fixture.close().await; +} diff --git a/crates/cli/src/container/tests.rs b/crates/cli/src/container/tests.rs new file mode 100644 index 00000000000..6282db120f9 --- /dev/null +++ b/crates/cli/src/container/tests.rs @@ -0,0 +1,720 @@ +use super::*; +use crate::spacetime_config::SpacetimeConfig; +use serde_json::json; +use std::{collections::BTreeMap, sync::Mutex}; + +static TEST_PREPARATIONS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn platform() -> ImagePlatform { + ImagePlatform { + os: "linux".into(), + architecture: "amd64".into(), + } +} +pub(crate) fn declaration(image: serde_json::Value) -> ContainerConfig { + serde_json::from_value(json!({"image":image,"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}})).unwrap() +} +fn blob(layout: &Path, bytes: &[u8], media: &str) -> Descriptor { + let digest = spacetimedb_oci::sha256(bytes); + fs::create_dir_all(layout.join("blobs/sha256")).unwrap(); + fs::write( + layout + .join("blobs/sha256") + .join(digest.to_string().strip_prefix("sha256:").unwrap()), + bytes, + ) + .unwrap(); + Descriptor { + media_type: media.into(), + digest, + size: bytes.len() as u64, + platform: None, + urls: vec![], + data: None, + artifact_type: None, + } +} +pub(crate) fn fixture(layout: &Path) -> Descriptor { + let mut archive = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(4); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, "app.js", &b"code"[..]).unwrap(); + let layer = archive.into_inner().unwrap(); + let layer = blob(layout, &layer, "application/vnd.oci.image.layer.v1.tar"); + let config = blob(layout, &serde_json::to_vec(&json!({"architecture":"amd64","os":"linux","config":{"Entrypoint":["/usr/bin/env"],"Cmd":["node","app.js"],"User":"1000:1000","WorkingDir":"/app","Env":["BAKED=private-image-value"]},"rootfs":{"type":"layers","diff_ids":[layer.digest]}})).unwrap(), spacetimedb_oci::OCI_CONFIG); + let manifest = blob( + layout, + &serde_json::to_vec( + &json!({"schemaVersion":2,"mediaType":spacetimedb_oci::OCI_MANIFEST,"config":config,"layers":[layer]}), + ) + .unwrap(), + spacetimedb_oci::OCI_MANIFEST, + ); + fs::write(layout.join("oci-layout"), br#"{"imageLayoutVersion":"1.0.0"}"#).unwrap(); + fs::write( + layout.join("index.json"), + serde_json::to_vec(&json!({"schemaVersion":2,"mediaType":spacetimedb_oci::OCI_INDEX,"manifests":[manifest]})) + .unwrap(), + ) + .unwrap(); + manifest +} +fn copy_layout(source: &Path, output: &Path) { + fs::create_dir_all(output.join("blobs/sha256")).unwrap(); + for name in ["oci-layout", "index.json"] { + fs::copy(source.join(name), output.join(name)).unwrap(); + } + for file in fs::read_dir(source.join("blobs/sha256")).unwrap() { + let file = file.unwrap(); + fs::copy(file.path(), output.join("blobs/sha256").join(file.file_name())).unwrap(); + } +} +type RecordedCall = (String, Vec, Vec); + +struct FakeRunner { + layout: PathBuf, + calls: Mutex>, + fail_detection: bool, + version: &'static str, +} +impl FakeRunner { + fn new(layout: &Path) -> Self { + Self { + layout: layout.into(), + calls: Mutex::new(vec![]), + fail_detection: false, + version: "railpack 0.35.0\n", + } + } +} +impl Runner for FakeRunner { + async fn run(&self, invocation: Invocation) -> Result { + self.calls.lock().unwrap().push(( + invocation.label.into(), + invocation.args.clone(), + invocation.env.iter().map(|(name, _)| name.clone()).collect(), + )); + match invocation.label { + "Railpack version check" => { + return Ok(process::Output { + stdout: self.version.as_bytes().to_vec(), + }) + } + "Railpack detection" => { + ensure!(!self.fail_detection, "unsupported source detection"); + fs::write(invocation.workspace.path().join("railpack-plan.json"), b"{}")?; + } + "Skopeo image import" => copy_layout(&self.layout, &invocation.workspace.path().join("input")), + "BuildKit OCI build" => { + let file = fs::File::create(invocation.workspace.path().join("image.tar"))?; + let mut tar = tar::Builder::new(file); + for name in ["oci-layout", "index.json"] { + tar.append_path_with_name(self.layout.join(name), name)?; + } + for blob in fs::read_dir(self.layout.join("blobs/sha256"))? { + let blob = blob?; + tar.append_path_with_name(blob.path(), Path::new("blobs/sha256").join(blob.file_name()))?; + } + tar.finish()?; + } + _ => anyhow::bail!("unexpected fake image tool"), + } + Ok(process::Output { stdout: vec![] }) + } +} + +#[test] +fn image_source_is_exclusive_and_builder_defaults_are_strict() { + for image in [ + json!({}), + json!({"build":{},"oci_ref":"example/image"}), + json!({"build":{"builder":"unknown"}}), + json!({"build":{"builder":"railpack","dockerfile":"Dockerfile"}}), + ] { + let mut config = serde_json::to_value(declaration(json!({"oci_ref":"example/image"}))).unwrap(); + config["image"] = image; + assert!(serde_json::from_value::(config).is_err()); + } + assert!(matches!( + declaration(json!({"build":{}})).image, + ImageSource::Build(config::BuildImage { + build: SourceBuild::Dockerfile { .. } + }) + )); +} + +#[test] +fn container_declarations_never_inherit_through_children_or_overrides() { + let a = serde_json::to_value(declaration(json!({"build":{}}))).unwrap(); + let b = serde_json::to_value(declaration(json!({"oci_ref":"example/child"}))).unwrap(); + let config: SpacetimeConfig = serde_json::from_value(json!({"database":"root","module-path":"shared","container":a,"children":[{"database":"plain","children":[{"database":"grandchild"}]},{"database":"own","container":b,"children":[{"database":"own-grandchild"}]}]})).unwrap(); + let targets = config.collect_all_targets_with_inheritance(); + let declarations: BTreeMap<_, _> = targets + .iter() + .map(|target| (target.fields["database"].as_str().unwrap(), target.container.is_some())) + .collect(); + assert_eq!( + declarations, + BTreeMap::from([ + ("root", true), + ("plain", false), + ("grandchild", false), + ("own", true), + ("own-grandchild", false) + ]) + ); + assert!(targets.iter().all(|target| !target.fields.contains_key("container"))); + assert!(crate::subcommands::container::select(&config, Some("plain")).is_err()); + assert!(crate::subcommands::container::select(&config, Some("not-a-local-target")).is_err()); + assert!(crate::subcommands::container::select(&config, None).is_err()); + assert!(crate::subcommands::container::select(&config, Some("own")).is_ok()); +} + +#[tokio::test] +async fn local_prebuilt_output_is_owned_verified_and_keeps_image_defaults() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("prebuilt image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let token = CancellationToken::new(); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"oci:prebuilt image"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + token.clone(), + ) + .await + .unwrap(); + assert!(!token.is_cancelled()); + assert!(runner.calls.lock().unwrap().is_empty()); + assert_eq!(prepared.metadata.manifest.digest, manifest.digest); + assert_eq!(prepared.metadata.container.argv, ["/usr/bin/env", "node", "app.js"]); + assert_eq!(prepared.metadata.container.user, "1000:1000"); + assert_eq!(prepared.metadata.container.working_directory, "/app"); + assert!(!serde_json::to_string(&prepared.metadata) + .unwrap() + .contains("private-image-value")); + assert_eq!(prepared.metadata.objects.len(), 3); + let temporary = prepared.layout(); + let output = root.path().join("reusable"); + prepared.persist(&output).unwrap(); + assert!(!temporary.exists()); + assert!(output.join("prepared.json").is_file()); + let reread = prepare_container( + &declaration(json!({"oci_ref":"oci:reusable"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + token, + ) + .await + .unwrap(); + assert_eq!(reread.metadata.manifest.digest, manifest.digest); + let temporary = reread.layout(); + drop(reread); + assert!(!temporary.exists()); +} + +#[tokio::test] +async fn command_override_replaces_image_argv_and_invalid_bytes_never_persist() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let mut config = declaration(json!({"oci_ref":"oci:image"})); + config.command = Some(vec!["/bin/custom".into()]); + let prepared = prepare_container( + &config, + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(prepared.metadata.container.argv, ["/bin/custom"]); + drop(prepared); + fs::write( + input + .join("blobs/sha256") + .join(manifest.digest.to_string().strip_prefix("sha256:").unwrap()), + b"tampered", + ) + .unwrap(); + assert!(prepare_container( + &config, + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert!(fs::read_dir(root.path()).unwrap().all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".spacetime-image-"))); +} + +#[tokio::test] +async fn dockerfile_and_railpack_have_explicit_local_endpoint_and_separate_secrets() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("fixture"); + fixture(&input); + let context = root.path().join("source with spaces"); + fs::create_dir(&context).unwrap(); + fs::write(context.join("Dockerfile"), b"FROM scratch").unwrap(); + let secret = root.path().join("secret file"); + fs::write(&secret, b"sensitive-build-value").unwrap(); + for builder in ["dockerfile", "railpack"] { + let runner = FakeRunner::new(&input); + let tools = BuildTools { + buildkit_host: Some("unix:///disposable/fake-buildkit.sock".into()), + secrets: vec![BuildSecret { + name: "BUILD_KEY".into(), + file: secret.clone(), + }], + ..Default::default() + }; + let config = declaration(json!({"build":{"builder":builder,"context":"source with spaces"}})); + let prepared = prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + let calls = runner.calls.lock().unwrap(); + let (_, argv, environment) = calls + .iter() + .find(|(label, _, _)| label == "BuildKit OCI build") + .unwrap(); + assert_eq!(&argv[..2], ["--addr", "unix:///disposable/fake-buildkit.sock"]); + assert!(argv.contains(&format!("context={}", context.canonicalize().unwrap().display()).into())); + assert!(argv.contains(&"--no-cache".into())); + assert!(environment.contains(&"DOCKER_CONFIG".into())); + assert!(!format!("{argv:?}").contains("sensitive-build-value")); + assert!(!serde_json::to_string(&prepared.metadata).unwrap().contains("BUILD_KEY")); + if builder == "railpack" { + assert!(argv.contains(&format!("source={RAILPACK_FRONTEND}").into())); + let (_, args, _) = calls + .iter() + .find(|(label, _, _)| label == "Railpack detection") + .unwrap(); + assert!(args.contains(&"BUILD_KEY=".into())); + } else { + assert_eq!(calls.len(), 1); + } + } +} + +#[tokio::test] +async fn unsupported_railpack_and_remote_builder_never_fall_back() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let config = declaration(json!({"build":{"builder":"railpack"}})); + let tools = BuildTools { + buildkit_host: Some("unix:///disposable/fake-buildkit.sock".into()), + ..Default::default() + }; + let mut runner = FakeRunner::new(&input); + runner.fail_detection = true; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert_eq!(runner.calls.lock().unwrap().len(), 2); + runner.calls.lock().unwrap().clear(); + runner.version = "railpack 99.0.0"; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert_eq!(runner.calls.lock().unwrap().len(), 1); + runner.calls.lock().unwrap().clear(); + let tools = BuildTools { + buildkit_host: Some("tcp://untrusted:1234".into()), + ..Default::default() + }; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert!(runner.calls.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn registry_import_uses_explicit_anonymous_auth_and_records_selected_digest() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"example.invalid/team/image:tag"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(prepared.metadata.manifest.digest, manifest.digest); + let calls = runner.calls.lock().unwrap(); + assert_eq!(calls.len(), 1); + let args = &calls[0].1; + assert!(args.contains(&"--preserve-digests".into())); + assert!(args.contains(&"docker://example.invalid/team/image:tag".into())); + let auth = PathBuf::from(&args[args.iter().position(|arg| arg == "--authfile").unwrap() + 1]); + assert_eq!(fs::read(auth).unwrap(), br#"{"auths":{}}"#); +} + +#[test] +fn archive_rejects_links_and_cancelled_reads() { + let root = tempfile::tempdir().unwrap(); + for kind in [tar::EntryType::Symlink, tar::EntryType::Link] { + let path = root.path().join("input.tar"); + let mut tar = tar::Builder::new(fs::File::create(&path).unwrap()); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(kind); + header.set_size(0); + header.set_link_name("/outside").unwrap(); + header.set_cksum(); + tar.append_data(&mut header, "index.json", std::io::empty()).unwrap(); + tar.finish().unwrap(); + let output = tempfile::tempdir().unwrap(); + assert!(oci::extract_archive( + &path, + output.path(), + &CancellationToken::new(), + Instant::now() + Duration::from_secs(1) + ) + .is_err()); + } + let token = CancellationToken::new(); + token.cancel(); + assert!(oci::check(&token, Instant::now() + Duration::from_secs(1)).is_err()); +} + +#[test] +fn container_build_command_requires_explicit_platform_and_output() { + crate::subcommands::container::cli().debug_assert(); + assert!(crate::subcommands::container::cli() + .try_get_matches_from(["container", "build"]) + .is_err()); + assert!(crate::subcommands::container::cli() + .try_get_matches_from([ + "container", + "build", + "local-target", + "--platform", + "linux/amd64", + "--out-dir", + "output" + ]) + .is_ok()); +} + +#[test] +fn publish_preserves_container_target_metadata_without_affecting_plain_children() { + use crate::subcommands::publish::{build_publish_schema, get_filtered_publish_configs}; + let config: SpacetimeConfig = serde_json::from_value(json!({ + "database":"container-db", "container":declaration(json!({"build":{}})), + "children":[{"database":"ordinary-db"}] + })) + .unwrap(); + let command = crate::subcommands::publish::cli(); + let schema = build_publish_schema(&command).unwrap(); + for selected in ["container-db", "*"] { + let args = command.clone().try_get_matches_from(["publish", selected]).unwrap(); + let targets = get_filtered_publish_configs(&config, &command, &schema, &args).unwrap(); + assert!(targets[0].container().is_some()); + assert!(targets.iter().skip(1).all(|target| target.container().is_none())); + } + let args = command + .clone() + .try_get_matches_from(["publish", "ordinary-db"]) + .unwrap(); + assert_eq!( + get_filtered_publish_configs(&config, &command, &schema, &args) + .unwrap() + .len(), + 1 + ); +} + +#[tokio::test] +async fn local_index_defaults_media_type_but_never_overwrites_existing_output() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let mut index: serde_json::Value = serde_json::from_slice(&fs::read(input.join("index.json")).unwrap()).unwrap(); + index.as_object_mut().unwrap().remove("mediaType"); + fs::write(input.join("index.json"), serde_json::to_vec(&index).unwrap()).unwrap(); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"oci:image"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &FakeRunner::new(&input), + CancellationToken::new(), + ) + .await + .unwrap(); + let output = root.path().join("existing-empty-output"); + fs::create_dir(&output).unwrap(); + assert!(prepared.persist(&output).is_err()); + assert!(fs::read_dir(&output).unwrap().next().is_none()); +} + +#[test] +fn wrong_platform_and_duplicate_archive_object_are_rejected() { + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let wrong = ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }; + assert!(oci::verify_layout( + &input, + &root.path().join("output"), + &wrong, + &CancellationToken::new(), + Instant::now() + Duration::from_secs(2) + ) + .is_err()); + let archive = root.path().join("duplicate.tar"); + let mut tar = tar::Builder::new(fs::File::create(&archive).unwrap()); + for _ in 0..2 { + tar.append_path_with_name(input.join("index.json"), "index.json") + .unwrap(); + } + tar.finish().unwrap(); + assert!(oci::extract_archive( + &archive, + &root.path().join("duplicate"), + &CancellationToken::new(), + Instant::now() + Duration::from_secs(2) + ) + .is_err()); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[tokio::test] +async fn subprocess_exit_cancellation_caller_drop_and_output_overflow_reap_before_workspace_release() { + use process::LocalRunner; + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + // These scripts exercise only owned local subprocesses. No Docker, server, + // network, saved configuration, or user credentials are involved. + for mode in [ + "exit", + "exit_without_descendants", + "cancel", + "drop", + "overflow", + "timeout", + ] { + let workspace = Arc::new( + tempfile::Builder::new() + .prefix("fake-tool-") + .tempdir_in(root.path()) + .unwrap(), + ); + let path = workspace.path().to_path_buf(); + let pid_file = root.path().join(format!("{mode}.pid")); + let script = match mode { + "exit" => "echo $$ > \"$1\"; sleep 60 & exit 0", + "exit_without_descendants" => "echo $$ > \"$1\"; exit 0", + "overflow" => "echo $$ > \"$1\"; yes x", + _ => "echo $$ > \"$1\"; sleep 60 & wait", + }; + let token = CancellationToken::new(); + let task = tokio::spawn(LocalRunner.run(Invocation { + tool: "/bin/sh".into(), + label: "owned fake builder", + args: vec![ + "-c".into(), + script.into(), + "fake-builder".into(), + pid_file.clone().into_os_string(), + ], + env: vec![], + cwd: root.path().into(), + workspace, + timeout: if mode == "timeout" { + Duration::from_millis(250) + } else { + Duration::from_secs(4) + }, + cancel: token.clone(), + })); + tokio::time::timeout(Duration::from_secs(2), async { + while !pid_file.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let pid = fs::read_to_string(&pid_file).unwrap().trim().parse::().unwrap(); + let pid = rustix::process::Pid::from_raw(pid).unwrap(); + if mode == "cancel" { + token.cancel(); + } + if mode == "drop" { + task.abort(); + let _ = task.await; + } else { + let result = tokio::time::timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap(); + assert_eq!(result.is_ok(), matches!(mode, "exit" | "exit_without_descendants")); + } + tokio::time::timeout(Duration::from_secs(3), async { + while path.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!( + matches!( + rustix::process::waitpid(Some(pid), rustix::process::WaitOptions::NOHANG), + Err(rustix::io::Errno::CHILD) + ), + "builder leader must already be reaped" + ); + } +} + +#[test] +fn configuration_overlay_replaces_only_the_selected_container_declaration() { + let root = tempfile::tempdir().unwrap(); + let parent = declaration(json!({"build":{}})); + let child = declaration(json!({"oci_ref":"example.invalid/child:v1"})); + fs::write(root.path().join("spacetime.json"),serde_json::to_vec(&json!({"database":"parent","container":parent,"children":[{"database":"plain"},{"database":"own","container":child}]})).unwrap()).unwrap(); + let replacement = declaration(json!({"build":{"builder":"railpack"}})); + fs::write( + root.path().join("spacetime.dev.json"), + serde_json::to_vec(&json!({"container":replacement})).unwrap(), + ) + .unwrap(); + let loaded = crate::spacetime_config::find_and_load_with_env_from(Some("dev"), root.path().into()) + .unwrap() + .unwrap(); + assert!(matches!( + crate::subcommands::container::select(&loaded.config, Some("parent")) + .unwrap() + .image, + ImageSource::Build(config::BuildImage { + build: SourceBuild::Railpack { .. } + }) + )); + assert!(crate::subcommands::container::select(&loaded.config, Some("plain")).is_err()); + assert!(matches!( + crate::subcommands::container::select(&loaded.config, Some("own")) + .unwrap() + .image, + ImageSource::Prebuilt(_) + )); +} + +#[tokio::test] +async fn build_command_executes_local_import_with_only_project_configuration() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + fixture(&root.path().join("input")); + fs::write( + root.path().join("spacetime.json"), + serde_json::to_vec(&json!({ + "database":"local-selection", "server":"https://must-not-be-contacted.invalid", + "container":declaration(json!({"oci_ref":"oci:input"})) + })) + .unwrap(), + ) + .unwrap(); + let output = root.path().join("output"); + let args = crate::subcommands::container::cli() + .try_get_matches_from([ + OsString::from("container"), + "build".into(), + "local-selection".into(), + "--project-path".into(), + root.path().into(), + "--out-dir".into(), + output.clone().into_os_string(), + "--platform".into(), + "linux/amd64".into(), + ]) + .unwrap(); + crate::exec_local_subcommand("container", &args).await.unwrap().unwrap(); + let metadata: PreparedMetadata = serde_json::from_slice(&fs::read(output.join("prepared.json")).unwrap()).unwrap(); + assert_eq!(metadata.container.argv, ["/usr/bin/env", "node", "app.js"]); + assert_eq!(metadata.objects.len(), 3); +} + +#[test] +fn omitted_image_working_directory_normalizes_to_linux_root() { + let declaration = declaration(json!({"oci_ref":"example.invalid/no-working-dir"})); + let image = spacetimedb_oci::ContainerConfig { + cmd: Some(vec!["/app".into()]), + ..Default::default() + }; + let normalized = declaration + .normalize(spacetimedb_oci::sha256(b"fixture"), platform(), &image) + .unwrap(); + assert_eq!(normalized.working_directory, "/"); + let mut explicit = declaration; + explicit.working_directory = Some(String::new()); + assert!(explicit + .normalize(spacetimedb_oci::sha256(b"fixture"), platform(), &image) + .is_err()); +} diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index e38542595c8..aec1536f28e 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,6 +1,7 @@ pub mod api; mod common_args; mod config; +pub mod container; pub(crate) mod detect; mod edit_distance; mod errors; @@ -35,14 +36,31 @@ pub fn get_subcommands() -> Vec { logout::cli(), init::cli(), build::cli(), + subcommands::container::cli(), server::cli(), - sidecar::cli(), subscribe::cli(), start::cli(), subcommands::version::cli(), ] } +/// Dispatch commands that need only project files before opening saved CLI +/// server settings or credentials. Future container network commands use the +/// ordinary authenticated dispatcher below. +pub async fn exec_local_subcommand(cmd: &str, args: &ArgMatches) -> Option> { + if cmd == "container" + && let Some(("build", args)) = args.subcommand() + { + Some( + subcommands::container::exec_build(args) + .await + .map(|()| ExitCode::SUCCESS), + ) + } else { + None + } +} + pub async fn exec_subcommand( config: Config, paths: &SpacetimePaths, @@ -63,8 +81,8 @@ pub async fn exec_subcommand( "list" => list::exec(config, args).await, "init" => init::exec(config, args).await.map(|_| ()), "build" => build::exec(config, args).await.map(drop), + "container" => subcommands::container::exec(config, args).await, "server" => server::exec(config, paths, args).await, - "sidecar" => sidecar::exec(config, args).await, "subscribe" => subscribe::exec(config, args).await, "start" => return start::exec(config, paths, args).await, "login" => login::exec(config, args).await, diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 6f2e4d23f08..f1092aa8b62 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -32,6 +32,10 @@ async fn main() -> anyhow::Result { let matches = get_command().get_matches(); let (cmd, subcommand_args) = matches.subcommand().unwrap(); + if let Some(result) = exec_local_subcommand(cmd, subcommand_args).await { + return result; + } + let root_dir = matches.get_one::("root_dir"); let paths = match root_dir { Some(dir) => SpacetimePaths::from_root_dir(dir), @@ -135,3 +139,14 @@ Commands: "#, ) } + +#[cfg(test)] +mod tests { + #[test] + fn managed_publish_help_passes_entrypoint_validation() { + let help = super::get_command() + .try_get_matches_from(["spacetime", "publish", "--help"]) + .unwrap_err(); + assert_eq!(help.kind(), clap::error::ErrorKind::DisplayHelp); + } +} diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index b3316f2c318..dc493abba8f 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -108,6 +108,9 @@ pub enum CommandConfigError { #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "kebab-case")] pub struct SpacetimeConfig { + /// Container declaration belongs only to this database, never its children. + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, /// Configuration for the dev command. Root-level only, not inherited. #[serde(skip_serializing_if = "Option::is_none")] pub dev: Option, @@ -144,6 +147,7 @@ pub struct DevConfig { /// Contains all fields needed for both publish and generate operations. #[derive(Debug, Clone)] pub struct FlatTarget { + pub container: Option, /// All entity-level fields (database, module-path, server, etc.) pub fields: HashMap, /// Name of the config file from which this target's `database` value was merged. @@ -201,6 +205,7 @@ impl SpacetimeConfig { let effective_generate = self.generate.clone(); let target = FlatTarget { + container: self.container.clone(), fields: fields.clone(), source_config: self.source_config.clone(), generate: effective_generate, @@ -253,6 +258,8 @@ pub struct CommandConfig<'a> { config_values: HashMap, /// CLI arguments matches: &'a ArgMatches, + /// A declaration belongs to this exact target and never inherits. + container: Option, } /// Schema that defines the contract between CLI arguments and config file keys. @@ -699,9 +706,23 @@ impl<'a> CommandConfig<'a> { schema, config_values: normalized_values, matches, + container: None, }) } + pub fn with_container(mut self, container: Option) -> Self { + self.container = container; + self + } + + pub fn container(&self) -> Option<&crate::container::config::ContainerConfig> { + self.container.as_ref() + } + + pub(crate) fn matches(&self) -> &ArgMatches { + self.matches + } + /// Get a single value from the config as a specific type. /// First checks clap args (via schema), then falls back to config values. /// diff --git a/crates/cli/src/subcommands/container.rs b/crates/cli/src/subcommands/container.rs new file mode 100644 index 00000000000..5e66ef2eeed --- /dev/null +++ b/crates/cli/src/subcommands/container.rs @@ -0,0 +1,186 @@ +//! Container build and operation commands. Local builds do not read saved +//! server credentials; network commands use the explicitly selected server. +mod operations; +mod url; + +use crate::{ + container::{config::ContainerConfig, prepare_container, process::LocalRunner, BuildSecret, BuildTools}, + spacetime_config::{find_and_load_with_env_from, SpacetimeConfig}, +}; +use anyhow::{ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use std::path::{Path, PathBuf}; +use tokio_util::sync::CancellationToken; + +pub fn cli() -> Command { + let command = Command::new("container") + .about("Build and manage a database's container") + .subcommand_required(true) + .subcommand(url::cli()) + .subcommand( + Command::new("build") + .about("Prepare verified OCI artifacts locally without publishing") + .arg(Arg::new("database").help("Database target in local spacetime.json; no server lookup")) + .arg( + Arg::new("project_path") + .long("project-path") + .default_value(".") + .value_parser(clap::value_parser!(PathBuf)) + .help("Directory in which to find spacetime.json"), + ) + .arg( + Arg::new("out_dir") + .long("out-dir") + .required(true) + .value_parser(clap::value_parser!(PathBuf)) + .help("New directory for verified OCI artifacts and prepared.json"), + ) + .arg( + Arg::new("platform") + .long("platform") + .required(true) + .value_parser(["linux/amd64", "linux/arm64"]) + .help("Target Linux platform, independent of this computer's architecture"), + ) + .arg(Arg::new("env").long("env").help("Local configuration overlay name")) + .arg( + Arg::new("buildkit_host") + .long("buildkit-host") + .help("Explicit local BuildKit Unix socket, required for source builds"), + ) + .arg( + Arg::new("buildctl") + .long("buildctl") + .default_value("buildctl") + .value_parser(clap::value_parser!(PathBuf)) + .help("BuildKit client executable"), + ) + .arg( + Arg::new("railpack") + .long("railpack") + .default_value("railpack") + .value_parser(clap::value_parser!(PathBuf)) + .help("Pinned Railpack executable for explicitly selected Railpack builds"), + ) + .arg( + Arg::new("skopeo") + .long("skopeo") + .default_value("skopeo") + .value_parser(clap::value_parser!(PathBuf)) + .help("Skopeo executable for prebuilt registry images"), + ) + .arg( + Arg::new("registry_auth_file") + .long("registry-auth-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Explicit registry auth JSON; omitted means anonymous, never saved Docker credentials"), + ) + .arg( + Arg::new("build_secret") + .long("build-secret") + .action(ArgAction::Append) + .value_name("NAME=FILE") + .help("Explicit build secret file; separate from runtime env_keys"), + ), + ); + operations::commands(command) +} + +pub(crate) fn select(config: &SpacetimeConfig, database: Option<&str>) -> Result { + let targets = config.collect_all_targets_with_inheritance(); + let selected = if let Some(database) = database { + let mut matches = targets + .iter() + .filter(|target| target.fields.get("database").and_then(|v| v.as_str()) == Some(database)); + let target = matches + .next() + .context("database target is not in local spacetime.json")?; + ensure!( + matches.next().is_none(), + "database target is ambiguous in local spacetime.json" + ); + target + } else { + let mut matches = targets.iter().filter(|target| target.container.is_some()); + let target = matches + .next() + .context("no container declaration in local spacetime.json")?; + ensure!( + matches.next().is_none(), + "several container targets exist; select a DATABASE from local spacetime.json" + ); + target + }; + selected + .container + .clone() + .context("selected database has no container declaration; containers are not inherited") +} + +pub async fn exec(mut config: crate::Config, args: &ArgMatches) -> Result<()> { + match args.subcommand().context("missing container command")? { + ("build", args) => exec_build(args).await, + ("url", args) => url::exec(&config, args).await, + (name @ ("status" | "start" | "stop" | "restart"), args) => operations::exec(&mut config, name, args).await, + _ => anyhow::bail!("unsupported container command"), + } +} + +pub async fn exec_build(args: &ArgMatches) -> Result<()> { + let project = args.get_one::("project_path").unwrap().canonicalize()?; + let loaded = find_and_load_with_env_from(args.get_one::("env").map(String::as_str), project)? + .context("spacetime.json not found")?; + let declaration = select(&loaded.config, args.get_one::("database").map(String::as_str))?; + let output = std::env::current_dir()?.join(args.get_one::("out_dir").unwrap()); + ensure!(!output.exists(), "output already exists: {}", output.display()); + let parent = output.parent().unwrap_or(Path::new(".")); + ensure!(parent.is_dir(), "output parent directory does not exist"); + let mut tools = BuildTools { + buildctl: args.get_one::("buildctl").unwrap().clone(), + railpack: args.get_one::("railpack").unwrap().clone(), + skopeo: args.get_one::("skopeo").unwrap().clone(), + buildkit_host: args.get_one::("buildkit_host").cloned(), + registry_auth_file: args.get_one::("registry_auth_file").cloned(), + secrets: vec![], + }; + for secret in args.get_many::("build_secret").into_iter().flatten() { + let (name, file) = secret + .split_once('=') + .context("build secret must be NAME=FILE, not a value")?; + ensure!(!file.is_empty(), "build secret file is missing"); + tools.secrets.push(BuildSecret { + name: name.into(), + file: file.into(), + }); + } + let (os, architecture) = args.get_one::("platform").unwrap().split_once('/').unwrap(); + let cancel = CancellationToken::new(); + let prepared = { + let prepare = prepare_container( + &declaration, + &loaded.config_dir, + spacetimedb_lib::container::ImagePlatform { + os: os.into(), + architecture: architecture.into(), + }, + &tools, + parent, + &LocalRunner, + cancel.clone(), + ); + tokio::pin!(prepare); + tokio::select! { + result = &mut prepare => result?, + signal = tokio::signal::ctrl_c() => { + signal?; + cancel.cancel(); + let _ = prepare.await; + anyhow::bail!("container build cancelled; no output was published"); + } + } + }; + let digest = prepared.metadata.manifest.digest; + prepared.persist(&output)?; + println!("Prepared {digest} at {}", output.display()); + Ok(()) +} diff --git a/crates/cli/src/subcommands/container/operations.rs b/crates/cli/src/subcommands/container/operations.rs new file mode 100644 index 00000000000..3236baca42d --- /dev/null +++ b/crates/cli/src/subcommands/container/operations.rs @@ -0,0 +1,315 @@ +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use reqwest::{ + header::{HeaderValue, AUTHORIZATION}, + Client, Method, StatusCode, Url, +}; +use serde::de::DeserializeOwned; +use spacetimedb_lib::{ + container::{endpoints::ContainerEndpoints, operations::*}, + Identity, Uuid, +}; +use std::time::Duration; + +const MAX_RESPONSE: usize = 64 * 1024; +pub(super) fn commands(command: Command) -> Command { + let base = |name: &'static str, about: &'static str| { + Command::new(name) + .about(about) + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg(crate::common_args::server()) + .arg(crate::common_args::yes()) + .arg( + Arg::new("json") + .long("json") + .action(ArgAction::SetTrue) + .help("Print the typed response as JSON"), + ) + }; + let action = |name, about| { + base(name, about) + .arg(Arg::new("request_id").long("request-id") + .help("Retry an original UUIDv7 request; DATABASE must be its recorded Identity")) + .after_help("Acceptance records the desired action; physical stop and readiness are asynchronous. A timeout must be retried with the original Identity and request ID printed on stderr.") + }; + command + .subcommand(base( + "status", + "Inspect container control state without opening its database", + )) + .subcommand(action("start", "Request container execution")) + .subcommand(action("stop", "Request container stop")) + .subcommand(action( + "restart", + "Request a new container instance and environment snapshot", + )) +} + +pub(super) async fn exec(config: &mut crate::Config, name: &str, args: &ArgMatches) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let database = args.get_one::("database").context("database is required")?; + validate_database(database)?; + let server = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; + let supplied_request = args + .try_get_one::("request_id") + .ok() + .flatten() + .map(|id| Uuid::parse_str(id).context("request-id must be a UUIDv7")) + .transpose()?; + if supplied_request.is_some() { + ensure!( + database.parse::().is_ok(), + "retry with the original database Identity printed by the first command" + ); + } + let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; + let client = ContainerClient::new( + server, + auth.to_header().context("container operations require a login")?, + )?; + if name == "status" { + let status = client.status(database).await?; + if args.get_flag("json") { + println!("{}", serde_json::to_string(&status)?); + } else { + print_status(&status); + } + return Ok(()); + } + let action = match name { + "start" => ContainerAction::Start, + "stop" => ContainerAction::Stop, + "restart" => ContainerAction::Restart, + _ => bail!("unsupported container operation"), + }; + // Names are resolved once before mutation. The replay target is immutable. + let identity = match database.parse::() { + Ok(identity) => identity, + Err(_) => client.status(database).await?.database_identity, + }; + let request_id = supplied_request.unwrap_or_else(|| Uuid::from_u128(uuid::Uuid::now_v7().as_u128())); + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), + )?; + spacetimedb_lib::deployment::operation_expiry_ms(request_id, now) + .context("request ID is not within its seven-day retry window")?; + // Structured retry parameters preserve the selected origin without + // presenting unescaped server text as an executable shell command. + let retry = serde_json::json!({ + "command": format!("container {}", action.path()), + "database": identity.to_hex().to_string(), + "request_id": request_id.to_string(), + "server": client.server.as_str(), + }); + eprintln!("Container request retry parameters: {retry}"); + let receipt = client.operate(identity, request_id, action).await.with_context(|| { + format!("request did not return a verified receipt; retain these exact retry parameters: {retry}") + })?; + if args.get_flag("json") { + println!("{}", serde_json::to_string(&receipt)?); + } else { + println!( + "Accepted {} for {} at generation {}", + action.path(), + identity.to_hex(), + receipt.generation + ); + } + Ok(()) +} + +struct ContainerClient { + http: Client, + server: Url, + authorization: HeaderValue, +} +impl ContainerClient { + fn new(server: Url, mut authorization: HeaderValue) -> Result { + ensure!( + authorization.as_bytes().starts_with(b"Bearer "), + "container operations require ordinary Bearer authentication" + ); + authorization.set_sensitive(true); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { + http, + server, + authorization, + }) + } + fn url(&self, database: &str, operation: &str) -> Result { + validate_database(database)?; + let mut url = self.server.clone(); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid server origin"))? + .pop_if_empty() + .extend(["v1", "database", database, "container", operation]); + Ok(url) + } + async fn status(&self, database: &str) -> Result { + let response = self + .http + .get(self.url(database, "status")?) + .header(AUTHORIZATION, self.authorization.clone()) + .send() + .await + .map_err(|_| anyhow::anyhow!("container status could not be reached on the selected server"))?; + let status: ContainerStatus = read_response(response, StatusCode::OK).await?; + if let Ok(identity) = database.parse::() { + ensure!( + status.database_identity == identity, + "status returned another database Identity" + ); + } + validate_status(&status)?; + Ok(status) + } + async fn operate( + &self, + identity: Identity, + request_id: Uuid, + action: ContainerAction, + ) -> Result { + let response = self + .http + .request(Method::POST, self.url(identity.to_hex().as_ref(), action.path())?) + .header(AUTHORIZATION, self.authorization.clone()) + .json(&ContainerOperationRequest { request_id }) + .send() + .await + .map_err(|_| anyhow::anyhow!("container operation response was not received"))?; + let receipt: ContainerOperationReceipt = read_response(response, StatusCode::ACCEPTED).await?; + ensure!( + receipt.database_identity == identity && receipt.request_id == request_id && receipt.action == action, + "container operation receipt does not match this request" + ); + Ok(receipt) + } +} +fn validate_database(database: &str) -> Result<()> { + ensure!( + !database.is_empty() + && database.len() <= 1024 + && !matches!(database, "." | "..") + && !database.chars().any(char::is_control), + "invalid database name or Identity" + ); + Ok(()) +} +async fn read_response(mut response: reqwest::Response, expected: StatusCode) -> Result { + let status = response.status(); + ensure!( + response.content_length().is_none_or(|size| size <= MAX_RESPONSE as u64), + "container response exceeds its size limit" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_RESPONSE.saturating_sub(bytes.len()), + "container response exceeds its size limit" + ); + bytes.extend_from_slice(&chunk); + } + if status != expected { + let error = serde_json::from_slice::(&bytes) + .ok() + .map(|value| value.error); + bail!( + "{} (HTTP {})", + match error { + Some(ContainerErrorCode::AccessDenied) => "database role does not permit this operation", + Some(ContainerErrorCode::NotFound) => "database was not found", + Some(ContainerErrorCode::InvalidRequest) => "container request is invalid or expired", + Some(ContainerErrorCode::Conflict) => "container request conflicts with current state", + Some(ContainerErrorCode::OutcomeUnknown) => + "container request outcome is unknown; retry the same request ID", + _ => "container service is unavailable on the selected server", + }, + status.as_u16() + ); + } + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid container response")) +} +fn validate_status(status: &ContainerStatus) -> Result<()> { + if let Some(state) = &status.operational + && let Some(instance) = &state.current_instance + { + ensure!( + instance.generation == state.generation, + "status instance is not the current generation" + ); + if let Some(id) = &instance.applied_env_generation { + ensure!(Uuid::parse_str(id).is_ok(), "invalid applied environment generation"); + } + } + if let EndpointStatus::Available { endpoints } = &status.endpoints { + super::url::validate(&ContainerEndpoints { + database_identity: status.database_identity, + endpoints: endpoints.clone(), + })?; + } + Ok(()) +} +fn print_status(status: &ContainerStatus) { + println!("Database: {}", status.database_identity.to_hex()); + if !status.published { + println!("Container: none published"); + } + if let Some(state) = &status.operational { + println!("Desired: {:?} (generation {})", state.desired_state, state.generation); + println!("Condition: {:?}", state.condition); + if let Some(instance) = &state.current_instance { + println!("Observed: {:?}", instance.state); + if let Some(environment) = &instance.applied_env_generation { + println!("Environment generation: {environment}"); + } + if let Some(code) = instance.exit_code { + println!("Exit code: {code}"); + } + if instance.oom_killed { + println!("Out of memory: yes"); + } + } else { + println!("Observed: no instance admitted for this generation"); + } + } + match &status.endpoints { + EndpointStatus::Available { endpoints } => { + for endpoint in endpoints { + println!("{}: {}", endpoint.name, endpoint.url); + } + } + EndpointStatus::Pending => println!("Endpoints: pending"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn target_path_encodes_one_database_and_rejects_dot_segments() { + let client = ContainerClient::new( + Url::parse("http://127.0.0.1:43123/").unwrap(), + HeaderValue::from_static("Bearer unusable"), + ) + .unwrap(); + assert_eq!( + client.url("name/child?x#y", "status").unwrap().path(), + "/v1/database/name%2Fchild%3Fx%23y/container/status" + ); + for invalid in ["", ".", "..", "name\n"] { + assert!(client.url(invalid, "status").is_err()); + } + } +} diff --git a/crates/cli/src/subcommands/container/url.rs b/crates/cli/src/subcommands/container/url.rs new file mode 100644 index 00000000000..b66eee410aa --- /dev/null +++ b/crates/cli/src/subcommands/container/url.rs @@ -0,0 +1,172 @@ +//! Anonymous discovery prints one validated public address, never a constructed +//! hostname, authentication token or private runtime address. + +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgMatches, Command}; +use reqwest::{redirect::Policy, Client, StatusCode}; +use spacetimedb_lib::{ + container::{endpoints::ContainerEndpoints, MAX_PORTS}, + Identity, +}; +use std::{collections::BTreeSet, time::Duration}; +use url::Url; + +const MAX_RESPONSE_BYTES: usize = 64 * 1024; + +pub(super) fn cli() -> Command { + Command::new("url") + .about("Print a container's published HTTPS URL") + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg( + Arg::new("port") + .long("port") + .help("Declared port name; required when several ports are published"), + ) + .arg(crate::common_args::server()) + .after_help("Discovery does not start the container or wait for readiness. No login is required.") +} + +pub(super) async fn exec(config: &crate::Config, args: &ArgMatches) -> Result<()> { + let server = args.get_one::("server").map(String::as_str); + let database = args.get_one::("database").context("database is required")?; + let port = args.get_one::("port").map(String::as_str); + let endpoints = fetch(&config.get_host_url(server)?, database).await?; + println!("{}", select(&endpoints, port)?); + Ok(()) +} + +async fn fetch(server: &str, database: &str) -> Result { + ensure!( + !database.is_empty() + && !matches!(database, "." | "..") + && database.len() <= 1024 + && !database.chars().any(char::is_control), + "invalid database name or Identity" + ); + let mut url = Url::parse(server).context("invalid server URL")?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && url.path() == "/", + "server must be an HTTP or HTTPS origin without credentials or a path" + ); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid server URL"))? + .clear() + .extend(["v1", "database", database, "container", "endpoints"]); + // This endpoint is public. Do not load a login, resolve an Identity through + // another request, follow redirects, or forward saved credentials. + let client = Client::builder() + .no_proxy() + .redirect(Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(15)) + .build()?; + let mut response = client + .get(url) + .send() + .await + .map_err(|_| anyhow::anyhow!("could not reach container endpoint discovery on the selected server"))?; + let status = response.status(); + if status == StatusCode::NOT_FOUND { + bail!("database or container endpoint discovery was not found on the selected server"); + } + if status == StatusCode::SERVICE_UNAVAILABLE { + bail!("container endpoints are not available yet; retry shortly"); + } + if status != StatusCode::OK { + bail!("container endpoint discovery failed (HTTP {})", status.as_u16()); + } + ensure!( + response + .content_length() + .is_none_or(|length| length <= MAX_RESPONSE_BYTES as u64), + "container endpoint discovery response exceeds its size limit" + ); + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container endpoint discovery response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_RESPONSE_BYTES.saturating_sub(body.len()), + "container endpoint discovery response exceeds its size limit" + ); + body.extend_from_slice(&chunk); + } + let endpoints: ContainerEndpoints = + serde_json::from_slice(&body).map_err(|_| anyhow::anyhow!("invalid container endpoint discovery response"))?; + if let Ok(identity) = database.parse::() { + ensure!( + identity == endpoints.database_identity, + "container endpoint discovery returned another database Identity" + ); + } + validate(&endpoints)?; + Ok(endpoints) +} + +pub(super) fn validate(endpoints: &ContainerEndpoints) -> Result<()> { + ensure!( + endpoints.endpoints.len() <= MAX_PORTS, + "too many container endpoints in discovery response" + ); + let mut names = BTreeSet::new(); + for endpoint in &endpoints.endpoints { + let name = endpoint.name.as_bytes(); + ensure!( + (1..=32).contains(&name.len()) + && name[0].is_ascii_lowercase() + && name + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') + && names.insert(&endpoint.name), + "invalid or duplicate port name in discovery response" + ); + let address = Url::parse(&endpoint.url).context("invalid container endpoint URL")?; + ensure!( + endpoint.url.len() <= 2048 + && endpoint.url.is_ascii() + && !endpoint.url.bytes().any(|byte| byte.is_ascii_control()) + && address.as_str() == endpoint.url + && address.scheme() == "https" + && matches!(address.host(), Some(url::Host::Domain(_))) + && address.username().is_empty() + && address.password().is_none() + && address.port().is_none() + && address.query().is_none() + && address.fragment().is_none() + && address.path() == "/", + "discovery returned an invalid public HTTPS endpoint" + ); + } + Ok(()) +} + +fn select<'a>(endpoints: &'a ContainerEndpoints, port: Option<&str>) -> Result<&'a str> { + match (port, endpoints.endpoints.as_slice()) { + (_, []) => bail!("this database has no declared public HTTP ports"), + (None, [endpoint]) => Ok(&endpoint.url), + (Some(port), entries) => entries + .iter() + .find(|endpoint| endpoint.name == port) + .map(|endpoint| endpoint.url.as_str()) + .context("the requested port name is not published by this database"), + (None, entries) => bail!( + "several ports are published; select one with --port: {}", + entries + .iter() + .map(|entry| entry.name.as_str()) + .collect::>() + .join(", ") + ), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/container/url/tests.rs b/crates/cli/src/subcommands/container/url/tests.rs new file mode 100644 index 00000000000..37a544d8b75 --- /dev/null +++ b/crates/cli/src/subcommands/container/url/tests.rs @@ -0,0 +1,198 @@ +use super::*; +use axum::{ + body::{Body, Bytes}, + extract::Path, + http::{header, HeaderMap}, + response::{IntoResponse, Response}, + routing::get, + Json, Router, +}; +use spacetimedb_lib::container::{endpoints::ContainerEndpoint, PortProtocol}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn response() -> ContainerEndpoints { + ContainerEndpoints { + database_identity: Identity::ZERO, + endpoints: vec![ContainerEndpoint { + name: "http".into(), + protocol: PortProtocol::Http, + url: "https://aaaqeayeaudaocajbifqydiob4.container.example.net/".into(), + }], + } +} + +struct Server { + origin: String, + stop: Option>, + task: tokio::task::JoinHandle>, +} + +impl Server { + async fn start(router: Router) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let origin = format!("http://{}", listener.local_addr()?); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + Ok(Self { + origin, + stop: Some(stop), + task, + }) + } + + async fn shutdown(mut self) -> Result<()> { + self.stop.take().unwrap().send(()).ok(); + tokio::time::timeout(Duration::from_secs(5), &mut self.task).await???; + Ok(()) + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[tokio::test] +async fn discovery_is_anonymous_and_encodes_the_entire_database_path_segment() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let count = requests.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/endpoints", + get(move |Path(database): Path, headers: HeaderMap| { + let count = count.clone(); + async move { + assert_eq!(database, "project/child?query=value#fragment"); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + count.fetch_add(1, Ordering::SeqCst); + Json(response()) + } + }), + )) + .await?; + for invalid in [".", ".."] { + assert!(fetch(&server.origin, invalid).await.is_err()); + } + assert_eq!(requests.load(Ordering::SeqCst), 0); + let discovered = fetch(&server.origin, "project/child?query=value#fragment").await?; + assert_eq!(select(&discovered, None)?, response().endpoints[0].url); + assert_eq!(requests.load(Ordering::SeqCst), 1); + server.shutdown().await +} + +#[test] +fn port_selection_never_picks_an_arbitrary_endpoint() -> Result<()> { + let mut endpoints = response(); + assert_eq!(select(&endpoints, Some("http"))?, endpoints.endpoints[0].url); + assert!(select(&endpoints, Some("absent")).is_err()); + let mut second = endpoints.endpoints[0].clone(); + second.name = "metrics".into(); + second.url = "https://bbbbbbbbbbbbbbbbbbbbbbbbaa.container.example.net/".into(); + endpoints.endpoints.push(second); + assert!(select(&endpoints, None) + .unwrap_err() + .to_string() + .contains("--port: http, metrics")); + assert_eq!(select(&endpoints, Some("metrics"))?, endpoints.endpoints[1].url); + endpoints.endpoints.clear(); + assert!(select(&endpoints, None) + .unwrap_err() + .to_string() + .contains("no declared public HTTP ports")); + Ok(()) +} + +#[test] +fn endpoint_output_rejects_terminal_controls_credentials_and_ambiguous_names() { + let mut endpoints = response(); + for address in [ + "javascript:alert(1)", + "http://application.example/", + "https://user:secret@application.example/", + "https://127.0.0.1/", + "https://application.example/path", + "https://application.example/?secret=value", + "https://application.example/#fragment", + "https://application.example:8443/", + "https://application.example/\n", + "\u{1b}[2Jhttps://application.example/", + ] { + endpoints.endpoints[0].url = address.into(); + assert!(validate(&endpoints).is_err()); + } + endpoints = response(); + for name in ["", "HTTP", "1http", "http\n", "http/metrics"] { + endpoints.endpoints[0].name = name.into(); + assert!(validate(&endpoints).is_err()); + } + endpoints = response(); + endpoints.endpoints.push(endpoints.endpoints[0].clone()); + assert!(validate(&endpoints).is_err()); +} + +#[tokio::test] +async fn discovery_bounds_streams_rejects_redirects_and_preserves_pending_errors() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let count = requests.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/endpoints", + get(move |Path(database): Path| { + let count = count.clone(); + async move { + count.fetch_add(1, Ordering::SeqCst); + match database.as_str() { + "pending" => (StatusCode::SERVICE_UNAVAILABLE, "private diagnostic").into_response(), + "missing" => StatusCode::NOT_FOUND.into_response(), + "redirect" => (StatusCode::FOUND, [(header::LOCATION, "/unexpected")]).into_response(), + "invalid" => Json(serde_json::json!({"private diagnostic": "do not print"})).into_response(), + "oversize" => Response::new(Body::from_stream(futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from(vec![b' '; MAX_RESPONSE_BYTES])), + Ok(Bytes::from_static(b"x")), + ]))), + _ => Json(response()).into_response(), + } + } + }), + )) + .await?; + for (database, expected) in [ + ("pending", "not available yet"), + ("missing", "not found"), + ("redirect", "HTTP 302"), + ("invalid", "invalid container endpoint discovery response"), + ("oversize", "size limit"), + ] { + let error = format!("{:#}", fetch(&server.origin, database).await.unwrap_err()); + assert!(error.contains(expected), "{error}"); + assert!(!error.contains("private diagnostic")); + } + let other_identity = Identity::from_u256(1_u64.into()).to_string(); + assert!(fetch(&server.origin, &other_identity) + .await + .unwrap_err() + .to_string() + .contains("another database Identity")); + assert_eq!(requests.load(Ordering::SeqCst), 6); + server.shutdown().await +} + +#[test] +fn url_command_requires_database_and_parses_explicit_server_and_port() { + assert!(cli().try_get_matches_from(["url"]).is_err()); + let args = cli() + .try_get_matches_from(["url", "demo", "--server", "http://127.0.0.1:3000", "--port", "http"]) + .unwrap(); + assert_eq!(args.get_one::("database").unwrap(), "demo"); + assert_eq!(args.get_one::("server").unwrap(), "http://127.0.0.1:3000"); + assert_eq!(args.get_one::("port").unwrap(), "http"); +} diff --git a/crates/cli/src/subcommands/describe.rs b/crates/cli/src/subcommands/describe.rs index e774224855c..da2046bc19d 100644 --- a/crates/cli/src/subcommands/describe.rs +++ b/crates/cli/src/subcommands/describe.rs @@ -98,6 +98,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error let api = ClientApi::new(conn); let module_def = api.module_def().await?; + let canonical = spacetimedb_schema::def::ModuleDef::try_from(module_def.clone())?; if json { fn sats_to_json(v: &T) -> serde_json::Result { @@ -105,18 +106,25 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error } let json = match entity { Some((EntityType::Reducer, reducer_name)) => { + let source_name = &canonical + .reducer(reducer_name) + .context("no such reducer")? + .accessor_name; let reducer = module_def - .reducers - .iter() - .find(|r| *r.name == *reducer_name) + .reducers() + .into_iter() + .flatten() + .find(|r| *r.source_name == **source_name) .context("no such reducer")?; sats_to_json(reducer)? } Some((EntityType::Table, table_name)) => { + let source_name = &canonical.table(table_name).context("no such table")?.accessor_name; let table = module_def - .tables - .iter() - .find(|t| *t.name == *table_name) + .tables() + .into_iter() + .flatten() + .find(|t| *t.source_name == **source_name) .context("no such table")?; sats_to_json(table)? } diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs index 8badb72773a..a14aff0d16a 100644 --- a/crates/cli/src/subcommands/dev.rs +++ b/crates/cli/src/subcommands/dev.rs @@ -928,6 +928,10 @@ fn determine_publish_configs<'a>( } if !publish_configs.is_empty() { + anyhow::ensure!( + publish_configs.iter().all(|target| target.container().is_none()), + "spacetime dev does not yet run container targets; use spacetime publish for managed publication" + ); return Ok(publish_configs); } diff --git a/crates/cli/src/subcommands/generate.rs b/crates/cli/src/subcommands/generate.rs index 6e5378fede5..234cd3eef82 100644 --- a/crates/cli/src/subcommands/generate.rs +++ b/crates/cli/src/subcommands/generate.rs @@ -260,7 +260,7 @@ pub fn cli() -> clap::Command { .long("include-private") .action(SetTrue) .default_value("false") - .help("Include private tables and functions in generated code (types are always included)."), + .help("Include private tables and private/internal non-lifecycle functions (types are always included)."), ) .arg(common_args::yes()) .arg( diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index 5d993a88b3e..677a5e9ce9b 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -1,5 +1,6 @@ pub mod build; pub mod call; +pub mod container; pub mod db_arg_resolution; pub mod delete; pub mod describe; @@ -14,7 +15,6 @@ pub mod logs; pub mod publish; pub mod repl; pub mod server; -pub mod sidecar; pub mod sql; pub mod start; pub mod subscribe; diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index dd710da2832..68a90f6e178 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -19,6 +19,8 @@ use crate::util::{add_auth_header_opt, get_auth_header, strip_verbatim_prefix, A use crate::util::{decode_identity, y_or_n}; use crate::{build, common_args}; +mod managed; + /// Individual prompts that `--yes` can suppress. `All` is a shorthand for every category below. #[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] #[clap(rename_all = "kebab-case")] @@ -83,7 +85,7 @@ fn yes_flags_from_args(args: &ArgMatches) -> YesFlags { /// Build the CommandSchema for publish command pub fn build_publish_schema(command: &clap::Command) -> Result { - CommandSchemaBuilder::new() + managed::exclude_args(CommandSchemaBuilder::new()) .key(Key::new("database").from_clap("name|identity").required()) .key(Key::new("server")) .key(Key::new("module_path").module_specific()) @@ -171,7 +173,7 @@ pub fn get_filtered_publish_configs<'a>( let configs: Vec = filtered_targets .into_iter() .map(|target| { - let config = CommandConfig::new(schema, target.fields, args)?; + let config = CommandConfig::new(schema, target.fields, args)?.with_container(target.container); config.validate()?; Ok(config) }) @@ -189,7 +191,7 @@ pub fn get_filtered_publish_configs<'a>( } pub fn cli() -> clap::Command { - clap::Command::new("publish") + managed::add_args(clap::Command::new("publish") .about("Create and update a SpacetimeDB database") .arg( common_args::clear_database() @@ -319,7 +321,7 @@ i.e. only lowercase ASCII letters and numbers, separated by dashes."), .action(SetTrue) .help("Use NativeAOT-LLVM compilation for C# modules (experimental, Windows only)") ) - .after_help("Run `spacetime help publish` for more detailed information.") + .after_help("Run `spacetime help publish` for more detailed information.")) } fn confirm_and_clear( @@ -381,6 +383,9 @@ pub async fn exec_with_options( quiet_config: bool, pre_loaded_config: Option<&LoadedConfig>, ) -> Result<(), anyhow::Error> { + if args.get_one::("resume_publication").is_some() { + return managed::resume(&mut config, args, yes_flags_from_args(args)).await; + } // Build schema let cmd = cli(); let schema = build_publish_schema(&cmd)?; @@ -496,20 +501,7 @@ async fn execute_publish_configs<'a>( }) }; - if using_config { - if let Some(path_to_project) = path_to_project.as_ref() { - println!( - "Publishing module {} to database '{}'", - strip_verbatim_prefix(path_to_project).display(), - name_or_identity.unwrap() - ); - } else { - println!( - "Publishing precompiled module to database '{}'", - name_or_identity.unwrap() - ); - } - } + managed::validate_target_options(&command_config)?; let database_host = config.get_host_url(server)?; let build_options = command_config .get_one::("build_options")? @@ -530,6 +522,36 @@ async fn execute_publish_configs<'a>( let (name_or_identity, parent) = validate_name_and_parent(name_or_identity, parent)?; + if managed::try_execute( + &command_config, + config_dir, + &database_host, + &auth_header, + name_or_identity, + parent, + clear_database, + yes, + ) + .await? + { + continue; + } + + if using_config { + if let Some(path_to_project) = path_to_project.as_ref() { + println!( + "Publishing module {} to database '{}'", + strip_verbatim_prefix(path_to_project).display(), + name_or_identity.unwrap() + ); + } else { + println!( + "Publishing precompiled module to database '{}'", + name_or_identity.unwrap() + ); + } + } + if let Some(path_to_project) = path_to_project.as_ref() && !path_to_project.exists() { diff --git a/crates/cli/src/subcommands/publish/managed.rs b/crates/cli/src/subcommands/publish/managed.rs new file mode 100644 index 00000000000..f1e4bd055f0 --- /dev/null +++ b/crates/cli/src/subcommands/publish/managed.rs @@ -0,0 +1,969 @@ +//! Managed publication frontend. The resume journal owns every byte needed to +//! retry; project configuration and mutable image tags are only read initially. +use super::{confirm_major_version_upgrade, YesFlags}; +use crate::{ + common_args::ClearMode, + config::Config, + container::{ + self, + oci::ArtifactKind, + process::LocalRunner, + publish::{ + self, + client::{ObjectRef, PublisherClient, UploadKind}, + journal::{Journal, Record, UploadRecord}, + Outcome, + }, + BuildSecret, BuildTools, + }, + spacetime_config::{CommandConfig, CommandSchemaBuilder}, + util::{get_auth_header, y_or_n, AuthHeader}, +}; +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_client_api_messages::name::{is_identity, DomainName, PrePublishResult}; +use spacetimedb_lib::{ + container::{ContainerAction, ImagePlatform}, + deployment::{self, api::*, manifest::*, ModuleAction, PublishEnvelope, UserModule, UserModuleKind}, + Identity, Uuid, +}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; +use tokio_util::sync::CancellationToken; + +const ARGUMENTS: &[&str] = &[ + "managed", + "container_platform", + "artifact_endpoint", + "remove_container", + "remove_module", + "publication_state_dir", + "resume_publication", + "publication_wait", + "buildkit_host", + "buildctl", + "railpack", + "skopeo", + "registry_auth_file", + "build_secret", +]; +pub(super) fn exclude_args(mut schema: CommandSchemaBuilder) -> CommandSchemaBuilder { + for arg in ARGUMENTS { + schema = schema.exclude(*arg); + } + schema +} +pub(super) fn add_args(mut command: Command) -> Command { + command = command + .arg( + Arg::new("managed") + .long("managed") + .action(ArgAction::SetTrue) + .help("Use managed deployment publication, including for a module-only database"), + ) + .arg( + Arg::new("container_platform") + .long("container-platform") + .value_parser(["linux/amd64", "linux/arm64"]) + .help("Required platform when publishing a container declaration"), + ) + .arg( + Arg::new("artifact_endpoint") + .long("artifact-endpoint") + .help("Explicitly authorize this exact artifact URL to receive the publisher credential"), + ) + .arg( + Arg::new("remove_container") + .long("remove-container") + .action(ArgAction::SetTrue) + .help("Explicitly remove the container while preserving the module unless separately changed"), + ) + .arg( + Arg::new("remove_module") + .long("remove-module") + .action(ArgAction::SetTrue) + .conflicts_with_all(["module_path", "wasm_file", "js_file"]) + .help("Replace the module with the versioned empty module after migration preflight"), + ) + .arg( + Arg::new("publication_state_dir") + .long("publication-state-dir") + .value_parser(clap::value_parser!(PathBuf)) + .help("Private local directory retaining managed publication bytes and progress"), + ) + .arg( + Arg::new("resume_publication") + .long("resume-publication") + .value_parser(clap::value_parser!(PathBuf)) + .conflicts_with_all([ + "managed", + "remove_container", + "remove_module", + "module_path", + "wasm_file", + "js_file", + "container_platform", + "name|identity", + "parent", + "organization", + "clear-database", + ]) + .help("Resume this operation directory without rebuilding or reading spacetime.json"), + ) + .arg( + Arg::new("publication_wait") + .long("publication-wait") + .default_value("60") + .value_parser(clap::value_parser!(u64).range(0..=3600)) + .help("Seconds to wait for managed activation; pending operations retain their resume directory"), + ) + .arg( + Arg::new("buildkit_host") + .long("buildkit-host") + .help("Explicit local BuildKit Unix socket for container source builds"), + ) + .arg( + Arg::new("registry_auth_file") + .long("registry-auth-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Explicit registry auth JSON; otherwise image preparation is anonymous"), + ) + .arg( + Arg::new("build_secret") + .long("build-secret") + .action(ArgAction::Append) + .value_name("NAME=FILE") + .help("Build secret file, separate from runtime env_keys"), + ); + for (tool, help) in [ + ( + "buildctl", + "Path to the buildctl executable for Dockerfile or Railpack builds", + ), + ( + "railpack", + "Path to the Railpack executable for explicitly selected Railpack builds", + ), + ( + "skopeo", + "Path to the skopeo executable for copying prebuilt OCI images", + ), + ] { + command = command.arg( + Arg::new(tool) + .help(help) + .long(tool) + .default_value(tool) + .value_parser(clap::value_parser!(PathBuf)), + ); + } + command +} +fn approved(args: &ArgMatches) -> Option<&str> { + args.get_one::("artifact_endpoint").map(String::as_str) +} +fn wait(args: &ArgMatches) -> Duration { + Duration::from_secs(*args.get_one::("publication_wait").unwrap_or(&60)) +} +pub(super) fn validate_target_options(target: &CommandConfig<'_>) -> Result<()> { + let args = target.matches(); + if target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module") + { + ensure!( + !target.get_one::("anon_identity")?.unwrap_or(false), + "managed publication requires an authenticated publisher" + ); + } + Ok(()) +} +fn authenticated(server: &str, auth: &AuthHeader) -> Result { + PublisherClient::new( + server, + auth.to_header() + .context("managed publication requires an authenticated publisher; anonymous publication is unsupported")?, + ) +} +pub(super) async fn resume(config: &mut Config, args: &ArgMatches, yes: YesFlags) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let server = config.get_host_url(selection)?; + let mut journal = Journal::open(args.get_one::("resume_publication").unwrap())?; + ensure!( + container::publish::client::endpoint(&server)?.as_str() == journal.record.server, + "resume server differs from the original publication endpoint" + ); + let auth = get_auth_header(config, false, selection, !yes.skip_login).await?; + let client = authenticated(&server, &auth)?; + execute(&client, &mut journal, args).await +} + +/// False is only returned for an ordinary, un-managed module publication. +/// Once managed intent or a managed revision is known, errors cannot fall back. +#[allow(clippy::too_many_arguments)] +pub(super) async fn try_execute( + target: &CommandConfig<'_>, + config_dir: Option<&Path>, + server: &str, + auth: &AuthHeader, + name: Option<&str>, + parent: Option<&str>, + clear: ClearMode, + yes: YesFlags, +) -> Result { + let args = target.matches(); + let explicit = target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module"); + if !explicit && name.is_none() { + return Ok(false); + } + let client = match authenticated(server, auth) { + Ok(client) => client, + Err(error) if explicit => return Err(error), + Err(_) => return Ok(false), + }; + try_execute_with_client(target, config_dir, client, name, parent, clear, yes).await +} + +async fn try_execute_with_client( + target: &CommandConfig<'_>, + config_dir: Option<&Path>, + client: PublisherClient, + name: Option<&str>, + parent: Option<&str>, + clear: ClearMode, + yes: YesFlags, +) -> Result { + let args = target.matches(); + let explicit = target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module"); + let caps = client.capabilities().await?; + let Some(caps) = caps else { + ensure!( + !explicit, + "this server does not support managed publication; no module-only fallback was attempted" + ); + return Ok(false); + }; + let prior = if let Some(name) = name { + client.deployment(name).await? + } else { + None + }; + if !explicit && prior.as_ref().and_then(|p| p.revision).is_none() { + return Ok(false); + } + ensure!( + caps.enabled && caps.version == deployment::PUBLISH_PROTOCOL_VERSION, + "managed publication is disabled or incompatible on this server" + ); + ensure!( + clear == ClearMode::Never, + "managed publication does not support --delete-data; resolve migrations without replacing database storage" + ); + ensure!( + !(target.container().is_some() && args.get_flag("remove_container")), + "a container declaration and --remove-container cannot both select this target" + ); + if prior.is_none() { + ensure!( + !name.is_some_and(is_identity), + "a new database Identity must be generated by the reservation service; select a name or omit DATABASE" + ); + } + let artifact = client.artifact_endpoint( + caps.artifact_endpoint + .as_deref() + .context("server did not advertise an artifact endpoint")?, + approved(args), + )?; + let permission = client.permission().await?; + if target.container().is_some() || prior.is_none() { + ensure!( + permission.can_publish, + "the current publisher does not have container publication permission" + ); + } + if !container::publish::client::is_loopback(client.server()) { + ensure!( + y_or_n( + yes.publish_to_remote, + "Publish this managed deployment to the selected remote server?" + )?, + "publication cancelled" + ); + } + let cwd = std::env::current_dir()?; + let base = args + .get_one::("publication_state_dir") + .cloned() + .unwrap_or_else(|| config_dir.unwrap_or(&cwd).join(".spacetime/publications")); + std::fs::create_dir_all(&base)?; + let cancel = CancellationToken::new(); + let prepare = prepare_request( + &client, + target, + config_dir.unwrap_or(&cwd), + &base, + prior.as_ref(), + name, + parent, + permission.identity, + artifact.as_str(), + yes, + cancel.clone(), + ); + tokio::pin!(prepare); + let mut journal = tokio::select! { + result = &mut prepare => result?, + signal = tokio::signal::ctrl_c() => { + signal?; cancel.cancel(); let _ = prepare.await; + bail!("managed preparation cancelled before admission"); + } + }; + execute(&client, &mut journal, args).await?; + Ok(true) +} + +#[allow(clippy::too_many_arguments)] +async fn prepare_request( + client: &PublisherClient, + target: &CommandConfig<'_>, + config_dir: &Path, + base: &Path, + prior: Option<&DeploymentStatus>, + name: Option<&str>, + parent: Option<&str>, + publisher: Identity, + artifact_endpoint: &str, + yes: YesFlags, + cancel: CancellationToken, +) -> Result { + let args = target.matches(); + let image = if let Some(declaration) = target.container() { + let (os, architecture) = args + .get_one::("container_platform") + .context("publishing a container requires --container-platform linux/amd64 or linux/arm64")? + .split_once('/') + .unwrap(); + Some( + container::prepare_container( + declaration, + config_dir, + ImagePlatform { + os: os.into(), + architecture: architecture.into(), + }, + &tools(args)?, + base, + &LocalRunner, + cancel.clone(), + ) + .await?, + ) + } else { + None + }; + let has_module = ["module_path", "wasm_file", "js_file"] + .iter() + .any(|key| target.is_from_cli(key) || target.get_config_value(key).is_some()); + ensure!( + !(has_module && args.get_flag("remove_module")), + "module configuration and --remove-module cannot both select this target" + ); + let module = if !args.get_flag("remove_module") + && (has_module || (target.container().is_none() && !args.get_flag("remove_container"))) + { + Some(load_module(target, config_dir).await?) + } else { + None + }; + let module_action = if args.get_flag("remove_module") { + ModuleAction::Remove + } else if let Some((kind, bytes)) = &module { + ModuleAction::Set(UserModule { + kind: *kind, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }) + } else { + ModuleAction::Keep + }; + let container_action = if let Some(image) = &image { + ContainerAction::Set(image.metadata.container.clone()) + } else if args.get_flag("remove_container") { + ContainerAction::Remove + } else { + ContainerAction::Keep + }; + let envelope = PublishEnvelope { + version: deployment::PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), + expected_revision: prior.and_then(|p| p.revision), + module_action, + container_action, + }; + let deployment = envelope.resolve(prior.map(|p| &p.deployment), &Default::default())?; + let module_artifact = if let Some((_, bytes)) = &module { + ModuleArtifact { + digest: spacetimedb_oci::sha256(bytes), + size_bytes: bytes.len() as u64, + } + } else if let Some(prior) = prior.filter(|_| !matches!(envelope.module_action, ModuleAction::Remove)) { + let artifact = prior.module_artifact; + ModuleArtifact { + digest: artifact.digest, + size_bytes: artifact.size_bytes, + } + } else { + deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT + }; + let migration_policy = if let Some(prior) = prior { + if let Some((kind, bytes)) = &module { + migration(client, prior.database_identity, *kind, bytes, target, yes).await? + } else if matches!(envelope.module_action, ModuleAction::Remove) { + migration( + client, + prior.database_identity, + UserModuleKind::Wasm, + deployment::SYSTEM_EMPTY_MODULE_V1_BYTES, + target, + yes, + ) + .await? + } else { + PreparedMigrationPolicy::Compatible + } + } else { + PreparedMigrationPolicy::Compatible + }; + let creation = if prior.is_none() { + let parent = if let Some(parent) = parent { + Some( + client + .deployment(parent) + .await? + .context("parent database does not exist or is not accessible")? + .database_identity, + ) + } else { + None + }; + let organization = target + .get_one::("organization")? + .map(|value| { + value + .parse::() + .context("managed publication currently requires --organization IDENTITY, not an organization name") + }) + .transpose()?; + Some(CreationOptions { + parent, + organization, + num_replicas: target.get_one::("num_replicas")?.map(u32::from), + enforce_anti_affinity: true, + }) + } else { + None + }; + let request = PublishRequest { + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope, + deployment, + module_artifact, + migration_policy, + }), + creation, + image_source: image.as_ref().map(|image| ArtifactReference { + digest: image.metadata.manifest.digest, + size_bytes: image.metadata.manifest.size, + }), + }; + request.manifest.validate(&Default::default())?; + let request_json = serde_json::to_string(&request)?; + let mut uploads = image + .as_ref() + .map(|image| { + image + .metadata + .objects + .iter() + .map(|object| UploadRecord { + kind: match object.kind { + ArtifactKind::Manifest => UploadKind::Manifest, + ArtifactKind::Config => UploadKind::Config, + ArtifactKind::Layer => UploadKind::Layer, + }, + object: ObjectRef { + digest: object.descriptor.digest, + size: object.descriptor.size, + }, + session: None, + }) + .collect::>() + }) + .unwrap_or_default(); + if module.is_some() { + uploads.push(UploadRecord { + kind: UploadKind::Module, + object: ObjectRef { + digest: module_artifact.digest, + size: module_artifact.size_bytes, + }, + session: None, + }); + } + let requested_name = if prior.is_none() { + name.map(|name| name.parse::().map(|name| name.to_string())) + .transpose()? + } else { + None + }; + let record = Record { + version: 1, + server: client.server().to_string(), + artifact_endpoint: artifact_endpoint.into(), + publisher, + database: prior.map(|p| p.database_identity), + reservation: request.creation.clone().map(|options| ReserveDatabaseRequest { + version: deployment::PUBLISH_PROTOCOL_VERSION, + operation_id: request.manifest.current().envelope.operation_id, + options, + }), + requested_name, + request_digest: spacetimedb_oci::sha256(request_json.as_bytes()), + request_json, + uploads, + submitted: false, + status: None, + name_confirmed: false, + naming_attempted: false, + }; + Journal::create(base, record, image, module.as_ref().map(|(_, bytes)| bytes.as_slice())) +} +fn tools(args: &ArgMatches) -> Result { + let mut tools = BuildTools { + buildctl: args.get_one::("buildctl").unwrap().clone(), + railpack: args.get_one::("railpack").unwrap().clone(), + skopeo: args.get_one::("skopeo").unwrap().clone(), + buildkit_host: args.get_one::("buildkit_host").cloned(), + registry_auth_file: args.get_one::("registry_auth_file").cloned(), + secrets: vec![], + }; + for value in args.get_many::("build_secret").into_iter().flatten() { + let (name, file) = value.split_once('=').context("build secret must be NAME=FILE")?; + ensure!(!file.is_empty(), "build secret file is missing"); + tools.secrets.push(BuildSecret { + name: name.into(), + file: file.into(), + }); + } + Ok(tools) +} +async fn load_module(target: &CommandConfig<'_>, config_dir: &Path) -> Result<(UserModuleKind, Vec)> { + ensure!( + !target.get_one::("native_aot")?.unwrap_or(false), + "managed NativeAOT builds are not yet supported; build separately and pass --bin-path" + ); + let (path, kind) = if let Some(path) = target.get_resolved_path("wasm_file", Some(config_dir))? { + (path, "Wasm") + } else if let Some(path) = target.get_resolved_path("js_file", Some(config_dir))? { + (path, "Js") + } else { + let path = target + .get_resolved_path("module_path", Some(config_dir))? + .unwrap_or_else(|| super::default_publish_module_path(config_dir)); + crate::build::exec_with_argstring(&path, &target.get_one::("build_options")?.unwrap_or_default()) + .await? + }; + let kind = match kind { + "Wasm" => UserModuleKind::Wasm, + "Js" => UserModuleKind::Js, + _ => bail!("unsupported managed module kind"), + }; + let metadata = tokio::fs::metadata(&path).await?; + ensure!( + metadata.is_file() && metadata.len() > 0 && metadata.len() <= MAX_MODULE_ARTIFACT_BYTES, + "module must be a regular file of at most 32 MiB" + ); + use tokio::io::AsyncReadExt as _; + let mut bytes = Vec::new(); + tokio::fs::File::open(&path) + .await? + .take(MAX_MODULE_ARTIFACT_BYTES + 1) + .read_to_end(&mut bytes) + .await?; + ensure!( + !bytes.is_empty() && bytes.len() as u64 <= MAX_MODULE_ARTIFACT_BYTES, + "module exceeds 32 MiB" + ); + Ok((kind, bytes)) +} +async fn migration( + client: &PublisherClient, + database: Identity, + kind: UserModuleKind, + bytes: &[u8], + target: &CommandConfig<'_>, + yes: YesFlags, +) -> Result { + let pre = client + .preflight( + database, + match kind { + UserModuleKind::Wasm => "Wasm", + UserModuleKind::Js => "Js", + }, + bytes, + ) + .await?; + match pre { + PrePublishResult::ManualMigrate(_) => { + bail!("managed publication requires manual migration; no storage was cleared") + } + PrePublishResult::AutoMigrate(auto) => { + if auto.major_version_upgrade { + confirm_major_version_upgrade(yes.migrate_major_version)?; + } + println!("{}", auto.migrate_plan); + if auto.break_clients { + ensure!( + y_or_n( + yes.break_clients || target.get_one::("break_clients")?.unwrap_or(false), + "These changes will BREAK existing clients. Proceed?" + )?, + "publication cancelled" + ); + } + Ok(PreparedMigrationPolicy::BreakClients(auto.token)) + } + } +} +async fn execute(client: &PublisherClient, journal: &mut Journal, args: &ArgMatches) -> Result<()> { + println!( + "Publication {}. Resume directory: {}", + journal.operation()?, + journal.directory().display() + ); + let cancel = CancellationToken::new(); + let operation = publish::run(client, journal, approved(args), wait(args), cancel.clone()); + tokio::pin!(operation); + let outcome = tokio::select! { + result = &mut operation => result?, + signal = tokio::signal::ctrl_c() => { signal?; cancel.cancel(); bail!("publication interrupted; resume the same directory to determine its outcome"); } + }; + match outcome { + Outcome::Complete(status) => println!( + "Deployment {} is active on {}", + status.proposed_revision, status.database_identity + ), + Outcome::Pending(status) => println!( + "Publication {} is {:?} on {}; resume to observe activation", + status.operation_id, status.phase, status.database_identity + ), + Outcome::Aborted(status) => bail!( + "publication {} was aborted before commit on {}; resume state retained", + status.operation_id, + status.database_identity + ), + Outcome::NamingUnconfirmed(status) => bail!( + "deployment is active on {}, but naming was not confirmed; publication must not be repeated", + status.database_identity + ), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::container::publish::tests::{database, Fixture}; + use crate::spacetime_config::SpacetimeConfig; + use serde_json::json; + use spacetimedb_paths::FromPathUnchecked as _; + use std::collections::HashMap; + + #[tokio::test] + async fn container_only_frontend_uploads_verified_closure_with_server_reserved_identity() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let layout = temporary.path().join("layout"); + let selected = crate::container::tests::fixture(&layout); + let declaration = crate::container::tests::declaration(json!({"oci_ref":"oci:layout"})); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let state = temporary.path().join("state"); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--container-platform", + "linux/amd64", + "--publication-state-dir", + state.to_str().unwrap(), + "--publication-wait", + "0", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args) + .unwrap() + .with_container(Some(declaration)); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + { + let snapshot = fixture.state.lock().unwrap(); + assert_eq!(snapshot.reservations.len(), 1); + assert_eq!(snapshot.submits.len(), 1); + assert_eq!(snapshot.begin_count, 3); + let request: PublishRequest = serde_json::from_slice(&snapshot.submits[0]).unwrap(); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Keep + )); + assert!(matches!( + request.manifest.current().deployment.current().module, + deployment::ModuleComponent::SystemEmpty(1) + )); + assert_eq!( + request.manifest.current().module_artifact, + deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT + ); + assert_eq!(request.image_source.unwrap().digest, selected.digest); + let dir = state.join(request.manifest.current().envelope.operation_id.to_string()); + let journal = Journal::open(&dir).unwrap(); + assert_eq!(journal.record.database, Some(database())); + let bytes = std::fs::read_to_string(dir.join("publication.json")).unwrap(); + assert!(!bytes.contains("private-image-value")); + } + fixture.close().await; + } + #[tokio::test] + async fn existing_managed_module_update_keeps_container_and_preflights_without_pro() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let wasm = temporary.path().join("module.wasm"); + std::fs::write(&wasm, deployment::SYSTEM_EMPTY_MODULE_V1_BYTES).unwrap(); + let prior_request = fixture.record(false, false).request().unwrap(); + let prior = DeploymentStatus { + database_identity: database(), + revision: Some(prior_request.manifest.current().deployment.revision().unwrap()), + deployment: prior_request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size_bytes: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.size_bytes, + }, + }; + { + let mut state = fixture.state.lock().unwrap(); + state.permission = false; + state.prior = Some(prior.clone()); + } + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--bin-path", + wasm.to_str().unwrap(), + "--publication-state-dir", + temporary.path().join("state").to_str().unwrap(), + "--publication-wait", + "0", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.preflights, 1); + assert_eq!(state.reservations.len(), 0); + let request: PublishRequest = serde_json::from_slice(&state.submits[0]).unwrap(); + assert_eq!(request.manifest.current().envelope.expected_revision, prior.revision); + assert!(matches!( + request.manifest.current().envelope.container_action, + ContainerAction::Keep + )); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Set(_) + )); + } + fixture.close().await; + } + #[tokio::test] + async fn remove_module_requires_authorized_preflight_before_any_upload_or_submission() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let request = fixture.record(false, false).request().unwrap(); + { + let mut state = fixture.state.lock().unwrap(); + state.deny_preflight = true; + state.prior = Some(DeploymentStatus { + database_identity: database(), + revision: Some(request.manifest.current().deployment.revision().unwrap()), + deployment: request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size_bytes: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.size_bytes, + }, + }); + } + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--remove-module", + "--publication-state-dir", + temporary.path().to_str().unwrap(), + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .is_err()); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.preflights, 1); + assert_eq!(state.begin_count, 0); + assert!(state.submits.is_empty()); + } + fixture.close().await; + } + #[tokio::test] + async fn ordinary_module_target_returns_to_legacy_only_when_no_managed_revision_exists() { + let fixture = Fixture::new().await; + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from(["publish", "fixture-name", "--server", fixture.endpoint.as_str()]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(!try_execute_with_client( + &target, + None, + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + fixture.close().await; + } + #[tokio::test] + async fn resume_entrypoint_ignores_changed_project_and_reuses_original_bytes() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_before_commit = true; + let temporary = tempfile::tempdir().unwrap(); + let record = fixture.record(false, false); + let exact = record.request_json.clone(); + let mut journal = Journal::create( + temporary.path(), + record, + None, + Some(deployment::SYSTEM_EMPTY_MODULE_V1_BYTES), + ) + .unwrap(); + assert!(publish::run( + &fixture.client(), + &mut journal, + None, + Duration::ZERO, + CancellationToken::new() + ) + .await + .is_err()); + let resume = journal.directory().to_owned(); + drop(journal); + let cli_config = temporary.path().join("isolated-cli.toml"); + std::fs::write(&cli_config, "spacetimedb_token = 'isolated-fixture-credential'\n").unwrap(); + let config = Config::load(spacetimedb_paths::cli::CliTomlPath::from_path_unchecked(cli_config)).unwrap(); + let changed_project = crate::spacetime_config::LoadedConfig { + config: serde_json::from_value(json!({"database":"different-target", "module_path":"missing-build-source", "server":"must-never-resolve-this-alias"})).unwrap(), + config_dir: temporary.path().into(), loaded_files: vec![], has_dev_file: false, + }; + let args = super::super::cli() + .try_get_matches_from([ + "publish", + "--resume-publication", + resume.to_str().unwrap(), + "--server", + fixture.endpoint.as_str(), + "--publication-wait", + "0", + ]) + .unwrap(); + super::super::exec_with_options(config, &args, true, Some(&changed_project)) + .await + .unwrap(); + assert_eq!( + fixture.state.lock().unwrap().submits, + [exact.as_bytes(), exact.as_bytes()] + ); + fixture.close().await; + } + #[test] + fn nested_targets_keep_only_their_own_container_and_resume_rejects_new_inputs() { + let config: SpacetimeConfig = serde_json::from_value(json!({"database":"parent", "container":{"image":{"oci_ref":"oci:layout"},"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}}, "children":[{"database":"child"}]})).unwrap(); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command.clone().try_get_matches_from(["publish"]).unwrap(); + let targets = super::super::get_filtered_publish_configs(&config, &command, &schema, &args).unwrap(); + assert_eq!(targets.len(), 2); + assert!(targets[0].container().is_some()); + assert!(targets[1].container().is_none()); + for flags in [ + ["--resume-publication", "operation", "--remove-module"], + ["--resume-publication", "operation", "--managed"], + ] { + assert!(command + .clone() + .try_get_matches_from(std::iter::once("publish").chain(flags)) + .is_err()); + } + } +} diff --git a/crates/cli/src/subcommands/sidecar.rs b/crates/cli/src/subcommands/sidecar.rs deleted file mode 100644 index 68a422275a7..00000000000 --- a/crates/cli/src/subcommands/sidecar.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! `spacetime sidecar` (PROTOTYPE) -//! -//! Manage sidecars: long-running programs (e.g. agents) hosted beside a -//! database. This command is a thin client: it records the desired state by -//! calling the control database's `create_sidecar` / `delete_sidecar` reducers. -//! The actual container is launched and reconciled by the SpacetimeDB Cloud -//! node hosting the database (see `private/crates/cloud`); SpacetimeDB -//! Standalone has no control plane and does not support sidecars. -//! -//! See `proposals/00XX-agent-hosting.md`. - -use crate::api::{ClientApi, Connection}; -use crate::common_args; -use crate::config::Config; -use crate::util::{database_identity, get_auth_header, AuthHeader, UNSTABLE_WARNING}; -use anyhow::{bail, Context}; -use clap::{Arg, ArgAction, ArgMatches}; -use spacetimedb_lib::Identity; - -pub fn cli() -> clap::Command { - clap::Command::new("sidecar") - .about(format!( - "Manage sidecars (long-running programs hosted beside a database). {UNSTABLE_WARNING}" - )) - .args_conflicts_with_subcommands(true) - .subcommand_required(true) - .subcommands([ - clap::Command::new("run") - .about("Declare a sidecar for a database; the hosting node launches it") - .arg(Arg::new("database").required(true).help("The database name or identity")) - .arg(Arg::new("image").long("image").required(true).help("The OCI image to run")) - .arg( - Arg::new("name") - .long("name") - .default_value("default") - .help("A name for this sidecar, allowing several per database"), - ) - .arg( - Arg::new("env") - .long("env") - .short('e') - .value_name("KEY=VALUE") - .action(ArgAction::Append) - .help("Extra environment variables to inject (repeatable)"), - ) - .arg( - Arg::new("command") - .num_args(0..) - .last(true) - .help("Optional command to run in the container, after `--`"), - ) - .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) - .arg(common_args::anonymous()) - .arg(common_args::yes()), - clap::Command::new("ls") - .about("List declared sidecars") - .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")), - clap::Command::new("stop") - .about("Remove a sidecar from a database; the hosting node stops it") - .arg(Arg::new("database").required(true).help("The database name or identity")) - .arg( - Arg::new("name") - .long("name") - .default_value("default") - .help("The sidecar name to remove"), - ) - .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) - .arg(common_args::anonymous()) - .arg(common_args::yes()), - ]) - .after_help("Run `spacetime help sidecar` for more detailed information.\n") -} - -pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { - eprintln!("{UNSTABLE_WARNING}\n"); - let (cmd, sub) = args.subcommand().expect("subcommand required"); - match cmd { - "run" => exec_run(config, sub).await, - "ls" => exec_ls(config, sub).await, - "stop" => exec_stop(config, sub).await, - unknown => bail!("Invalid subcommand: {unknown}"), - } -} - -/// The control database is addressed by the all-zero identity. -fn control_api(config: &Config, server: Option<&str>, auth: AuthHeader) -> anyhow::Result { - Ok(ClientApi::new(Connection { - host: config.get_host_url(server)?, - database_identity: Identity::ZERO, - database: "control".to_string(), - auth_header: auth, - })) -} - -async fn exec_run(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { - let server = args.get_one::("server").map(|s| s.as_str()); - let force = args.get_flag("force"); - let anon = args.get_flag("anon_identity"); - let database = args.get_one::("database").unwrap(); - let image = args.get_one::("image").unwrap(); - let name = args.get_one::("name").unwrap(); - let mut env: Vec = args - .get_many::("env") - .map(|v| v.cloned().collect()) - .unwrap_or_default(); - let command: Vec = args - .get_many::("command") - .map(|v| v.cloned().collect()) - .unwrap_or_default(); - - let host = config.get_host_url(server)?; - let db_identity = database_identity(&config, database, server).await?; - let auth = get_auth_header(&mut config, anon, server, !force).await?; - let token = auth - .token() - .context("No auth token available; run `spacetime login` first (the sidecar connects back with it)")? - .to_string(); - - // Connection details for the sidecar to reach this database, made reachable - // from inside a container (Docker Desktop maps host.docker.internal). - let db_hex = db_identity.to_hex().to_string(); - let mut connect_env = vec![ - format!("SPACETIMEDB_URI={}", to_container_host(&http_to_ws(&host))), - format!("SPACETIMEDB_HTTP_URI={}", to_container_host(&host)), - format!("SPACETIMEDB_TOKEN={token}"), - format!("SPACETIMEDB_DB={db_hex}"), - ]; - connect_env.append(&mut env); - - let api = control_api(&config, server, auth)?; - let arg_json = serde_json::json!([db_hex, name, image, command, connect_env]).to_string(); - let res = api.call("create_sidecar", arg_json).await?; - let status = res.status(); - let body = res.text().await.unwrap_or_default(); - if !status.is_success() { - bail!("failed to declare sidecar ({status}): {body}"); - } - - println!("Declared sidecar `{name}` for database `{database}`."); - println!("The node hosting the database will launch it shortly. Check with:"); - println!(" spacetime sidecar ls"); - Ok(()) -} - -async fn exec_stop(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { - let server = args.get_one::("server").map(|s| s.as_str()); - let force = args.get_flag("force"); - let anon = args.get_flag("anon_identity"); - let database = args.get_one::("database").unwrap(); - let name = args.get_one::("name").unwrap(); - - let db_identity = database_identity(&config, database, server).await?; - let auth = get_auth_header(&mut config, anon, server, !force).await?; - let db_hex = db_identity.to_hex().to_string(); - - let api = control_api(&config, server, auth)?; - let arg_json = serde_json::json!([db_hex, name]).to_string(); - let res = api.call("delete_sidecar", arg_json).await?; - let status = res.status(); - let body = res.text().await.unwrap_or_default(); - if !status.is_success() { - bail!("failed to remove sidecar ({status}): {body}"); - } - - println!("Removed sidecar `{name}` from database `{database}`. The hosting node will stop it shortly."); - Ok(()) -} - -async fn exec_ls(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { - let server = args.get_one::("server").map(|s| s.as_str()); - // Reading the control db requires a token; reuse the logged-in identity. - let auth = get_auth_header(&mut config, false, server, true).await?; - let api = control_api(&config, server, auth)?; - let res = api - .sql() - .body("SELECT id, database_id, name, image, desired_state FROM sidecar") - .send() - .await?; - let status = res.status(); - let body = res.text().await.unwrap_or_default(); - if !status.is_success() { - bail!("failed to list sidecars ({status}): {body}"); - } - println!("{body}"); - Ok(()) -} - -fn http_to_ws(url: &str) -> String { - if let Some(rest) = url.strip_prefix("https://") { - format!("wss://{rest}") - } else if let Some(rest) = url.strip_prefix("http://") { - format!("ws://{rest}") - } else { - url.to_string() - } -} - -fn to_container_host(url: &str) -> String { - url.replace("localhost", "host.docker.internal") - .replace("127.0.0.1", "host.docker.internal") -} diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index 6586d8edd5f..ebb49ca5408 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -7,12 +7,12 @@ use reqwest::Url; use serde_json::Value; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2, v3 as ws_v3}; use spacetimedb_data_structures::map::HashMap; -use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::{DeserializeWrapper, SeedWrapper}; use spacetimedb_lib::de::DeserializeSeed as BsatnDeserializeSeed; use spacetimedb_lib::sats::WithTypespace; use spacetimedb_lib::ser::serde::SerializeWrapper; use spacetimedb_lib::{bsatn, AlgebraicType}; +use spacetimedb_schema::def::ModuleDef; use std::collections::VecDeque; use std::io; use std::time::Duration; @@ -115,7 +115,7 @@ impl SubscribeConnection { } /// Wait for the initial subscription result and optionally print it. - async fn await_initial_update(&mut self, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> { + async fn await_initial_update(&mut self, module_def: Option<&ModuleDef>) -> Result<(), Error> { match self { Self::V3 { ws, pending } => await_initial_update_v3(ws, pending, module_def).await, Self::V1 { ws } => await_initial_update_v1(ws, module_def).await, @@ -123,11 +123,7 @@ impl SubscribeConnection { } /// Print transaction updates until the requested count is reached. - async fn consume_transaction_updates( - &mut self, - num: Option, - module_def: &RawModuleDefV9, - ) -> Result<(), Error> { + async fn consume_transaction_updates(&mut self, num: Option, module_def: &ModuleDef) -> Result<(), Error> { match self { Self::V3 { ws, pending } => consume_transaction_updates_v3(ws, pending, num, module_def).await, Self::V1 { ws } => consume_transaction_updates_v1(ws, num, module_def).await, @@ -179,7 +175,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error database: resolved.database.clone(), }; let api = ClientApi::new(conn); - let module_def = api.module_def().await?; + let module_def: ModuleDef = api.module_def().await?.try_into()?; let mut conn = connect_with_fallback(&api, confirmed).await?; let task = async { @@ -311,14 +307,14 @@ enum Error { #[error("error sending subscription queries")] Subscribe { #[source] - source: WsError, + source: Box, }, #[error("protocol error: {details}")] Protocol { details: &'static str }, #[error("websocket error: {source}")] Websocket { #[source] - source: WsError, + source: Box, }, #[error("encountered failed transaction: {reason}")] TransactionFailure { reason: Box }, @@ -347,12 +343,7 @@ enum Error { impl Error { fn is_server_closed_connection(&self) -> bool { - matches!( - self, - Self::Websocket { - source: WsError::ConnectionClosed - } - ) + matches!(self, Self::Websocket { source } if matches!(source.as_ref(), WsError::ConnectionClosed)) } } @@ -368,7 +359,9 @@ where }, ))) .unwrap(); - ws.send(msg.into()).await.map_err(|source| Error::Subscribe { source }) + ws.send(msg.into()) + .await + .map_err(|source| Error::Subscribe { source: source.into() }) } /// Send a v3 BSATN subscribe message. @@ -384,7 +377,7 @@ where let msg = bsatn::to_vec(&msg).map_err(|source| Error::BsatnEncode { source })?; ws.send(WsMessage::Binary(msg.into())) .await - .map_err(|source| Error::Subscribe { source }) + .map_err(|source| Error::Subscribe { source: source.into() }) } /// Parse a v1 text websocket message as JSON. @@ -398,13 +391,17 @@ fn parse_msg_json(msg: &WsMessage) -> Option(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> +async fn await_initial_update_v1(ws: &mut S, module_def: Option<&ModuleDef>) -> Result<(), Error> where S: TryStream + Unpin, { const RECV_TX_UPDATE: &str = "protocol error: received transaction update before initial subscription update"; - while let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? { + while let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + { let Some(msg) = parse_msg_json(&msg) else { continue }; match msg { ws_v1::ServerMessage::InitialSubscription(sub) => { @@ -442,7 +439,7 @@ where async fn await_initial_update_v3( ws: &mut S, pending: &mut VecDeque, - module_def: Option<&RawModuleDefV9>, + module_def: Option<&ModuleDef>, ) -> Result<(), Error> where S: TryStream + Unpin, @@ -475,11 +472,7 @@ where /// Print `num` v1 [`ws_v1::ServerMessage::TransactionUpdate`] messages as JSON. /// If `num` is `None`, keep going indefinitely. -async fn consume_transaction_updates_v1( - ws: &mut S, - num: Option, - module_def: &RawModuleDefV9, -) -> Result<(), Error> +async fn consume_transaction_updates_v1(ws: &mut S, num: Option, module_def: &ModuleDef) -> Result<(), Error> where S: TryStream + Unpin, { @@ -489,10 +482,14 @@ where if num.is_some_and(|n| num_received >= n) { return Ok(()); } - let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + else { eprintln!("disconnected by server"); return Err(Error::Websocket { - source: WsError::ConnectionClosed, + source: Box::new(WsError::ConnectionClosed), }); }; @@ -526,7 +523,7 @@ async fn consume_transaction_updates_v3( ws: &mut S, pending: &mut VecDeque, num: Option, - module_def: &RawModuleDefV9, + module_def: &ModuleDef, ) -> Result<(), Error> where S: TryStream + Unpin, @@ -540,7 +537,7 @@ where let Some(msg) = next_server_message(ws, pending).await? else { eprintln!("disconnected by server"); return Err(Error::Websocket { - source: WsError::ConnectionClosed, + source: Box::new(WsError::ConnectionClosed), }); }; @@ -580,7 +577,11 @@ where return Ok(Some(msg)); } - let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + else { return Ok(None); }; let WsMessage::Binary(msg) = msg else { continue }; @@ -619,25 +620,19 @@ fn decode_server_payload(msg: Bytes, pending: &mut VecDeque, - schema: &RawModuleDefV9, -) -> Result { +fn format_output_json_v1(msg: &ws_v1::DatabaseUpdate, schema: &ModuleDef) -> Result { let formatted = reformat_update_v1(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } /// Format initial v3 subscription rows using the CLI's existing JSON output shape. -fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &RawModuleDefV9) -> Result { +fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &ModuleDef) -> Result { let formatted = reformat_query_rows(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } /// Format a v3 transaction update using the CLI's existing JSON output shape. -fn format_output_json_transaction_update( - msg: &ws_v2::TransactionUpdate, - schema: &RawModuleDefV9, -) -> Result { +fn format_output_json_transaction_update(msg: &ws_v2::TransactionUpdate, schema: &ModuleDef) -> Result { let formatted = reformat_transaction_update(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } @@ -651,12 +646,12 @@ fn format_output_json_from_tables(formatted: &HashMap<&str, SubscriptionTable>) /// Convert a v1 JSON-format database update to the normalized table output map. fn reformat_update_v1<'a>( msg: &'a ws_v1::DatabaseUpdate, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { msg.tables .iter() .map(|upd| { - let table_ty = schema.typespace.resolve( + let table_ty = schema.typespace().resolve( schema .type_ref_for_table_like(&upd.table_name) .context("table not found in schema")?, @@ -690,12 +685,12 @@ fn reformat_update_v1<'a>( /// Convert v3 initial subscription rows to the normalized table output map. fn reformat_query_rows<'a>( msg: &'a ws_v2::QueryRows, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { let mut formatted = HashMap::default(); for table in &msg.tables { - let table_ty = schema.typespace.resolve( + let table_ty = schema.typespace().resolve( schema .type_ref_for_table_like(&table.table) .context("table not found in schema")?, @@ -713,13 +708,13 @@ fn reformat_query_rows<'a>( /// Convert a v3 transaction update to the normalized table output map. fn reformat_transaction_update<'a>( msg: &'a ws_v2::TransactionUpdate, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { let mut formatted = HashMap::default(); for query_set in &msg.query_sets { for table in &query_set.tables { - let table_ty = schema.typespace.resolve( + let table_ty = schema.typespace().resolve( schema .type_ref_for_table_like(&table.table_name) .context("table not found in schema")?, diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 5fd05be8320..ea9b15c12da 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -197,12 +197,6 @@ impl AuthHeader { val }) } - - /// The raw bearer token, if any. Used by `spacetime sidecar` to pass the - /// caller's credentials into a sidecar container. - pub fn token(&self) -> Option<&str> { - self.token.as_deref() - } } pub const VALID_PROTOCOLS: [&str; 2] = ["http", "https"]; diff --git a/crates/cli/tests/container_build_acceptance/README.md b/crates/cli/tests/container_build_acceptance/README.md new file mode 100644 index 00000000000..fd8954c0f07 --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/README.md @@ -0,0 +1,77 @@ +# Real local container builder acceptance + +This opt-in fixture exercises the CLI with actual BuildKit and explicitly +selected Railpack. It does not connect to a SpacetimeDB server. The resulting +verified OCI layouts can be consumed by the separate managed publication +acceptance test. + +The checked-in tool lock currently supports macOS on arm64. It records official +release download URLs, SHA-256 checksums, and image digests. The fixture downloads +the two tools into its own workspace and extracts only the expected regular +binary. Nothing is installed globally. Release checksum provenance is recorded +in `tool-lock.json`; image pins were verified against the primary registries. + +Build the CLI and deadline harness from the public workspace, without selecting +or connecting to a server: + +```sh +cargo build --locked --offline -p spacetimedb-cli --bin spacetimedb-cli +cargo test --locked --offline -p spacetimedb-cli --test real_container_builder --no-run +``` + +Use the absolute test executable path printed by the second command. Invoke the +script with absolute paths and a new workspace directory: + +```sh +python3 crates/cli/tests/container_build_acceptance/acceptance.py \ + --docker-socket /absolute/path/to/verified/docker-desktop.sock \ + --cli /absolute/path/to/public/target/debug/spacetimedb-cli \ + --deadline-test-binary /absolute/path/to/public/target/debug/deps/real_container_builder-HASH \ + --workspace /private/tmp/new-owned-builder-workspace +``` + +The socket must already be verified as an owned disposable Docker Desktop +endpoint. The script checks its reported host identity again. Every Docker +command specifies that socket and an empty fixture configuration; saved Docker +contexts and registry credentials are not used. The fixture pulls pinned public +images, starts a uniquely named BuildKit container with a dedicated cache volume, +and binds its port to numeric loopback. A private Unix socket forwards only to +that port. The container is limited to two CPUs, 2 GiB of memory and 256 processes. +Container logs rotate at 4 MiB with at most two files. BuildKit requires +privileged execution in this local fixture; the Docker socket +is not mounted into it. This is a trusted-tool test, not a sandbox for arbitrary +build programs. + +The checks cover: + +- A Dockerfile using a build-secret mount and an explicitly selected Railpack + shell-script build. Neither successful nor failed build logs may reveal the + secret, and retained image objects must not contain it. +- Exact digest, size, platform and executable manifest/config/layer closure of + both prepared outputs. A modified retained object is rejected on import. +- Failed builds and failed Railpack detection, with no automatic builder fallback. +- A destination created while a build is running, which must remain untouched. +- SIGINT cancellation of real `buildctl`, followed by positive PID absence and + workspace release. +- The same cleanup on a two-second deadline through the production local process + runner. Only the test runner shortens the normal build deadline. + +Commands have finite deadlines. Teardown stops the proxy, validates the exact +owned container's name and label, waits for its successful synchronous removal, +then removes its cache volume. If the run reply is lost, cleanup looks up only +the original generated name and requires the same ownership label. An ambiguous +daemon answer fails cleanup; an arbitrary inspection error is not treated as +proof of absence. A teardown failure fails the fixture. Downloaded binaries, +diagnostic inputs and the two OCI layouts remain in the private workspace; public image cache entries +are not pruned. `acceptance.json` records the completed checks and the output +paths without credentials or build-secret values. + +The retained layouts establish build and artifact preparation behavior. They do +not establish container execution, readiness, or isolation in the production +runtime. Those are separate Linux supervisor and Kata acceptance boundaries. + +The cleanup failure tests need no Docker daemon or network: + +```sh +python3 -B -m unittest discover -s crates/cli/tests/container_build_acceptance -p test_fixture.py -v +``` diff --git a/crates/cli/tests/container_build_acceptance/acceptance.py b/crates/cli/tests/container_build_acceptance/acceptance.py new file mode 100644 index 00000000000..ebbc575dd6c --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/acceptance.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Explicitly invoked real local-builder acceptance, never part of cargo test. + +Only an explicitly verified Docker Desktop Unix socket is accepted. The CLI +uses a fixture-owned BuildKit Unix proxy and never opens Spacetime credentials. +Downloaded tools and retained OCI output live under a new private workspace. +""" +import argparse +import concurrent.futures +import gzip +import hashlib +import json +import os +from pathlib import Path +import platform +import selectors +import shutil +import shlex +import signal +import socket +import subprocess +import tarfile +import threading +import time +import urllib.request +import uuid + + +LOCK = json.loads(Path(__file__).with_name("tool-lock.json").read_text()) +LIMIT = 32 * 1024 * 1024 + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def environment(root): + return { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(root / "home"), + "TMPDIR": str(root / "tmp"), + } + + +def run(args, env, timeout=120, check=True): + # No shell expansion, inherited proxy/daemon/server variables or credentials. + child = subprocess.Popen(args, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + out, err = child.communicate(timeout=timeout) + except BaseException: + child.kill() + child.communicate(timeout=10) + raise + require(len(out) <= LIMIT and len(err) <= LIMIT, "fixture command output exceeded bound") + if check: + require(child.returncode == 0, + f"fixture command failed: {Path(args[0]).name} (status {child.returncode})") + return child.returncode, out, err + + +def tools(root): + require(platform.system() == "Darwin" and platform.machine() == "arm64", + "the checked-in tool lock supports Darwin arm64") + directory = root / "tools" + directory.mkdir() + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + binaries = {} + for name, pin in LOCK["tools"].items(): + print(f"Downloading and verifying pinned {name} {pin['version']}", flush=True) + with opener.open(pin["url"], timeout=60) as response: + data = response.read(LIMIT + 1) + require(len(data) <= LIMIT, "tool archive exceeded bound") + require(hashlib.sha256(data).hexdigest() == pin["sha256"], "tool archive checksum mismatch") + archive = directory / (name + ".tar.gz") + archive.write_bytes(data) + with tarfile.open(archive) as contents: + matches = [item for item in contents if item.name.removeprefix("./") == pin["member"]] + require(len(matches) == 1 and matches[0].isreg(), "unexpected tool archive member") + require(matches[0].size <= 128 * 1024 * 1024, "tool binary exceeded bound") + binary = directory / name + with contents.extractfile(matches[0]) as source: + binary.write_bytes(source.read()) + binary.chmod(0o700) + binaries[name] = binary + return binaries + + +class UnixProxy: + """Forward only to this fixture's Docker-published numeric-loopback port.""" + def __init__(self, path, port): + self.path, self.port = path, port + self.stop = threading.Event() + self.slots = threading.BoundedSemaphore(32) + self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=32) + self.listener = socket.socket(socket.AF_UNIX) + self.listener.bind(str(path)) + path.chmod(0o600) + self.listener.listen(32) + self.listener.settimeout(0.2) + self.thread = threading.Thread(target=self.accept) + self.thread.start() + + def accept(self): + while not self.stop.is_set(): + try: + client, _ = self.listener.accept() + except TimeoutError: + continue + except OSError: + break + if self.slots.acquire(blocking=False): + self.pool.submit(self.forward, client) + else: + client.close() + + def forward(self, client): + try: + with client, socket.create_connection(("127.0.0.1", self.port), timeout=5) as upstream: + # Bounded socket writes and periodic stop checks; no filesystem + # path or request header can select another destination. + client.settimeout(1) + upstream.settimeout(1) + with selectors.DefaultSelector() as select: + select.register(client, selectors.EVENT_READ, upstream) + select.register(upstream, selectors.EVENT_READ, client) + while not self.stop.is_set(): + for key, _ in select.select(0.2): + data = key.fileobj.recv(64 * 1024) + if not data: + return + key.data.sendall(data) + except (OSError, TimeoutError): + pass + finally: + self.slots.release() + + def close(self): + self.stop.set() + self.listener.close() + self.thread.join(timeout=3) + require(not self.thread.is_alive(), "BuildKit proxy did not stop accepting") + self.pool.shutdown(wait=True) + self.path.unlink() + + +class Builder: + def __init__(self, root, docker_socket, binaries): + self.root, self.binaries = root, binaries + self.env = environment(root) + docker = shutil.which("docker") + require(docker is not None, "Docker executable is required") + self.docker = [docker, "--host", "unix://" + str(docker_socket), + "--config", str(root / "docker-config")] + self.name = "stdb-cli-build-" + uuid.uuid4().hex + self.volume = self.name + "-cache" + self.container = None + self.run_attempted = False + self.proxy = None + self.volume_created = False + + def command(self, *args, timeout=120, check=True): + return run(self.docker + list(args), self.env, timeout, check) + + def start(self): + _, info, _ = self.command("info", "--format", "{{.Name}}|{{.OperatingSystem}}|{{.OSType}}") + require(info.decode().strip() == "docker-desktop|Docker Desktop|linux", + "explicit socket is not the expected local Docker Desktop fixture host") + print("Verified explicit Docker Desktop socket; starting isolated BuildKit", flush=True) + self.command("pull", "--platform", "linux/arm64", LOCK["images"]["buildkit"], timeout=300) + _, raw, _ = self.command("image", "inspect", LOCK["images"]["buildkit"]) + inspected = json.loads(raw)[0] + require(LOCK["images"]["buildkit"] in inspected["RepoDigests"], "pulled BuildKit digest mismatch") + require(inspected["Architecture"] == "arm64", "BuildKit architecture mismatch") + self.command("volume", "create", "--label", "spacetimedb.fixture=" + self.name, self.volume) + self.volume_created = True + # A timed-out CLI may have created the container before losing its + # response. Retain the exact name/label cleanup key before dispatch. + self.run_attempted = True + _, identifier, _ = self.command( + "run", "--detach", "--pull", "never", "--platform", "linux/arm64", + "--name", self.name, "--label", "spacetimedb.fixture=" + self.name, + "--privileged", "--cpus", "2", "--memory", "2g", "--memory-swap", "2g", + "--log-driver", "json-file", "--log-opt", "max-size=4m", "--log-opt", "max-file=2", + "--pids-limit", "256", "--mount", f"type=volume,src={self.volume},dst=/var/lib/buildkit", + "--publish", "127.0.0.1::1234", LOCK["images"]["buildkit"], + "--addr", "tcp://0.0.0.0:1234", "--oci-worker-snapshotter=native") + self.container = identifier.decode().strip() + _, raw, _ = self.command("inspect", self.container) + owned = json.loads(raw)[0] + require(owned["Config"]["Labels"]["spacetimedb.fixture"] == self.name, "container ownership mismatch") + ports = owned["NetworkSettings"]["Ports"]["1234/tcp"] + require(len(ports) == 1 and ports[0]["HostIp"] == "127.0.0.1", "BuildKit is not loopback bound") + self.proxy = UnixProxy(self.root / "buildkit.sock", int(ports[0]["HostPort"])) + deadline = time.monotonic() + 30 + while True: + status, _, _ = run([str(self.binaries["buildctl"]), "--addr", self.endpoint, + "debug", "workers"], self.env, timeout=5, check=False) + if status == 0: + break + require(time.monotonic() < deadline, "BuildKit fixture did not become ready") + time.sleep(0.2) + + @property + def endpoint(self): + return "unix://" + str(self.root / "buildkit.sock") + + def close(self): + errors = [] + if self.proxy: + try: + self.proxy.close() + except Exception as error: + errors.append(str(error)) + if self.run_attempted: + try: + # Inspect only the exact returned ID or generated name. A + # failed/ambiguous inspect is not proof of absence. The label + # must match before deletion, including after a lost run reply. + _, raw, _ = self.command("container", "inspect", self.container or self.name) + objects = json.loads(raw) + require(len(objects) == 1, "ambiguous owned container lookup") + owned = objects[0] + require(owned["Name"] == "/" + self.name and + owned["Config"]["Labels"]["spacetimedb.fixture"] == self.name, + "container cleanup ownership mismatch") + if self.container: + require(owned["Id"] == self.container, "container cleanup ID mismatch") + # A successful synchronous removal is positive completion. + self.command("rm", "--force", "--volumes", owned["Id"]) + except Exception as error: + errors.append(str(error)) + if self.volume_created: + try: + self.command("volume", "rm", self.volume) + except Exception as error: + errors.append(str(error)) + require(not errors, "positive builder teardown failed: " + "; ".join(errors)) + + +def config(project, image): + project.mkdir() + document = { + "database": "local-builder-fixture", + "container": { + "image": image, + "env_keys": ["RUNTIME_ONLY"], + "resources": {"cpu_millicores": 1000, "memory_bytes": 536870912, + "scratch_bytes": 1073741824, "pids_max": 128}, + }, + } + (project / "spacetime.json").write_text(json.dumps(document)) + + +def verify_layout(layout, secret): + metadata = json.loads((layout / "prepared.json").read_text()) + require(secret not in (layout / "prepared.json").read_bytes(), "build secret entered prepared metadata") + objects = {} + for item in metadata["objects"]: + descriptor = item["descriptor"] + path = Path(item["path"]) + require(not path.is_absolute() and ".." not in path.parts, "artifact path escaped layout") + data = (layout / path).read_bytes() + require(len(data) == descriptor["size"], "artifact size mismatch") + require("sha256:" + hashlib.sha256(data).hexdigest() == descriptor["digest"], "artifact hash mismatch") + require(secret not in data, "build secret entered retained image object") + if item["kind"] == "layer" and data[:2] == b"\x1f\x8b": + require(secret not in gzip.decompress(data), "build secret entered retained image layer") + objects[descriptor["digest"]] = (item["kind"], data) + manifest = json.loads(objects[metadata["manifest"]["digest"]][1]) + require(set(objects) == {metadata["manifest"]["digest"], manifest["config"]["digest"], + *(entry["digest"] for entry in manifest["layers"])}, + "prepared closure is not the exact executable manifest/config/layers") + image = json.loads(objects[manifest["config"]["digest"]][1]) + require(image["os"] == "linux" and image["architecture"] == "arm64", "image platform mismatch") + return metadata, image + + +def cases(root, cli, binaries, builder, deadline_test): + secret = ("build-secret-" + uuid.uuid4().hex).encode() + secret_file = root / "build-secret" + secret_file.write_bytes(secret) + secret_file.chmod(0o600) + base = [str(cli), "--root-dir", str(root / "cli-root"), "--config-path", str(root / "unused-config"), + "container", "build", "--platform", "linux/arm64", "--buildctl", str(binaries["buildctl"]), + "--railpack", str(binaries["railpack"]), "--buildkit-host", builder.endpoint] + + def build(project, output, success=True, secrets=False): + args = base + ["--project-path", str(project), "--out-dir", str(output)] + if secrets: + args += ["--build-secret", "BUILD_SENTINEL=" + str(secret_file)] + status, out, err = run(args, environment(root), timeout=1200, check=False) + require(secret not in out + err, "CLI leaked builder secret diagnostics") + if (status == 0) != success: + # Error output has been checked for the generated sentinel and the + # fixture never supplies ordinary or registry credentials. + raise RuntimeError("CLI build outcome mismatch: " + (out + err).decode(errors="replace")[:2048]) + return out + err + + project = root / "dockerfile-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\n" + "RUN --mount=type=secret,id=BUILD_SENTINEL test -s /run/secrets/BUILD_SENTINEL && cat /run/secrets/BUILD_SENTINEL >&2\n" + "WORKDIR /app\nUSER 1001:1001\nENV IMAGE_DEFAULT=retained\n" + 'ENTRYPOINT ["/bin/sh"]\nCMD ["-c", "echo dockerfile-ready"]\n') + output = root / "dockerfile-output" + print("Actual Dockerfile build with explicit secret mount", flush=True) + build(project, output, secrets=True) + metadata, image = verify_layout(output, secret) + require(metadata["container"]["argv"] == ["/bin/sh", "-c", "echo dockerfile-ready"], "argv was not normalized") + require(metadata["container"]["user"] == "1001:1001", "image user was lost") + require(metadata["container"]["working_directory"] == "/app", "image working directory was lost") + require("IMAGE_DEFAULT=retained" in image["config"]["Env"], "image environment defaults were lost") + + project = root / "railpack-project" + config(project, {"build": {"builder": "railpack", "context": "."}}) + (project / "start.sh").write_text("#!/bin/sh\nprintf 'railpack-ready\\n'\n") + (project / "start.sh").chmod(0o700) + output = root / "railpack-output" + print("Actual explicitly selected Railpack build", flush=True) + build(project, output) + verify_layout(output, secret) + + print("Failed secret-using build must not expose logs or publish output", flush=True) + project = root / "failed-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\n" + "RUN --mount=type=secret,id=BUILD_SENTINEL cat /run/secrets/BUILD_SENTINEL >&2; exit 23\n") + output = root / "failed-output" + build(project, output, success=False, secrets=True) + require(not output.exists(), "failed build published output") + + print("Explicit unsupported Railpack detection cannot fall back to Dockerfile", flush=True) + project = root / "unsupported-project" + config(project, {"build": {"builder": "railpack", "context": "."}}) + (project / "Dockerfile").write_text('FROM scratch\nCMD ["would-be-wrong-builder"]\n') + output = root / "unsupported-output" + build(project, output, success=False) + require(not output.exists(), "unsupported Railpack silently used another builder") + + print("Tampering with a retained OCI object must fail before output publication", flush=True) + tampered = root / "tampered-layout" + shutil.copytree(root / "dockerfile-output", tampered) + metadata = json.loads((tampered / "prepared.json").read_text()) + target = next(item for item in metadata["objects"] if item["kind"] == "config") + with (tampered / target["path"]).open("ab") as changed: + changed.write(b" ") + project = root / "tampered-project" + config(project, {"oci_ref": "oci:" + str(tampered)}) + output = root / "tampered-output" + build(project, output, success=False) + require(not output.exists(), "tampered OCI image was accepted") + + print("A path created during the actual build must never be replaced", flush=True) + project = root / "replacement-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\nRUN sleep 3\nCMD [\"/bin/true\"]\n") + output = root / "replacement-output" + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task = pool.submit(build, project, output, False) + deadline = time.monotonic() + 30 + while not list(root.glob(".spacetime-image-*")): + require(not task.done(), "build finished before concurrent replacement could be tested") + require(time.monotonic() < deadline, "build never created its private workspace") + time.sleep(0.01) + output.mkdir() + (output / "sentinel").write_text("must survive") + task.result(timeout=60) + require(list(output.iterdir()) == [output / "sentinel"], "existing output was replaced") + require((output / "sentinel").read_text() == "must survive", "existing output content changed") + + print("SIGINT must cancel the real buildctl and positively reap it", flush=True) + project = root / "cancel-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\nRUN sleep 120\nCMD [\"/bin/true\"]\n") + output = root / "cancel-output" + pid_file = root / "actual-buildctl.pid" + wrapper = root / "record-buildctl" + wrapper.write_text("#!/bin/sh\nprintf '%s\\n' \"$$\" > " + shlex.quote(str(pid_file)) + + "\nexec " + shlex.quote(str(binaries["buildctl"])) + ' "$@"\n') + wrapper.chmod(0o700) + args = list(base) + args[args.index("--buildctl") + 1] = str(wrapper) + args += ["--project-path", str(project), "--out-dir", str(output)] + child = subprocess.Popen(args, env=environment(root), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 30 + while not pid_file.exists(): + require(child.poll() is None, "CLI exited before invoking real buildctl") + require(time.monotonic() < deadline, "real buildctl did not start") + time.sleep(0.01) + tool_pid = int(pid_file.read_text()) + child.send_signal(signal.SIGINT) + out, err = child.communicate(timeout=15) + require(child.returncode != 0 and b"cancelled" in out + err, "CLI did not report cancellation") + require(secret not in out + err, "cancelled CLI leaked a secret") + try: + os.kill(tool_pid, 0) + except ProcessLookupError: + pass + else: + raise RuntimeError("real buildctl PID still exists after acknowledged CLI cancellation") + require(not output.exists(), "cancelled build published output") + finally: + if child.poll() is None: + child.kill() + child.communicate(timeout=10) + require(not list(root.glob(".spacetime-image-*")), "completed CLI left builder workspaces behind") + + print("A short harness deadline must reap real buildctl before releasing its workspace", flush=True) + pid_file.unlink() + deadline_env = environment(root) + deadline_env.update({ + "STDB_BUILDER_CONTEXT": str(project), + "STDB_BUILDER_BUILDCTL": str(wrapper), + "STDB_BUILDER_PID_FILE": str(pid_file), + "STDB_BUILDER_WORKSPACE": str(root), + "STDB_BUILDER_SOCKET": builder.endpoint, + }) + run([str(deadline_test), "--ignored", "--exact", + "actual_buildctl_deadline_reaps_before_workspace_release", "--test-threads=1"], + deadline_env, timeout=30) + return { + "version": 1, + "dockerfile": str(root / "dockerfile-output"), + "railpack": str(root / "railpack-output"), + "tool_lock": LOCK, + "checks": ["actual_dockerfile", "actual_railpack", "secret_logs", "failed_build", + "no_fallback", "tampered_closure", "concurrent_output", "cancel_and_reap", + "deadline_and_reap"], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--docker-socket", type=Path, required=True) + parser.add_argument("--cli", type=Path, required=True) + parser.add_argument("--deadline-test-binary", type=Path, required=True) + parser.add_argument("--workspace", type=Path, required=True) + args = parser.parse_args() + require(args.docker_socket.is_absolute() and args.docker_socket.is_socket(), "explicit Docker Unix socket required") + require(args.cli.is_absolute() and args.cli.is_file(), "absolute prebuilt CLI executable required") + require(args.deadline_test_binary.is_absolute() and args.deadline_test_binary.is_file(), + "absolute prebuilt deadline test executable required") + require(args.workspace.is_absolute() and not args.workspace.exists(), "workspace must be a new absolute directory") + args.workspace.mkdir(mode=0o700) + for child in ["home", "tmp", "docker-config"]: + (args.workspace / child).mkdir(mode=0o700) + # A missing named config proves local build dispatch does not read global + # credentials or open the supplied ordinary configuration path. + binaries = tools(args.workspace) + builder = Builder(args.workspace, args.docker_socket, binaries) + try: + builder.start() + receipt = cases(args.workspace, args.cli, binaries, builder, args.deadline_test_binary) + finally: + builder.close() + (args.workspace / "acceptance.json").write_text(json.dumps(receipt, indent=2)) + print("Builder acceptance passed; retained OCI layouts are ready for managed publication", flush=True) + + +if __name__ == "__main__": + main() diff --git a/crates/cli/tests/container_build_acceptance/test_fixture.py b/crates/cli/tests/container_build_acceptance/test_fixture.py new file mode 100644 index 00000000000..40ed63ea94d --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/test_fixture.py @@ -0,0 +1,64 @@ +"""Owned-resource failure paths, without contacting Docker or any server.""" +import json +import unittest + +from acceptance import Builder + + +class CleanupTests(unittest.TestCase): + def builder(self, reply): + builder = Builder.__new__(Builder) + builder.name = "fixture-owned-name" + builder.container = None # docker run reply was lost + builder.run_attempted = True + builder.proxy = None + builder.volume_created = False + calls = [] + + def command(*args): + calls.append(args) + if args[:2] == ("container", "inspect"): + if isinstance(reply, Exception): + raise reply + return 0, json.dumps(reply).encode(), b"" + return 0, b"", b"" + + builder.command = command + return builder, calls + + def owned(self): + return [{"Name": "/fixture-owned-name", "Id": "owned-id", "Config": { + "Labels": {"spacetimedb.fixture": "fixture-owned-name"}}}] + + def test_lost_run_reply_removes_only_exact_owned_container(self): + builder, calls = self.builder(self.owned()) + builder.close() + self.assertEqual(calls, [ + ("container", "inspect", "fixture-owned-name"), + ("rm", "--force", "--volumes", "owned-id"), + ]) + + def test_wrong_label_never_authorizes_removal(self): + reply = self.owned() + reply[0]["Config"]["Labels"]["spacetimedb.fixture"] = "another-fixture" + builder, calls = self.builder(reply) + with self.assertRaisesRegex(RuntimeError, "ownership mismatch"): + builder.close() + self.assertEqual(len(calls), 1) + + def test_inspect_error_is_incomplete_cleanup(self): + builder, calls = self.builder(RuntimeError("daemon unavailable")) + with self.assertRaisesRegex(RuntimeError, "positive builder teardown failed"): + builder.close() + self.assertEqual(len(calls), 1) + + def test_returned_identifier_must_match_inspected_object(self): + builder, calls = self.builder(self.owned()) + builder.container = "different-id" + with self.assertRaisesRegex(RuntimeError, "ID mismatch"): + builder.close() + self.assertEqual(len(calls), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/crates/cli/tests/container_build_acceptance/tool-lock.json b/crates/cli/tests/container_build_acceptance/tool-lock.json new file mode 100644 index 00000000000..e62b8e64ec2 --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/tool-lock.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "platform": "darwin-arm64", + "tools": { + "buildctl": { + "version": "0.33.0", + "url": "https://github.com/moby/buildkit/releases/download/v0.33.0/buildkit-v0.33.0.darwin-arm64.tar.gz", + "sha256": "730ac4ffd6f4a88dc404fc675aeaf4cfee414915036042847f0a60861cc8790c", + "member": "bin/buildctl", + "source": "https://api.github.com/repos/moby/buildkit/releases/tags/v0.33.0" + }, + "railpack": { + "version": "0.35.0", + "url": "https://github.com/railwayapp/railpack/releases/download/v0.35.0/railpack-v0.35.0-arm64-apple-darwin.tar.gz", + "sha256": "fb4c16d57458eb7868d48ed8a454014ef40716a6c939929d7b7d5986563a0c65", + "member": "railpack", + "source": "https://api.github.com/repos/railwayapp/railpack/releases/tags/v0.35.0" + } + }, + "images": { + "buildkit": "moby/buildkit@sha256:6c2fa84a6b61ccd72899dde4239f8d5717f05f9a8ca6f3cad185fb1a95a94de3", + "buildkit_arm64": "sha256:e8efce994e456acb94944bcc0530b3188478572914f413d4476568d8b63515c6", + "alpine": "alpine@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1", + "railpack_frontend": "ghcr.io/railwayapp/railpack-frontend@sha256:bc73534934e7929ab3dc41765fb7e25c8c69d9be98c43ef8792fea51f65317bd" + }, + "image_sources": [ + "https://registry-1.docker.io/v2/moby/buildkit/manifests/v0.33.0", + "https://registry-1.docker.io/v2/library/alpine/manifests/3.22.1", + "https://ghcr.io/v2/railwayapp/railpack-frontend/manifests/v0.35.0" + ] +} diff --git a/crates/cli/tests/real_container_builder.rs b/crates/cli/tests/real_container_builder.rs new file mode 100644 index 00000000000..cad32c9c63b --- /dev/null +++ b/crates/cli/tests/real_container_builder.rs @@ -0,0 +1,80 @@ +//! Run only from container_build_acceptance/acceptance.py, which owns and +//! verifies the Docker/BuildKit endpoint and all paths below. No saved defaults. +#![cfg(any(target_os = "macos", target_os = "linux"))] + +use anyhow::{ensure, Context, Result}; +use spacetimedb_cli::container::{ + config::ContainerConfig, + prepare_container, + process::{Invocation, LocalRunner, Output, Runner}, + BuildTools, +}; +use spacetimedb_lib::container::ImagePlatform; +use std::{path::PathBuf, time::Duration}; +use tokio_util::sync::CancellationToken; + +struct ShortDeadline; +impl Runner for ShortDeadline { + async fn run(&self, mut invocation: Invocation) -> Result { + // Exercise the production local owner/kill/reap path without making + // acceptance wait for the ordinary thirty-minute build deadline. + invocation.timeout = Duration::from_secs(2); + LocalRunner.run(invocation).await + } +} + +fn input(name: &str) -> Result { + let path = PathBuf::from(std::env::var_os(name).with_context(|| format!("explicit {name} is required"))?); + ensure!(path.is_absolute(), "fixture input must be absolute"); + Ok(path) +} + +#[tokio::test] +#[ignore = "requires the explicitly owned local BuildKit acceptance fixture"] +async fn actual_buildctl_deadline_reaps_before_workspace_release() -> Result<()> { + let context = input("STDB_BUILDER_CONTEXT")?.canonicalize()?; + let tool = input("STDB_BUILDER_BUILDCTL")?.canonicalize()?; + let pid_file = input("STDB_BUILDER_PID_FILE")?; + let workspace = input("STDB_BUILDER_WORKSPACE")?.canonicalize()?; + let endpoint = std::env::var("STDB_BUILDER_SOCKET")?; + ensure!( + endpoint.starts_with("unix:///"), + "explicit local BuildKit Unix socket is required" + ); + let document: serde_json::Value = serde_json::from_slice(&std::fs::read(context.join("spacetime.json"))?)?; + let configuration: ContainerConfig = serde_json::from_value(document["container"].clone())?; + let result = prepare_container( + &configuration, + &context, + ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + &BuildTools { + buildctl: tool, + buildkit_host: Some(endpoint), + ..Default::default() + }, + &workspace, + &ShortDeadline, + CancellationToken::new(), + ) + .await; + let error = result.err().context("long-running real build unexpectedly succeeded")?; + ensure!( + error.to_string().contains("build deadline"), + "unexpected build failure: {error:#}" + ); + let pid = std::fs::read_to_string(pid_file)?.trim().parse()?; + let pid = rustix::process::Pid::from_raw(pid).context("invalid recorded builder PID")?; + ensure!( + rustix::process::test_kill_process(pid) == Err(rustix::io::Errno::SRCH), + "real buildctl PID still exists after the deadline returned" + ); + ensure!( + std::fs::read_dir(workspace)?.all(|entry| entry + .is_ok_and(|entry| !entry.file_name().to_string_lossy().starts_with(".spacetime-image-"))), + "deadline returned before its owned workspace was released" + ); + Ok(()) +} diff --git a/crates/client-api/src/auth.rs b/crates/client-api/src/auth.rs index 03c122510aa..493c91f0994 100644 --- a/crates/client-api/src/auth.rs +++ b/crates/client-api/src/auth.rs @@ -88,6 +88,7 @@ pub struct SpacetimeAuth { pub claims: SpacetimeIdentityClaims, /// The JWT payload as a json string (after base64 decoding). pub jwt_payload: Box, + pub hosted: Option, } impl SpacetimeAuth { @@ -100,6 +101,7 @@ impl SpacetimeAuth { creds, claims, jwt_payload: payload, + hosted: None, }) } } @@ -109,6 +111,7 @@ impl From for ConnectionAuthCtx { ConnectionAuthCtx { claims: auth.claims, jwt_payload: auth.jwt_payload.clone(), + hosted: auth.hosted, } } } @@ -214,6 +217,9 @@ impl SpacetimeAuth { signer: &impl TokenSigner, expiry: Duration, ) -> Result<(SpacetimeIdentityClaims, String), JwtError> { + if self.hosted.is_some() { + return Err(JwtErrorKind::InvalidToken.into()); + } TokenClaims::from(self.clone()).encode_and_sign_with_expiry(signer, Some(expiry)) } } @@ -401,13 +407,47 @@ pub struct SpacetimeAuthHeader { } #[async_trait::async_trait] -impl axum::extract::FromRequestParts for SpacetimeAuthHeader { +impl axum::extract::FromRequestParts for SpacetimeAuthHeader { type Rejection = AuthorizationRejection; async fn from_request_parts(parts: &mut request::Parts, state: &S) -> Result { let Some(creds) = SpacetimeCreds::from_request_parts(parts)? else { return Ok(Self { auth: None }); }; + if spacetimedb::auth::hosted_tokens::has_reserved_hosted_token_kind(&creds.token) + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))? + { + // Use Axum's matched route parameters, never a client header or a + // hand-parsed URL suffix. Tokens are ineligible for root publishing, + // identity allocation, and generic token exchange routes. + #[derive(Deserialize)] + struct HostedTargetPath { + name_or_identity: crate::util::NameOrIdentity, + } + let axum::extract::Path(params) = axum::extract::Path::::from_request_parts(parts, state) + .await + .map_err(|_| AuthorizationRejection::Required)?; + let target = params + .name_or_identity + .resolve(state) + .await + .map_err(|_| AuthorizationRejection::Required)?; + let verified = state + .authenticate_hosted_token(&creds.token, target) + .await + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))?; + let connection = verified + .into_connection_auth() + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))?; + let auth = SpacetimeAuth { + creds, + claims: connection.claims, + jwt_payload: connection.jwt_payload, + hosted: connection.hosted, + }; + return Ok(Self { auth: Some(auth) }); + } + let claims = validate_token(state, &creds.token) .await .map_err(AuthorizationRejection::Custom)?; @@ -419,6 +459,7 @@ impl axum::extract::FromRequestParts for Space creds, claims, jwt_payload: payload.into(), + hosted: None, }; Ok(Self { auth: Some(auth) }) } @@ -477,7 +518,9 @@ impl SpacetimeAuthHeader { pub struct SpacetimeAuthRequired(pub SpacetimeAuth); #[async_trait::async_trait] -impl axum::extract::FromRequestParts for SpacetimeAuthRequired { +impl axum::extract::FromRequestParts + for SpacetimeAuthRequired +{ type Rejection = AuthorizationRejection; async fn from_request_parts(parts: &mut request::Parts, state: &S) -> Result { let auth = SpacetimeAuthHeader::from_request_parts(parts, state).await?; diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 7ec13a670de..5a2793e0d17 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -55,6 +55,15 @@ pub trait NodeDelegate: Send + Sync { type JwtAuthProviderT: auth::JwtAuthProvider; fn jwt_auth_provider(&self) -> &Self::JwtAuthProviderT; + /// Authenticate a platform-issued hosted credential for the exact resolved + /// database in this request. Editions without container hosting fail closed. + async fn authenticate_hosted_token( + &self, + _token: &str, + _target: Identity, + ) -> anyhow::Result { + anyhow::bail!("Hosted database credentials are not enabled on this server") + } /// Return the leader [`Host`] of `database_id`. /// /// The [`Host`] is spawned implicitly if not already running. @@ -136,7 +145,7 @@ impl Host { pub async fn exec_sql( &self, - auth: AuthCtx, + auth: impl Into, _database: Database, confirmed_read: bool, body: String, @@ -146,7 +155,9 @@ impl Host { .await .map_err(|_| (StatusCode::NOT_FOUND, "module not found".to_string()))?; - tracing::info!(sql = body); + // Administrative SQL may contain environment values. The commitlog + // retains the table mutation, but routine request logs must not copy it. + tracing::info!(sql_bytes = body.len(), "executing SQL"); let mut header = vec![]; let sql_start = std::time::Instant::now(); let sql_span = tracing::trace_span!("execute_sql", total_duration = tracing::field::Empty,); @@ -164,7 +175,9 @@ impl Host { ) .await .map_err(|e| { - log::warn!("{e}"); + // Parser diagnostics can quote an environment value. Return the + // diagnostic to its caller without duplicating it in server logs. + log::warn!("SQL request rejected"); (StatusCode::BAD_REQUEST, e.to_string()) })?; @@ -204,6 +217,28 @@ impl Host { .update_module_host(database, host_type, self.replica_id, program_bytes, policy) .await } + + /// Used only by an authenticated publication coordinator after control + /// admission and quiescing. This does not authorize or start a container. + pub async fn update_with_deployment( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + deployment: spacetimedb::db::deployment::DeploymentCommit, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_deployment( + database, + host_type, + self.replica_id, + program_bytes, + policy, + Some(deployment), + ) + .await + } } /// Parameters for publishing a database. /// @@ -466,6 +501,14 @@ impl NodeDelegate for Arc { (**self).jwt_auth_provider() } + async fn authenticate_hosted_token( + &self, + token: &str, + target: Identity, + ) -> anyhow::Result { + (**self).authenticate_hosted_token(token, target).await + } + async fn leader(&self, database_id: u64) -> Result { (**self).leader(database_id).await } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 60576cf752b..1d6ce136dd7 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -158,16 +158,19 @@ pub async fn call( let (module, Database { owner_identity, .. }) = find_module_and_database(&worker_ctx, name_or_identity).await?; + let caller_auth: ConnectionAuthCtx = auth.into(); + let caller = spacetimedb::auth::invocation::InvocationCaller::from(&caller_auth); + // Call the database's `client_connected` reducer, if any. // If it fails or rejects the connection, bail. module - .call_identity_connected(auth.into(), connection_id) + .call_identity_connected(caller_auth, connection_id) .await .map_err(client_connected_error_to_response)?; let result = match module .call_reducer( - caller_identity, + caller.clone(), Some(connection_id), None, None, @@ -181,7 +184,7 @@ pub async fn call( Err(ReducerCallError::NoSuchReducer | ReducerCallError::ScheduleReducerNotFound) => { // Not a reducer — try procedure instead match module - .call_procedure(caller_identity, Some(connection_id), None, &reducer, args) + .call_procedure(caller, Some(connection_id), None, &reducer, args) .await .result { @@ -549,7 +552,8 @@ where let module_def = &module.info.module_def; let response_json = match version { SchemaVersion::V9 => { - let raw = RawModuleDefV9::from(module_def.as_ref().clone()); + let raw = RawModuleDefV9::try_from(module_def.as_ref().clone()) + .map_err(|err| bad_request(err.to_string().into()))?; axum::Json(sats::serde::SerdeWrapper(raw)).into_response() } SchemaVersion::V10 => { @@ -728,7 +732,7 @@ where // If it rejects the connection, bail before executing SQL. let module = host.module().await.map_err(log_and_500)?; module - .call_identity_connected(caller_auth, connection_id) + .call_identity_connected(caller_auth.clone(), connection_id) .await .map_err(client_connected_error_to_response)?; @@ -737,6 +741,8 @@ where .authorize_sql(caller_identity, database.database_identity) .await?; + let sql_auth = + spacetimedb::auth::invocation::SqlCallAuth::authenticated(sql_auth, &caller_auth).map_err(log_and_500)?; host.exec_sql( sql_auth, database, @@ -1067,6 +1073,11 @@ pub async fn publish( | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { tx_offset, durable_offset, + } + | UpdateDatabaseResult::DeploymentAlreadyCommitted { + tx_offset, + durable_offset, + .. }, ) => { timeout(confirmation_timeout.min(MAX_UPDATE_CONFIRMATION_TIMEOUT), async { diff --git a/crates/codegen/src/util.rs b/crates/codegen/src/util.rs index a13a7ff2aba..938fd759146 100644 --- a/crates/codegen/src/util.rs +++ b/crates/codegen/src/util.rs @@ -10,8 +10,8 @@ use convert_case::{Case, Casing}; use itertools::Itertools; use spacetimedb_lib::db::raw_def::v9::TableAccess; use spacetimedb_lib::sats::layout::PrimitiveType; +use spacetimedb_lib::sats::AlgebraicTypeRef; use spacetimedb_lib::version; -use spacetimedb_lib::{db::raw_def::v9::Lifecycle, sats::AlgebraicTypeRef}; use spacetimedb_primitives::ColList; use spacetimedb_schema::{def::ViewDef, type_for_generate::ProductTypeDef}; use spacetimedb_schema::{ @@ -85,7 +85,10 @@ pub(super) fn type_ref_name(module: &ModuleDef, typeref: AlgebraicTypeRef) -> St pub(super) fn is_type_filterable(typespace: &TypespaceForGenerate, ty: &AlgebraicTypeUse) -> bool { match ty { AlgebraicTypeUse::Primitive(prim) => !matches!(prim, PrimitiveType::F32 | PrimitiveType::F64), - AlgebraicTypeUse::String | AlgebraicTypeUse::Identity | AlgebraicTypeUse::ConnectionId => true, + AlgebraicTypeUse::String + | AlgebraicTypeUse::Identity + | AlgebraicTypeUse::ConnectionId + | AlgebraicTypeUse::Uuid => true, // Sum types with all unit variants: AlgebraicTypeUse::Never => true, AlgebraicTypeUse::Option(inner) => matches!(&**inner, AlgebraicTypeUse::Unit), @@ -98,31 +101,20 @@ pub(super) fn is_reducer_invokable(reducer: &ReducerDef) -> bool { reducer.lifecycle.is_none() } -/// Iterate over all the [`ReducerDef`]s defined by the module, in alphabetical order by name. -/// -/// Skipping the `init` reducer and internal [`FunctionVisibiity::Internal`] reducers because -/// they should not be directly invokable. -/// Sorting is not necessary for reducers because they are already stored in an IndexMap. +/// Non-lifecycle reducer entry points in declaration order. Default clients see +/// only public functions; IncludePrivate adds Private and Internal methods. pub(super) fn iter_reducers(module: &ModuleDef, visibility: CodegenVisibility) -> impl Iterator { module .reducers() - // `RawModuleDefV10` already marks all lifecycle reducers as private, but we keep - // this filter for backward compatibility with older versions where `init` - // reducers were not private. - .filter(|reducer| reducer.lifecycle != Some(Lifecycle::Init)) - // Prior to `RawModuleDefV10`, all reducers were public by default. Filtering out - // internal reducers here does not break SDKs built against older versions. + .filter(|reducer| reducer.lifecycle.is_none()) .filter(move |reducer| match visibility { CodegenVisibility::IncludePrivate => true, - CodegenVisibility::OnlyPublic => !reducer.visibility.is_private(), + CodegenVisibility::OnlyPublic => reducer.visibility.is_client_callable(), }) } -/// Iterate over all the [`ProcedureDef`]s defined by the module, in alphabetical order by name. -/// -/// Skipping internal [`FunctionVisibiity::Internal`] procedures because they should not be -/// directly invokable. -/// Sorting is necessary to have deterministic reproducible codegen. +/// Procedure entry points in alphabetical order. Default clients see only Public +/// functions; IncludePrivate also generates Private and Internal methods. pub(super) fn iter_procedures( module: &ModuleDef, visibility: CodegenVisibility, @@ -132,7 +124,7 @@ pub(super) fn iter_procedures( .sorted_by_key(|procedure| &procedure.name) .filter(move |procedure| match visibility { CodegenVisibility::IncludePrivate => true, - CodegenVisibility::OnlyPublic => !procedure.visibility.is_private(), + CodegenVisibility::OnlyPublic => procedure.visibility.is_client_callable(), }) } @@ -222,3 +214,64 @@ pub(super) fn iter_constraints(table: &TableDef) -> impl Iterator impl Iterator { module.types().sorted_by_key(|table| &table.accessor_name) } + +#[cfg(test)] +mod visibility_tests { + use super::*; + use spacetimedb_lib::db::raw_def::{ + v10::{FunctionVisibility, RawModuleDefV10Builder}, + v9::Lifecycle, + }; + use spacetimedb_lib::{AlgebraicType, ProductType}; + + #[test] + fn public_codegen_excludes_internal_private_and_every_lifecycle() { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer("ordinary", ProductType::unit()); + for (name, visibility) in [ + ("public_function", FunctionVisibility::ClientCallable), + ("private_function", FunctionVisibility::Private), + ("internal_function", FunctionVisibility::Internal), + ] { + builder.add_reducer_with_visibility(name, ProductType::unit(), Some(visibility)); + builder.add_procedure_with_visibility( + format!("{name}_procedure"), + ProductType::unit(), + AlgebraicType::unit(), + Some(visibility), + ); + } + for (name, lifecycle) in [ + ("init", Lifecycle::Init), + ("connect", Lifecycle::OnConnect), + ("disconnect", Lifecycle::OnDisconnect), + ] { + builder.add_lifecycle_reducer(lifecycle, name, ProductType::unit()); + } + let module: ModuleDef = builder.finish().try_into().unwrap(); + let names = |visibility| { + iter_reducers(&module, visibility) + .map(|r| &r.name[..]) + .collect::>() + }; + assert_eq!(names(CodegenVisibility::OnlyPublic), ["ordinary", "public_function"]); + assert_eq!( + names(CodegenVisibility::IncludePrivate), + ["ordinary", "public_function", "private_function", "internal_function"] + ); + let names = |visibility| { + iter_procedures(&module, visibility) + .map(|p| &p.name[..]) + .collect::>() + }; + assert_eq!(names(CodegenVisibility::OnlyPublic), ["public_function_procedure"]); + assert_eq!( + names(CodegenVisibility::IncludePrivate), + [ + "internal_function_procedure", + "private_function_procedure", + "public_function_procedure" + ] + ); + } +} diff --git a/crates/codegen/tests/codegen.rs b/crates/codegen/tests/codegen.rs index 06dc3ebe8fc..faf3966faee 100644 --- a/crates/codegen/tests/codegen.rs +++ b/crates/codegen/tests/codegen.rs @@ -39,3 +39,45 @@ declare_tests! { test_codegen_typescript => TypeScript, test_codegen_rust => Rust, } + +#[test] +fn rust_uuid_primary_and_unique_keys_generate_registered_lookup_accessors() { + use spacetimedb_lib::{ + db::raw_def::{v10::RawModuleDefV10Builder, v9::btree}, + AlgebraicType, + }; + + let mut builder = RawModuleDefV10Builder::new(); + builder + .build_table_with_new_type( + "uuid_rows", + [ + ("id", AlgebraicType::uuid()), + ("alias", AlgebraicType::uuid()), + ("sequence", AlgebraicType::U64), + ], + true, + ) + .with_primary_key(0) + .with_unique_constraint(0) + .with_index(btree(0), "uuid_rows_id_idx", "id") + .with_unique_constraint(1) + .with_index(btree(1), "uuid_rows_alias_idx", "alias") + .with_unique_constraint(2) + .with_index(btree(2), "uuid_rows_sequence_idx", "sequence") + .finish(); + let module = ModuleDef::try_from(builder.finish()).unwrap(); + let table = generate(&module, &Rust, &CodegenOptions::default()) + .into_iter() + .find(|file| file.filename == "uuid_rows_table.rs") + .unwrap() + .code; + for column in ["id", "alias"] { + assert!(table.contains(&format!("pub fn {column}(&self)"))); + assert!(table.contains(&format!( + "add_unique_constraint::<__sdk::Uuid>({column:?}, |row| &row.{column})" + ))); + } + assert!(table.contains("pub fn find(&self, col_val: &__sdk::Uuid) -> Option")); + assert!(table.contains("add_unique_constraint::(\"sequence\", |row| &row.sequence)")); +} diff --git a/crates/core/src/auth/hosted_tokens.rs b/crates/core/src/auth/hosted_tokens.rs new file mode 100644 index 00000000000..77622b2706a --- /dev/null +++ b/crates/core/src/auth/hosted_tokens.rs @@ -0,0 +1,332 @@ +//! Explicit platform trust for hosted database credentials. No OIDC discovery or fallback. + +use anyhow::{ensure, Context}; +use jsonwebtoken::DecodingKey; +pub use spacetimedb_auth::hosted::{ + has_reserved_hosted_token_kind, sign_hosted_token, HostedTokenBinding, HostedTokenClaims, VerifiedHostedAuth, + HOSTED_TOKEN_KIND, HOSTED_TOKEN_TYPE, MAX_HOSTED_TOKEN_LIFETIME, +}; +use spacetimedb_auth::hosted::{unverified_hosted_token_claims, verify_hosted_token}; +use spacetimedb_lib::Identity; +use std::collections::HashMap; +use std::time::SystemTime; + +/// Only configured platform signers can attest registered source databases. +pub struct HostedTokenValidator { + trusted_issuers: HashMap, DecodingKey>, +} + +impl HostedTokenValidator { + pub fn new(issuers: impl IntoIterator, DecodingKey)>) -> anyhow::Result { + let mut trusted_issuers = HashMap::new(); + for (issuer, key) in issuers { + ensure!( + !issuer.is_empty() && issuer.len() <= 128, + "invalid trusted hosted issuer" + ); + ensure!( + trusted_issuers.insert(issuer, key).is_none(), + "duplicate trusted hosted issuer" + ); + } + Ok(Self { trusted_issuers }) + } + + /// `resolve_binding` reads authoritative state, including this issuer's source + /// registration, open admission, current placement/incarnation and target grant. + /// Return None if any requirement is absent. Its inputs are untrusted routing hints; + /// the callback must never copy them into a fabricated binding or mutate state. + /// The returned proof still requires target-fence checks at every later admission. + pub fn validate_token( + &self, + token: &str, + target: Identity, + now: SystemTime, + resolve_binding: impl FnOnce(&str, Identity, Identity) -> Option, + ) -> anyhow::Result { + let hints = unverified_hosted_token_claims(token)?; + ensure!(hints.target_database == target, "hosted credential target mismatch"); + let key = self + .trusted_issuers + .get(&hints.issuer) + .context("untrusted hosted credential issuer")?; + let binding = resolve_binding(&hints.issuer, hints.source_database, target) + .context("hosted source registration, instance, or target grant is unavailable")?; + ensure!( + binding.target_database == target, + "authoritative hosted binding target mismatch" + ); + verify_hosted_token(token, key, &hints.issuer, &binding, now) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::{ + token_validation::{FullTokenValidator, TokenValidator, UnimplementedTokenValidator}, + JwtKeys, + }; + use jsonwebtoken::{Algorithm, Header}; + use serde_json::{json, Value}; + use spacetimedb_auth::hosted::{has_reserved_hosted_token_kind, MAX_HOSTED_TOKEN_BYTES}; + use std::time::{Duration, UNIX_EPOCH}; + + fn fixture() -> (JwtKeys, HostedTokenValidator, HostedTokenBinding, SystemTime) { + let keys = JwtKeys::generate().unwrap(); + let validator = HostedTokenValidator::new([("platform.test".into(), keys.public.clone())]).unwrap(); + let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let binding = HostedTokenBinding { + source_database: Identity::from_claims("source", "database"), + target_database: Identity::from_claims("target", "database"), + generation: 9_007_199_254_740_993, + grant_revision: 9_007_199_254_740_995, + lease_expires_at: now + Duration::from_secs(30), + }; + (keys, validator, binding, now) + } + + fn mint(keys: &JwtKeys, binding: &HostedTokenBinding, now: SystemTime) -> String { + sign_hosted_token( + &keys.private, + "platform.test", + binding, + now, + now + Duration::from_secs(20), + "token-private-id", + ) + .unwrap() + } + + fn change(token: &str, keys: &JwtKeys, mutate: impl FnOnce(&mut Value, &mut Header)) -> String { + let mut claims = serde_json::to_value(unverified_hosted_token_claims(token).unwrap()).unwrap(); + let mut header = Header::new(Algorithm::ES256); + header.typ = Some(HOSTED_TOKEN_TYPE.into()); + mutate(&mut claims, &mut header); + jsonwebtoken::encode(&header, &claims, &keys.private).unwrap() + } + + #[test] + fn hosted_sender_target_authority_and_claims_are_preserved() { + let (keys, validator, mut binding, now) = fixture(); + for self_call in [false, true] { + if self_call { + binding.target_database = binding.source_database; + } + let token = mint(&keys, &binding, now); + assert!(has_reserved_hosted_token_kind(&token).unwrap()); + let verified = validator + .validate_token(&token, binding.target_database, now, |issuer, source, target| { + assert_eq!(issuer, "platform.test"); + assert_eq!(source, binding.source_database); + assert_eq!(target, binding.target_database); + Some(binding) + }) + .unwrap(); + assert_eq!(verified.is_internal(), self_call); + assert_eq!(verified.generation(), binding.generation); + assert_eq!(verified.grant_revision(), binding.grant_revision); + assert!(verified.check_at(now + Duration::from_secs(20)).is_err()); + let ctx = verified.into_connection_auth().unwrap(); + assert_eq!(ctx.claims.identity, binding.source_database); + assert_ne!( + ctx.claims.identity, + Identity::from_claims(&ctx.claims.issuer, &ctx.claims.subject) + ); + assert!(ctx.hosted.is_some()); + let payload: Value = serde_json::from_str(&ctx.jwt_payload).unwrap(); + assert_eq!(payload["generation"].as_u64(), Some(binding.generation)); + assert_eq!(payload["grant_revision"].as_u64(), Some(binding.grant_revision)); + assert_eq!(payload["aud"], binding.target_database.to_hex().as_str()); + assert_eq!(payload["iss"], "platform.test"); + let debug = format!("{ctx:?}"); + assert!(!debug.contains("token-private-id")); + assert!(!debug.contains(&token)); + assert!(!debug.contains("jwt_payload")); + } + } + + #[test] + fn hosted_validation_rejects_wrong_authority_binding_and_wire_shape() { + let (keys, validator, binding, now) = fixture(); + let token = mint(&keys, &binding, now); + let other_keys = JwtKeys::generate().unwrap(); + assert!(validator + .validate_token( + &mint(&other_keys, &binding, now), + binding.target_database, + now, + |_, _, _| Some(binding) + ) + .is_err()); + assert!(validator + .validate_token(&token, binding.source_database, now, |_, _, _| Some(binding)) + .is_err()); + assert!(validator + .validate_token(&token, binding.target_database, now, |_, _, _| None) + .is_err()); + let invalid_fields = [ + ("kind", json!("spacetimedb_hosted_v2")), + ("iss", json!("unknown.test")), + ("source_database", json!(binding.target_database.to_hex().as_str())), + ("sub", json!("other")), + ("aud", json!(binding.source_database.to_hex().as_str())), + ("aud", json!([binding.target_database.to_hex().as_str()])), + ("generation", json!(binding.generation - 1)), + ("grant_revision", json!(binding.grant_revision - 1)), + ("iat", json!(1_700_000_001_u64)), + ("exp", json!(1_700_000_000_u64)), + ("exp", json!(1_700_000_031_u64)), + ("exp", json!(u64::MAX)), + ("jti", json!("")), + ("hex_identity", json!(binding.source_database.to_hex().as_str())), + ]; + for (field, value) in invalid_fields { + let changed = change(&token, &keys, |claims, _| claims[field] = value); + assert!( + validator + .validate_token(&changed, binding.target_database, now, |_, _, _| Some(binding)) + .is_err(), + "accepted changed {field}" + ); + } + let wrong_type = change(&token, &keys, |_, header| header.typ = Some("JWT".into())); + assert!(validator + .validate_token(&wrong_type, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let missing_exp = change(&token, &keys, |claims, _| { + claims.as_object_mut().unwrap().remove("exp"); + }); + assert!(validator + .validate_token(&missing_exp, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let overflowing_time = change(&token, &keys, |claims, _| { + claims["iat"] = json!(u64::MAX - 20); + claims["exp"] = json!(u64::MAX); + }); + assert!(validator + .validate_token(&overflowing_time, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let short_lease = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(19), + ..binding + }; + assert!(validator + .validate_token(&token, binding.target_database, now, |_, _, _| Some(short_lease)) + .is_err()); + assert!(validator + .validate_token( + &"x".repeat(MAX_HOSTED_TOKEN_BYTES + 1), + binding.target_database, + now, + |_, _, _| Some(binding) + ) + .is_err()); + let claims = unverified_hosted_token_claims(&token).unwrap(); + let mut hs_header = Header::new(Algorithm::HS256); + hs_header.typ = Some(HOSTED_TOKEN_TYPE.into()); + let hs_token = jsonwebtoken::encode( + &hs_header, + &claims, + &jsonwebtoken::EncodingKey::from_secret(b"not-a-platform-key"), + ) + .unwrap(); + assert!(validator + .validate_token(&hs_token, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + } + + #[test] + fn broker_signing_obeys_confirmed_lease_and_lifetime() { + let (keys, _, binding, now) = fixture(); + for expiry in [now, now + Duration::from_secs(31)] { + assert!(sign_hosted_token(&keys.private, "platform.test", &binding, now, expiry, "id").is_err()); + } + let short_lease = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(10), + ..binding + }; + assert!(sign_hosted_token( + &keys.private, + "platform.test", + &short_lease, + now, + now + Duration::from_secs(11), + "id" + ) + .is_err()); + } + + #[tokio::test] + async fn ordinary_validation_rejects_reserved_platform_kinds_and_types() { + let (keys, _, binding, _) = fixture(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(30), + ..binding + }; + let token = mint(&keys, &binding, now); + let ordinary = FullTokenValidator { + local_key: keys.public.clone(), + local_issuer: "platform.test".into(), + oidc_validator: UnimplementedTokenValidator, + }; + for reserved in [ + token.clone(), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_hosted_future"); + header.typ = Some("JWT".into()); + }), + change(&token, &keys, |claims, header| { + claims.as_object_mut().unwrap().remove("kind"); + header.typ = Some("spacetimedb-hosted-v2+jwt".into()); + }), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_container_lease_v1"); + header.typ = Some("JWT".into()); + }), + change(&token, &keys, |claims, header| { + claims.as_object_mut().unwrap().remove("kind"); + header.typ = Some("spacetimedb-container-lease+jwt".into()); + }), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_container_registry_future"); + header.typ = Some("JWT".into()); + }), + ] { + assert!(spacetimedb_auth::hosted::has_reserved_platform_token_kind(&reserved).unwrap()); + assert!(keys.public.validate_token(&reserved).await.is_err()); + assert!(ordinary.validate_token(&reserved).await.is_err()); + } + } + + #[tokio::test] + async fn reserved_classification_preserves_ordinary_token_algorithms() { + let rsa = openssl::rsa::Rsa::generate(2048).unwrap(); + let rsa = openssl::pkey::PKey::from_rsa(rsa).unwrap(); + let ec = JwtKeys::generate().unwrap(); + let keys = [ + (Algorithm::ES256, ec.private, ec.public), + ( + Algorithm::RS256, + jsonwebtoken::EncodingKey::from_rsa_pem(&rsa.private_key_to_pem_pkcs8().unwrap()).unwrap(), + DecodingKey::from_rsa_pem(&rsa.public_key_to_pem().unwrap()).unwrap(), + ), + ( + Algorithm::HS256, + jsonwebtoken::EncodingKey::from_secret(b"ordinary-oidc-test-secret"), + DecodingKey::from_secret(b"ordinary-oidc-test-secret"), + ), + ]; + for (algorithm, private, public) in keys { + let claims = json!({ "iss": "ordinary.test", "sub": "a-user", "iat": 1_700_000_000_u64, "kind": "ordinary_application_kind" }); + let token = jsonwebtoken::encode(&Header::new(algorithm), &claims, &private).unwrap(); + assert!( + !has_reserved_hosted_token_kind(&token).unwrap(), + "misclassified {algorithm:?}" + ); + let validated = public.validate_token(&token).await.unwrap(); + assert_eq!(validated.identity, Identity::from_claims("ordinary.test", "a-user")); + } + } +} diff --git a/crates/core/src/auth/invocation.rs b/crates/core/src/auth/invocation.rs new file mode 100644 index 00000000000..dc9004effaa --- /dev/null +++ b/crates/core/src/auth/invocation.rs @@ -0,0 +1,127 @@ +//! Authority carried from authenticated admission to module execution. +//! +//! Identity equality never establishes internal authority. A hosted proof can +//! only originate in signature verification against trusted platform state. + +use super::hosted_tokens::VerifiedHostedAuth; +use spacetimedb_auth::identity::ConnectionAuthCtx; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_lib::Identity; +use spacetimedb_schema::def::ModuleDef; +use std::time::SystemTime; + +#[derive(Clone, Debug)] +pub struct InvocationCaller { + pub(crate) identity: Identity, + pub(crate) hosted: Option>, +} + +impl From for InvocationCaller { + fn from(identity: Identity) -> Self { + Self { identity, hosted: None } + } +} + +impl From<&ConnectionAuthCtx> for InvocationCaller { + fn from(auth: &ConnectionAuthCtx) -> Self { + Self { + identity: auth.claims.identity, + hosted: auth.hosted.clone().map(std::sync::Arc::new), + } + } +} + +/// SQL permissions and the authenticated container restrictions are independent. +/// Internal status never impersonates the database owner or grants SQL rights. +#[derive(Clone)] +pub struct SqlCallAuth { + permissions: spacetimedb_lib::identity::AuthCtx, + pub(crate) hosted: Option>, +} + +impl From for SqlCallAuth { + fn from(permissions: spacetimedb_lib::identity::AuthCtx) -> Self { + Self { + permissions, + hosted: None, + } + } +} + +impl std::ops::Deref for SqlCallAuth { + type Target = spacetimedb_lib::identity::AuthCtx; + fn deref(&self) -> &Self::Target { + &self.permissions + } +} + +impl SqlCallAuth { + pub fn authenticated( + permissions: spacetimedb_lib::identity::AuthCtx, + auth: &ConnectionAuthCtx, + ) -> anyhow::Result { + anyhow::ensure!( + permissions.caller() == auth.claims.identity, + "SQL caller does not match authenticated sender" + ); + if let Some(proof) = &auth.hosted { + anyhow::ensure!( + proof.source_database() == auth.claims.identity, + "SQL hosted proof does not match authenticated sender" + ); + } + Ok(Self { + permissions, + hosted: auth.hosted.clone().map(std::sync::Arc::new), + }) + } +} + +impl InvocationCaller { + pub(crate) fn flags_for(&self, target: Identity, module: &ModuleDef) -> anyhow::Result { + let Some(proof) = &self.hosted else { return Ok(0) }; + anyhow::ensure!( + proof.source_database() == self.identity, + "hosted caller does not match its proof" + ); + anyhow::ensure!( + proof.target_database() == target, + "hosted credential targets another database" + ); + anyhow::ensure!( + module.supports_hosted_auth_v1(), + "module does not support hosted authentication" + ); + proof.check_at(SystemTime::now())?; + Ok(u32::from(proof.is_internal())) + } +} + +/// Must run while holding the transaction that admits the database operation. +/// A check before queueing does not serialize with generation revocation. +pub(crate) fn check_hosted_admission( + state: &S, + database: &crate::db::relational_db::RelationalDB, + proof: Option<&VerifiedHostedAuth>, +) -> anyhow::Result<()> { + let Some(proof) = proof else { return Ok(()) }; + anyhow::ensure!( + database.hosted_admission().is_open(), + "receiving database has not reconciled hosted admission" + ); + anyhow::ensure!( + proof.target_database() == database.database_identity(), + "hosted credential targets another database" + ); + proof.check_at(SystemTime::now())?; + crate::db::deployment::check_container_fence( + state, + proof.source_database(), + proof.generation(), + proof.grant_revision(), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/auth/invocation/tests.rs b/crates/core/src/auth/invocation/tests.rs new file mode 100644 index 00000000000..0507ca32bdb --- /dev/null +++ b/crates/core/src/auth/invocation/tests.rs @@ -0,0 +1,199 @@ +use super::*; +use crate::auth::hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}; +use crate::auth::JwtKeys; +use crate::db::deployment::install_container_fence; +use crate::db::relational_db::tests_utils::TestDB; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::StContainerFenceRow; +use spacetimedb_lib::db::auth::StAccess; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; +use spacetimedb_lib::identity::AuthCtx; +use std::time::Duration; + +fn module(hosted_auth: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + if hosted_auth { + builder.add_capability("hosted_auth_v1"); + } + builder.finish().try_into().unwrap() +} + +/// Obtain every proof through the production signer and target-bound verifier. +fn authenticate(source: Identity, target: Identity, now: SystemTime) -> ConnectionAuthCtx { + let keys = JwtKeys::generate().unwrap(); + let binding = HostedTokenBinding { + source_database: source, + target_database: target, + generation: 3, + grant_revision: 7, + lease_expires_at: now + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "test.platform", + &binding, + now, + now + Duration::from_secs(20), + "invocation-test", + ) + .unwrap(); + HostedTokenValidator::new([("test.platform".into(), keys.public)]) + .unwrap() + .validate_token(&token, target, now, |issuer, requested_source, requested_target| { + (issuer == "test.platform" && requested_source == source && requested_target == target).then_some(binding) + }) + .unwrap() + .into_connection_auth() + .unwrap() +} + +fn fence(source: Identity, generation: u64, grant_revision: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: source.into(), + generation, + target_grant_revision: grant_revision, + target_set_hash: spacetimedb_lib::hash_bytes(b"configured targets"), + allowed, + } +} + +#[test] +fn internal_requires_verified_self_call_and_updated_bindings() { + let target = Identity::ONE; + let foreign = Identity::from_u256(2u64.into()); + let updated = module(true); + let old_bindings = module(false); + // This also covers an ordinary connection whose sender equals the database. + assert_eq!(InvocationCaller::from(target).flags_for(target, &updated).unwrap(), 0); + for (source, expected_flags) in [(target, 1), (foreign, 0)] { + let auth = authenticate(source, target, SystemTime::now()); + let caller = InvocationCaller::from(&auth); + assert_eq!(caller.flags_for(target, &updated).unwrap(), expected_flags); + assert!(caller.flags_for(target, &old_bindings).is_err()); + assert!(caller.flags_for(Identity::ZERO, &updated).is_err()); + } +} + +#[test] +fn authenticated_proof_cannot_be_paired_with_another_sender_or_sql_caller() { + let source = Identity::ONE; + let target = Identity::from_u256(2u64.into()); + let owner = Identity::from_u256(3u64.into()); + let mut auth = authenticate(source, target, SystemTime::now()); + assert!(SqlCallAuth::authenticated(AuthCtx::for_current(owner), &auth).is_err()); + // ConnectionAuthCtx has public fields for trusted host code. Defend the + // boundary against accidentally mixing separately authenticated contexts. + auth.claims.identity = owner; + assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); + assert!(SqlCallAuth::authenticated(AuthCtx::for_current(owner), &auth).is_err()); +} + +#[test] +fn internal_authentication_does_not_grant_owner_sql_permissions() { + let source = Identity::ONE; + let owner = Identity::from_u256(3u64.into()); + let auth = authenticate(source, source, SystemTime::now()); + assert_eq!( + InvocationCaller::from(&auth).flags_for(source, &module(true)).unwrap(), + 1 + ); + let sql = SqlCallAuth::authenticated(AuthCtx::new(owner, source), &auth).unwrap(); + assert_eq!(sql.caller(), source); + assert!(sql.has_read_access(StAccess::Public)); + assert!(!sql.has_read_access(StAccess::Private)); + assert!(!sql.has_write_access()); + assert!(!sql.bypass_rls()); +} + +#[test] +fn transaction_admission_rechecks_persisted_fences_after_initial_authentication() { + let db = TestDB::in_memory().unwrap(); + let target = db.database_identity(); + let source = Identity::ONE; + let auth = authenticate(source, target, SystemTime::now()); + let proof = auth.hosted.as_ref(); + let wrong_target = authenticate(source, Identity::from_u256(99u64.into()), SystemTime::now()); + db.hosted_admission().begin().unwrap().complete().unwrap(); + let caller = InvocationCaller::from(&auth); + assert_eq!(caller.flags_for(target, &module(true)).unwrap(), 0); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + assert!(check_hosted_admission(tx, &db, proof).is_err()); + install_container_fence(&db, tx, &fence(source, 3, 7, true))?; + assert!(!db.hosted_admission().is_open()); + db.hosted_admission().begin()?.complete()?; + check_hosted_admission(tx, &db, proof)?; + assert!(check_hosted_admission(tx, &db, wrong_target.hosted.as_ref()).is_err()); + Ok(()) + }) + .unwrap(); + // Revocation commits after token verification and before the queued call. + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + install_container_fence(&db, tx, &fence(source, 4, 8, false))?; + Ok(()) + }) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + assert!(check_hosted_admission(tx, &db, proof).is_err()); + check_hosted_admission(tx, &db, None)?; + // Another generation does not reactivate a copied credential. + install_container_fence(&db, tx, &fence(source, 5, 9, true))?; + assert!(check_hosted_admission(tx, &db, proof).is_err()); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn reopened_database_requires_a_fresh_sweep_despite_a_replayed_allowed_fence() { + let db = TestDB::durable().unwrap(); + let source = Identity::ONE; + let target = db.database_identity(); + let auth = authenticate(source, target, SystemTime::now()); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &fence(source, 3, 7, true)) + }) + .unwrap(); + let old_sweep = db.hosted_admission().begin().unwrap(); + let db = db.reopen().unwrap(); + // Shutdown seals the old object, including any late coordinator ticket. + assert!(old_sweep.complete().is_err()); + assert!(!db.hosted_admission().is_open()); + db.with_read_only(Workload::ForTests, |tx| { + crate::db::deployment::check_container_fence(tx, source, 3, 7).unwrap(); + let error = check_hosted_admission(tx, &db, auth.hosted.as_ref()).unwrap_err(); + assert!(error.to_string().contains("has not reconciled")); + check_hosted_admission(tx, &db, None).unwrap(); + }); + db.hosted_admission().begin().unwrap().complete().unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + check_hosted_admission(tx, &db, auth.hosted.as_ref()).unwrap(); + }); +} + +#[test] +fn expired_verified_proof_is_rejected_at_both_call_and_transaction_admission() { + let db = TestDB::in_memory().unwrap(); + let target = db.database_identity(); + let source = Identity::ONE; + db.hosted_admission().begin().unwrap().complete().unwrap(); + // Valid when received, expired before execution, without sleeps or forged proofs. + let auth = authenticate(source, target, SystemTime::now() - Duration::from_secs(60)); + assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + install_container_fence(&db, tx, &fence(source, 3, 7, true))?; + assert!(check_hosted_admission(tx, &db, auth.hosted.as_ref()).is_err()); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn shutdown_seals_hosted_admission_even_for_retained_memory_database_handles() { + let db = TestDB::in_memory().unwrap(); + let pending = db.hosted_admission().begin().unwrap(); + assert_eq!(db.shutdown().await, None); + assert!(pending.complete().is_err()); + assert!(!db.hosted_admission().is_open()); + assert!(db.hosted_admission().begin().is_err()); + assert_eq!(db.shutdown().await, None); +} diff --git a/crates/core/src/auth/mod.rs b/crates/core/src/auth/mod.rs index e1e38a667c4..49f2686f062 100644 --- a/crates/core/src/auth/mod.rs +++ b/crates/core/src/auth/mod.rs @@ -7,6 +7,8 @@ use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use crate::config::CertificateAuthority; pub use spacetimedb_auth::identity; +pub mod hosted_tokens; +pub mod invocation; pub mod token_validation; /// JWT verification and signing keys. diff --git a/crates/core/src/auth/token_validation.rs b/crates/core/src/auth/token_validation.rs index c38d732882d..e165be43ced 100644 --- a/crates/core/src/auth/token_validation.rs +++ b/crates/core/src/auth/token_validation.rs @@ -97,6 +97,7 @@ where T: TokenValidator + Send + Sync, { async fn validate_token(&self, token: &str) -> Result { + reject_reserved_hosted_credentials(token)?; let local_key_error = { let first_validator = BasicTokenValidator { public_key: self.local_key.clone(), @@ -145,6 +146,7 @@ lazy_static! { #[async_trait] impl TokenValidator for DecodingKey { async fn validate_token(&self, token: &str) -> Result { + reject_reserved_hosted_credentials(token)?; let mut validation = Validation::new(jsonwebtoken::Algorithm::ES256); validation.algorithms = vec![ jsonwebtoken::Algorithm::ES256, @@ -245,6 +247,7 @@ pub struct OidcTokenValidator; // Get the issuer out of a token without validating the signature. fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { + reject_reserved_hosted_credentials(token)?; let mut validation = Validation::new(jsonwebtoken::Algorithm::ES256); validation.set_required_spec_claims(&REQUIRED_CLAIMS); validation.validate_aud = false; @@ -254,6 +257,16 @@ fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { Ok(data.claims.issuer) } +fn reject_reserved_hosted_credentials(token: &str) -> Result<(), TokenValidationError> { + if spacetimedb_auth::hosted::has_reserved_platform_token_kind(token)? { + return Err(anyhow::anyhow!( + "platform container credentials require their dedicated validator and cannot be exchanged" + ) + .into()); + } + Ok(()) +} + #[async_trait] impl TokenValidator for OidcTokenValidator { async fn validate_token(&self, token: &str) -> Result { diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 8f19156d6bb..378842f3b29 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -8,6 +8,7 @@ use std::task::{Context, Poll}; use std::time::{Instant, SystemTime}; use super::{message_handlers, ClientActorId, MessageHandleError, OutboundMessage}; +use crate::auth::{hosted_tokens::VerifiedHostedAuth, invocation::check_hosted_admission}; use crate::db::relational_db::RelationalDB; use crate::error::DBError; use crate::host::module_host::{ClientConnectedError, ProcedureResultTarget}; @@ -23,6 +24,7 @@ use log::warn; use prometheus::{Histogram, IntCounter, IntGauge}; use spacetimedb_auth::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2}; +use spacetimedb_datastore::execution_context::Workload; use spacetimedb_durability::{DurableOffset, TxOffset}; use spacetimedb_lib::identity::{AuthCtx, RequestId}; use spacetimedb_lib::metrics::ExecutionMetrics; @@ -125,9 +127,29 @@ pub trait DurableOffsetSupply: Send { /// - `Ok(Some(DurableOffset))` otherwise /// fn durable_offset(&mut self) -> Result, NoSuchModule>; + + /// Recheck the authoritative target state, never a cached generation. + fn check_hosted_auth( + &mut self, + _proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + Box::pin(async { anyhow::bail!("hosted connection has no authoritative database state") }) + } } impl DurableOffsetSupply for watch::Receiver { + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + if self.has_changed().is_err() { + return Box::pin(async { Err(NoSuchModule.into()) }); + } + let module = self.borrow().clone(); + let mut db = module.relational_db().clone(); + db.check_hosted_auth(proof) + } + fn durable_offset(&mut self) -> Result, NoSuchModule> { let module = if self.has_changed().map_err(|_| NoSuchModule)? { self.borrow_and_update() @@ -140,6 +162,20 @@ impl DurableOffsetSupply for watch::Receiver { } impl DurableOffsetSupply for Arc { + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + let db = self.clone(); + let proof = proof.clone(); + Box::pin(async move { + tokio::task::spawn_blocking(move || { + db.with_read_only(Workload::Internal, |tx| check_hosted_admission(tx, &db, Some(&proof))) + }) + .await? + }) + } + fn durable_offset(&mut self) -> Result, NoSuchModule> { Ok(self.durable_tx_offset()) } @@ -157,6 +193,7 @@ pub struct ClientConnectionReceiver { channel: MeteredReceiver, pending: Vec, offset_supply: Box, + hosted_sender: Option>, } impl ClientConnectionReceiver { @@ -172,6 +209,7 @@ impl ClientConnectionReceiver { channel, pending: Vec::new(), offset_supply: Box::new(offset_supply), + hosted_sender: None, } } @@ -214,6 +252,9 @@ impl ClientConnectionReceiver { /// These values are stored internally, so calling `recv_many` again will /// not lose data. pub async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + if !self.hosted_connection_is_valid().await { + return 0; + } // If there are no pending updates and the input channel has been closed, // no more messages can be received from this receiver. if max == 0 || (self.pending.is_empty() && self.channel.recv_many(&mut self.pending, max).await == 0) { @@ -223,14 +264,14 @@ impl ClientConnectionReceiver { // If we don't have to wait for txns to be made durable, // drain the pending updates. if !self.confirmed_reads { - return self.drain_pending(buf, max); + return self.drain_pending(buf, max).await; } // If we do have to wait for txns to be made durable, // but the next client update doesn't have a tx offset, // there's no reason to wait - just send it. if !self.pending_update_has_offset() { - return self.drain_pending(buf, 1); + return self.drain_pending(buf, 1).await; } // Otherwise, grab the next offset that we should wait for. @@ -243,12 +284,12 @@ impl ClientConnectionReceiver { warn!("database went away while waiting for durable offset"); return 0; } - self.drain_pending(buf, n) + self.drain_pending(buf, n).await } // Database shut down or crashed. Err(NoSuchModule) => 0, // In-memory database. - Ok(None) => self.drain_pending(buf, max), + Ok(None) => self.drain_pending(buf, max).await, } } @@ -265,13 +306,40 @@ impl ClientConnectionReceiver { } /// Drain the pending [`ClientUpdate`]s, up to `max, into `buf`. - fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + async fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + // A queued update may predate revocation, and a confirmed-read wait may + // outlast the credential. Check again immediately before delivery. + if !self.hosted_connection_is_valid().await { + return 0; + } let n = self.pending.len().min(max); buf.reserve(n); buf.extend(self.pending.drain(..n).map(|u| u.message)); n } + async fn hosted_connection_is_valid(&mut self) -> bool { + let Some(sender) = &self.hosted_sender else { return true }; + let valid = match sender.upgrade() { + Some(sender) => { + let valid = match &sender.auth.hosted { + Some(proof) if !sender.is_cancelled() => self.offset_supply.check_hosted_auth(proof).await.is_ok(), + _ => false, + }; + if !valid { + sender.cancel_hosted_connection(); + } + valid + } + None => false, + }; + if !valid { + self.pending.clear(); + self.close(); + } + valid + } + /// Does the next pending update have a tx offset? /// /// Assumes that [`Self::pending`] is not empty. @@ -354,6 +422,13 @@ pub enum ClientSendError { } impl ClientConnectionSender { + /// The fence installer awaits the returned task's completion before ack. + pub(crate) fn cancel_hosted_connection(&self) -> AbortHandle { + self.cancelled.store(true, Ordering::Release); + self.abort_handle.abort(); + self.abort_handle.clone() + } + pub fn dummy_with_channel( id: ClientActorId, config: ClientConfig, @@ -423,6 +498,16 @@ impl ClientConnectionSender { } fn send(&self, message: ClientUpdate) -> Result<(), ClientSendError> { + // Do not acquire a database transaction here: broadcasts can already + // hold one. Durable fencing is checked at admission and delivery. + if self + .auth + .hosted + .as_ref() + .is_some_and(|proof| proof.check_at(SystemTime::now()).is_err()) + { + self.cancel_hosted_connection(); + } if self.cancelled.load(Relaxed) { return Err(ClientSendError::Cancelled); } @@ -469,6 +554,65 @@ impl ClientConnectionSender { } } +/// Runs independently of the socket actor, so blocked writes and idle sockets +/// cannot keep credentials alive. Expiry uses a monotonic deadline captured once. +fn spawn_hosted_connection_watchdog( + sender: std::sync::Weak, + mut supply: impl DurableOffsetSupply + 'static, + subscriptions: Option, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let actor = sender.upgrade().map(|connection| connection.abort_handle.clone()); + async { + let Some(connection) = sender.upgrade() else { return }; + let Some(proof) = connection.auth.hosted.clone() else { + return; + }; + let deadline = tokio::time::Instant::now() + proof.remaining_lifetime(SystemTime::now()); + drop(connection); + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + if let Some(connection) = sender.upgrade() { connection.cancel_hosted_connection(); } + return; + } + _ = interval.tick() => {} + } + let Some(connection) = sender.upgrade() else { return }; + if connection.abort_handle.is_finished() || connection.is_cancelled() { + return; + } + let checked = tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + connection.cancel_hosted_connection(); + return; + } + checked = supply.check_hosted_auth(&proof) => checked, + }; + if checked.is_err() { + connection.cancel_hosted_connection(); + return; + } + } + } + .await; + // Keep the registry entry until socket I/O has actually stopped. A + // concurrent target barrier must still find and await an aborted actor. + if let Some(actor) = actor { + while !actor.is_finished() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + if let Some(subscriptions) = subscriptions { + subscriptions.unregister_hosted_connection(&sender); + } + }) +} + #[derive(Clone)] #[non_exhaustive] pub struct ClientConnection { @@ -874,7 +1018,7 @@ impl ClientConnection { .abort_handle(); let metrics = ClientConnectionMetrics::new(database_identity, config.protocol); - let receiver = ClientConnectionReceiver::new( + let mut receiver = ClientConnectionReceiver::new( config.confirmed_reads, MeteredReceiver::with_gauge(sendrx, metrics.sendtx_queue_size.clone()), module_rx.clone(), @@ -889,6 +1033,18 @@ impl ClientConnection { cancelled: AtomicBool::new(false), metrics: Some(metrics), }); + if sender.auth.hosted.is_some() { + receiver.hosted_sender = Some(Arc::downgrade(&sender)); + if module.subscriptions().register_hosted_connection(&sender).is_err() { + sender.cancel_hosted_connection(); + } else { + spawn_hosted_connection_watchdog( + Arc::downgrade(&sender), + module_rx.clone(), + Some(module.subscriptions().clone()), + ); + } + } let this = Self { sender, replica_id, @@ -998,7 +1154,7 @@ impl ClientConnection { self.module() .call_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), caller, Some(request_id), @@ -1019,7 +1175,7 @@ impl ClientConnection { ) -> Result { self.module() .call_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(self.sender()), Some(request_id), @@ -1045,7 +1201,7 @@ impl ClientConnection { self.module() .enqueue_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), caller, Some(request_id), @@ -1066,7 +1222,7 @@ impl ClientConnection { ) -> Result<(), ReducerCallError> { self.module() .enqueue_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(self.sender()), Some(request_id), @@ -1086,7 +1242,7 @@ impl ClientConnection { ) -> Result<(), BroadcastError> { self.module() .enqueue_procedure( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(timer), procedure, @@ -1106,7 +1262,7 @@ impl ClientConnection { ) -> Result<(), BroadcastError> { self.module() .enqueue_procedure( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(timer), procedure, @@ -1335,6 +1491,354 @@ mod tests { assert_matches!(futures::poll!(f), Poll::Pending); } + fn hosted_auth(db: &RelationalDB, lifetime: std::time::Duration) -> ConnectionAuthCtx { + // These fixtures model an already reconciled receiving host. Tests of + // startup closure explicitly close the gate after constructing it. + if !db.hosted_admission().is_open() { + db.hosted_admission().begin().unwrap().complete().unwrap(); + } + use crate::auth::{ + hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, + JwtKeys, + }; + let keys = JwtKeys::generate().unwrap(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + source_database: db.database_identity(), + target_database: db.database_identity(), + generation: 1, + grant_revision: 1, + lease_expires_at: now + std::time::Duration::from_secs(30), + }; + let token = sign_hosted_token(&keys.private, "platform.test", &binding, now, now + lifetime, "test").unwrap(); + HostedTokenValidator::new([("platform.test".into(), keys.public)]) + .unwrap() + .validate_token(&token, db.database_identity(), now, |_, _, _| Some(binding)) + .unwrap() + .into_connection_auth() + .unwrap() + } + + fn set_fence(db: &RelationalDB, generation: u64, allowed: bool) { + use crate::db::deployment::install_container_fence; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + db, + tx, + &StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: spacetimedb_lib::hash_bytes(b"targets"), + allowed, + }, + ) + }) + .unwrap(); + } + + fn hosted_client( + db: &RelationalDB, + supply: impl DurableOffsetSupply + 'static, + confirmed_reads: bool, + lifetime: std::time::Duration, + ) -> ( + Arc, + ClientConnectionReceiver, + tokio::task::JoinHandle<()>, + ) { + let (mut sender, mut receiver) = ClientConnectionSender::dummy_with_channel( + ClientActorId::for_test(db.database_identity()), + ClientConfig { + confirmed_reads, + ..ClientConfig::for_test() + }, + supply, + ); + sender.auth = hosted_auth(db, lifetime); + let actor = tokio::spawn(std::future::pending()); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + receiver.hosted_sender = Some(Arc::downgrade(&sender)); + (sender, receiver, actor) + } + + struct HostedConfirmedSupply { + db: Arc, + durable: FakeDurableOffset, + durability_requested: Arc, + } + impl DurableOffsetSupply for HostedConfirmedSupply { + fn durable_offset(&mut self) -> Result, NoSuchModule> { + self.durability_requested.store(true, Ordering::Release); + self.durable.durable_offset() + } + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + self.db.check_hosted_auth(proof) + } + } + + #[tokio::test] + async fn hosted_queued_delivery_rechecks_committed_fence_and_preserves_ordinary_identity() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, mut receiver, actor) = + hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + sender.send_message(None, empty_tx_update()).unwrap(); + set_fence(&db, 2, false); + assert_receiver_closed(receiver.recv()).await; + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + let (ordinary, mut ordinary_rx) = default_client(db.db.clone()); + ordinary.send_message(None, empty_tx_update()).unwrap(); + assert_received_update(ordinary_rx.recv()).await; + } + + #[tokio::test] + async fn hosted_queued_delivery_rejects_closed_startup_gate_with_unchanged_fence() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, mut receiver, actor) = + hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + sender.send_message(None, empty_tx_update()).unwrap(); + db.hosted_admission().close(); + assert_receiver_closed(receiver.recv()).await; + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + let (ordinary, mut ordinary_rx) = default_client(db.db.clone()); + ordinary.send_message(None, empty_tx_update()).unwrap(); + assert_received_update(ordinary_rx.recv()).await; + } + + #[tokio::test] + async fn hosted_confirmed_delivery_rechecks_fence_after_durability_wait() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let durable = FakeDurableOffset::new(); + let durability_requested = Arc::new(AtomicBool::new(false)); + let supply = HostedConfirmedSupply { + db: db.db.clone(), + durable: durable.clone(), + durability_requested: durability_requested.clone(), + }; + let (sender, mut receiver, actor) = hosted_client(&db, supply, true, std::time::Duration::from_secs(20)); + sender.send_message(Some(7), empty_tx_update()).unwrap(); + let mut receiving = Box::pin(receiver.recv()); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !durability_requested.load(Ordering::Acquire) { + assert_pending(&mut receiving).await; + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + set_fence(&db, 2, false); + durable.mark_durable_at(7); + assert_receiver_closed(receiving).await; + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_idle_connection_expires_without_outbound_traffic() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(2)); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), None); + tokio::time::timeout(std::time::Duration::from_secs(3), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_connection_uses_confirmed_expiry_while_authority_read_is_blocked() { + struct BlockedAuthority; + impl DurableOffsetSupply for BlockedAuthority { + fn durable_offset(&mut self) -> Result, NoSuchModule> { + Ok(None) + } + + fn check_hosted_auth( + &mut self, + _: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + Box::pin(std::future::pending()) + } + } + + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (mut sender, _receiver) = ClientConnectionSender::dummy_with_channel( + ClientActorId::for_test(db.database_identity()), + ClientConfig::for_test(), + db.db.clone(), + ); + let proof = hosted_auth(&db, std::time::Duration::from_secs(20)).hosted.unwrap(); + let confirmed_time = proof.expires_at() - std::time::Duration::from_secs(5); + // Authority is ahead of the receiving wall clock. Almost all of the + // five remaining seconds elapsed while its confirmation was in flight. + let started = std::time::Instant::now() - std::time::Duration::from_millis(4_900); + sender.auth = proof + .constrain_expiration(confirmed_time, started) + .unwrap() + .into_connection_auth() + .unwrap(); + let actor = tokio::spawn(std::future::pending::<()>()); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), BlockedAuthority, None); + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_idle_connection_rechecks_durable_revocation() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), None); + set_fence(&db, 2, false); + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_watchdog_ends_and_unregisters_after_socket_actor_finishes() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + subscriptions.register_hosted_connection(&sender).unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 1); + let watchdog = + spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), Some(subscriptions.clone())); + actor.abort(); + let _ = actor.await; + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 0); + // The registry releases the entry even while another owner retains sender. + assert!(!sender.is_cancelled()); + } + + #[tokio::test] + async fn hosted_registry_retains_cancelled_actor_until_delivery_cleanup_finishes() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (mut sender, _receiver) = default_client(db.db.clone()); + sender.auth = hosted_auth(&db, std::time::Duration::from_secs(20)); + let (release, blocked) = std::sync::mpsc::channel(); + let (started, ready) = oneshot::channel(); + // A started blocking task models cleanup that cannot complete merely + // because abort was requested. The barrier must retain its handle. + let actor = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + let _ = blocked.recv(); + }); + ready.await.unwrap(); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + subscriptions.register_hosted_connection(&sender).unwrap(); + let watchdog = + spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), Some(subscriptions.clone())); + set_fence(&db, 2, false); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !sender.is_cancelled() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 1); + let handles = db.with_read_only(Workload::ForTests, |tx| { + subscriptions.cancel_invalid_hosted_connections(tx) + }); + assert_eq!(handles.len(), 1); + assert!(!handles[0].is_finished()); + release.send(()).unwrap(); + actor.await.unwrap(); + watchdog.await.unwrap(); + assert!(handles[0].is_finished()); + assert_eq!(subscriptions.hosted_connection_count(), 0); + } + + #[tokio::test] + async fn hosted_target_barrier_cancels_connections_without_subscriptions_and_waits_for_actor() { + use crate::db::deployment::install_container_fence; + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + subscriptions.register_hosted_connection(&sender).unwrap(); + let handles = db + .with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + &db, + tx, + &StContainerFenceRow { + source_identity: db.database_identity().into(), + generation: 2, + target_grant_revision: 1, + target_set_hash: spacetimedb_lib::hash_bytes(b"targets"), + allowed: true, + }, + )?; + Ok::<_, anyhow::Error>(subscriptions.cancel_invalid_hosted_connections(tx)) + }) + .unwrap(); + assert_eq!(handles.len(), 1); + assert!(actor.await.unwrap_err().is_cancelled()); + assert!(handles[0].is_finished()); + assert!(subscriptions.register_hosted_connection(&sender).is_err()); + } + + #[tokio::test] + async fn hosted_subscription_rejected_under_transaction_before_query_compilation() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + // There is deliberately no installed fence. The malformed SQL verifies + // authentication fails before query parsing or view materialization. + let result = subscriptions + .add_legacy_subscriber( + None, + sender.clone(), + AuthCtx::new(db.database_identity(), db.database_identity()), + ws_v1::Subscribe { + query_strings: ["invalid SQL".into()].into(), + request_id: 0, + }, + Instant::now(), + None, + ) + .await; + assert!(result.unwrap_err().to_string().contains("container")); + sender.cancel_hosted_connection(); + let _ = actor.await; + } + fn default_client( offset_supply: impl DurableOffsetSupply + 'static, ) -> (ClientConnectionSender, ClientConnectionReceiver) { diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 7821f61f5d9..8fc03b44165 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -171,6 +171,7 @@ impl<'de> serde::Deserialize<'de> for ConfigFile { v8: V8Config { procedure_instance_pool_size: config.v8.procedure_instance_pool_size, heap_policy: config.v8_heap_policy, + execution_timeout: config.v8.execution_timeout, }, }) } @@ -247,6 +248,9 @@ impl Default for WasmConfigToml { pub struct V8Config { pub procedure_instance_pool_size: NonZeroUsize, pub heap_policy: V8HeapPolicyConfig, + /// Wall-clock limit for one JavaScript startup, description or function call. + /// Must be positive and no greater than 120 seconds. + pub execution_timeout: Duration, } impl Default for V8Config { @@ -254,11 +258,20 @@ impl Default for V8Config { Self { procedure_instance_pool_size: default_v8_procedure_instance_pool_size(), heap_policy: V8HeapPolicyConfig::default(), + execution_timeout: default_v8_execution_timeout(), } } } impl V8Config { + pub fn validate_execution_timeout(&self) -> anyhow::Result<()> { + anyhow::ensure!( + !self.execution_timeout.is_zero() && self.execution_timeout <= default_v8_execution_timeout(), + "V8 execution timeout must be positive and no greater than 120 seconds" + ); + Ok(()) + } + pub fn normalized(mut self) -> Self { self.heap_policy = self.heap_policy.normalized(); self @@ -273,16 +286,35 @@ struct V8ConfigToml { deserialize_with = "de_nz_usize" )] pub procedure_instance_pool_size: NonZeroUsize, + #[serde( + default = "default_v8_execution_timeout", + deserialize_with = "de_v8_execution_timeout" + )] + pub execution_timeout: Duration, } impl Default for V8ConfigToml { fn default() -> Self { Self { procedure_instance_pool_size: default_v8_procedure_instance_pool_size(), + execution_timeout: default_v8_execution_timeout(), } } } +fn default_v8_execution_timeout() -> Duration { + Duration::from_secs(120) +} + +fn de_v8_execution_timeout<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let timeout = de_nz_duration(deserializer)? + .filter(|timeout| *timeout <= default_v8_execution_timeout()) + .ok_or_else(|| { + serde::de::Error::custom("V8 execution timeout must be positive and no greater than 120 seconds") + })?; + Ok(timeout) +} + #[derive(Clone, Copy, Debug, serde::Deserialize)] #[serde(rename_all = "kebab-case")] pub struct V8HeapPolicyConfig { @@ -558,6 +590,25 @@ mod tests { .unwrap_err(); } + #[test] + fn v8_execution_timeout_is_finite_and_bounded() { + let defaults: ConfigFile = toml::from_str("").unwrap(); + assert_eq!(defaults.v8.execution_timeout, Duration::from_secs(120)); + for value in ["0", "121", "\"0s\"", "\"121s\""] { + assert!(toml::from_str::(&format!("[v8]\nexecution-timeout = {value}")).is_err()); + } + let config: ConfigFile = toml::from_str("[v8]\nexecution-timeout = \"150ms\"").unwrap(); + assert_eq!(config.v8.execution_timeout, Duration::from_millis(150)); + for timeout in [Duration::ZERO, Duration::from_secs(121)] { + assert!(V8Config { + execution_timeout: timeout, + ..V8Config::default() + } + .validate_execution_timeout() + .is_err()); + } + } + #[test] fn v8_heap_policy_defaults_when_omitted() { let config: ConfigFile = toml::from_str("").unwrap(); diff --git a/crates/core/src/db/container_environment.rs b/crates/core/src/db/container_environment.rs new file mode 100644 index 00000000000..def8e8569ed --- /dev/null +++ b/crates/core/src/db/container_environment.rs @@ -0,0 +1,354 @@ +//! Transactional immutable container environment snapshots. +//! +//! These are host-only operations. Before invoking them, the adapter must +//! authenticate the assigned service and confirm the exact current control +//! intent and authoritative leader. Identity equality and a cached control row +//! are insufficient. Helpers retain the caller's transaction and do no IO. +//! +//! Historical restore must close admission and reconcile newer operational +//! fences before any snapshot request. A missing Ready snapshot is an error, +//! never permission to recapture values from a restored or current `st_env`. + +use super::{ + deployment, environment, + relational_db::{MutTx, RelationalDB}, +}; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + StContainerEnvironmentRow, StContainerFenceRow, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, +}; +use spacetimedb_lib::container::{validate_env_key, validate_exec_size, ContainerSpec, MAX_ENV_KEYS}; +use spacetimedb_lib::container_environment::{EnvironmentSnapshotReceipt, EnvironmentSnapshotScope}; +use spacetimedb_lib::{bsatn, SpacetimeType, Uuid}; +use spacetimedb_primitives::ColId; +use spacetimedb_sats::AlgebraicValue; +use std::{collections::BTreeMap, fmt}; + +pub const MAX_SNAPSHOT_BYTES: usize = 256 * 1024; +pub const MAX_RETAINED_SNAPSHOTS: u64 = 64; + +/// Error text and Debug never include selected values or serialized records. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum EnvironmentSnapshotError { + #[error("container environment request is invalid")] + InvalidScope, + #[error("container environment generation is fenced")] + Fenced, + #[error("container environment deployment revision does not match")] + RevisionConflict, + #[error("container environment snapshot belongs to another immutable scope")] + ScopeConflict, + #[error("container environment snapshot has not been captured")] + NotCaptured, + #[error("required container environment keys are missing: {0:?}")] + MissingKeys(Vec), + #[error("container environment does not satisfy startup limits")] + InvalidEnvironment, + #[error("container environment snapshot capacity exhausted")] + Capacity, + #[error("container environment snapshot metadata is invalid")] + CorruptMetadata, + #[error("container environment storage operation failed")] + Storage, + #[error("container environment requires durable storage")] + DurabilityUnavailable, + #[error("container environment durability could not be confirmed")] + DurabilityFailed, +} + +impl From for EnvironmentSnapshotError { + fn from(_: crate::error::DBError) -> Self { + Self::Storage + } +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +enum Record { + V1(RecordV1), +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct RecordV1 { + receipt: EnvironmentSnapshotReceipt, + selected_values: Vec, +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct CapturedValue { + key: String, + value: String, +} + +/// Selected database values only. The trusted adapter merges verified image +/// defaults and current platform variables, then validates the complete exec. +pub struct SecretEnvironment { + pub receipt: EnvironmentSnapshotReceipt, + pub selected_values: BTreeMap, +} + +impl fmt::Debug for SecretEnvironment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecretEnvironment") + .field("receipt", &self.receipt) + .field("selected_values", &"[redacted]") + .finish() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnvironmentClosedReceipt { + pub scope: EnvironmentSnapshotScope, + pub closed_through_generation: u64, +} + +fn valid_uuid(id: Uuid) -> bool { + matches!( + id.get_version(), + Some(spacetimedb_sats::uuid::Version::V4 | spacetimedb_sats::uuid::Version::V7) + ) +} + +fn validate_scope(db: &RelationalDB, scope: &EnvironmentSnapshotScope) -> Result<(), EnvironmentSnapshotError> { + if db.database_identity() != scope.database_identity + || scope.database_id == 0 + || scope.node_id == 0 + || scope.generation == 0 + || scope.cluster.is_empty() + || scope.cluster.len() > 256 + || scope.cluster.contains('\0') + || !valid_uuid(scope.node_incarnation) + || !valid_uuid(scope.start_request) + || !valid_uuid(scope.env_generation) + || scope.env_keys.len() > MAX_ENV_KEYS + || scope.env_keys.windows(2).any(|keys| keys[0] >= keys[1]) + || scope.env_keys.iter().any(|key| validate_env_key(key).is_err()) + { + return Err(EnvironmentSnapshotError::InvalidScope); + } + Ok(()) +} + +fn fence( + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result { + state + .iter_by_col_eq( + ST_CONTAINER_FENCE_ID, + ColId(0), + &AlgebraicValue::U256(scope.database_identity.to_u256().into()), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .map(StContainerFenceRow::try_from) + .transpose() + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::Fenced) +} + +fn admitted_spec( + db: &RelationalDB, + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result { + validate_scope(db, scope)?; + let current = fence(state, scope)?; + if current.generation != scope.generation || !current.allowed { + return Err(EnvironmentSnapshotError::Fenced); + } + let (revision, deployment) = deployment::current_deployment(state) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if revision != scope.deployment_revision { + return Err(EnvironmentSnapshotError::RevisionConflict); + } + let spec = deployment + .current() + .container + .as_ref() + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if spec.env_keys != scope.env_keys { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + Ok(spec.clone()) +} + +fn lookup( + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result, EnvironmentSnapshotError> { + let Some(row) = state + .iter_by_col_eq( + ST_CONTAINER_ENVIRONMENT_ID, + ColId(0), + &AlgebraicValue::U64(scope.generation), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .map(StContainerEnvironmentRow::try_from) + .transpose() + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + else { + return Ok(None); + }; + if row.payload.len() > MAX_SNAPSHOT_BYTES { + return Err(EnvironmentSnapshotError::CorruptMetadata); + } + let Record::V1(record) = bsatn::from_slice(&row.payload).map_err(|_| EnvironmentSnapshotError::CorruptMetadata)?; + if record.receipt.scope != *scope { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + if !valid_uuid(record.receipt.capture_receipt) + || !record + .selected_values + .iter() + .map(|entry| &entry.key) + .eq(scope.env_keys.iter()) + { + return Err(EnvironmentSnapshotError::CorruptMetadata); + } + Ok(Some(record)) +} + +fn validate_values(spec: &ContainerSpec, values: &BTreeMap) -> Result<(), EnvironmentSnapshotError> { + // The at-most-256 individually bounded values also bound this temporary allocation. + if values + .values() + .any(|value| value.contains('\0') || spacetimedb_lib::environment::validate_value(value).is_err()) + { + return Err(EnvironmentSnapshotError::InvalidEnvironment); + } + let env = values + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>(); + validate_exec_size(&spec.argv, &env).map_err(|_| EnvironmentSnapshotError::InvalidEnvironment) +} + +/// Capture exactly once under an open confirmed control intent. Returning this +/// receipt is not a durability acknowledgment; the host wrapper supplies that. +pub fn capture( + db: &RelationalDB, + tx: &mut MutTx, + scope: &EnvironmentSnapshotScope, +) -> Result { + let spec = admitted_spec(db, tx, scope)?; + if let Some(record) = lookup(tx, scope)? { + validate_values( + &spec, + &record + .selected_values + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect(), + )?; + return Ok(record.receipt); + } + if tx + .table_row_count(ST_CONTAINER_ENVIRONMENT_ID) + .ok_or(EnvironmentSnapshotError::Storage)? + >= MAX_RETAINED_SNAPSHOTS + { + return Err(EnvironmentSnapshotError::Capacity); + } + let mut values = BTreeMap::new(); + let mut missing = Vec::new(); + for key in &scope.env_keys { + match environment::get(tx, key).map_err(|_| EnvironmentSnapshotError::Storage)? { + Some(value) => { + values.insert(key.clone(), value); + } + None => missing.push(key.clone()), + } + } + if !missing.is_empty() { + return Err(EnvironmentSnapshotError::MissingKeys(missing)); + } + validate_values(&spec, &values)?; + let receipt = EnvironmentSnapshotReceipt { + scope: scope.clone(), + capture_receipt: Uuid::from_u128(uuid::Uuid::new_v4().as_u128()), + }; + let payload = bsatn::to_vec(&Record::V1(RecordV1 { + receipt: receipt.clone(), + selected_values: values + .into_iter() + .map(|(key, value)| CapturedValue { key, value }) + .collect(), + })) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)?; + if payload.len() > MAX_SNAPSHOT_BYTES { + return Err(EnvironmentSnapshotError::Capacity); + } + tx.insert_via_serialize_bsatn( + ST_CONTAINER_ENVIRONMENT_ID, + &StContainerEnvironmentRow { + generation: scope.generation, + payload: payload.into(), + }, + ) + .map_err(|_| EnvironmentSnapshotError::Storage)?; + Ok(receipt) +} + +/// Read only an existing receipt after fresh exact control/lease confirmation. +pub fn read( + db: &RelationalDB, + state: &impl StateView, + receipt: &EnvironmentSnapshotReceipt, +) -> Result { + let spec = admitted_spec(db, state, &receipt.scope)?; + let record = lookup(state, &receipt.scope)?.ok_or(EnvironmentSnapshotError::NotCaptured)?; + if record.receipt != *receipt { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + let selected_values = record + .selected_values + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect(); + validate_values(&spec, &selected_values)?; + Ok(SecretEnvironment { + receipt: record.receipt, + selected_values, + }) +} + +/// Delete only after a newer own-source fence irreversibly rejects every old +/// capture. The adapter also confirms positive historical control closure. +/// Missing rows remain idempotently closed; no per-UUID TTL is an authority. +pub fn close( + db: &RelationalDB, + tx: &mut MutTx, + scope: &EnvironmentSnapshotScope, + closed_through_generation: u64, +) -> Result { + validate_scope(db, scope)?; + let current = fence(tx, scope)?; + if closed_through_generation <= scope.generation || current.generation < closed_through_generation { + return Err(EnvironmentSnapshotError::Fenced); + } + if lookup(tx, scope)?.is_some() { + let pointer = tx + .iter_by_col_eq( + ST_CONTAINER_ENVIRONMENT_ID, + ColId(0), + &AlgebraicValue::U64(scope.generation), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .ok_or(EnvironmentSnapshotError::Storage)? + .pointer(); + db.delete(tx, ST_CONTAINER_ENVIRONMENT_ID, [pointer]); + } + Ok(EnvironmentClosedReceipt { + scope: scope.clone(), + closed_through_generation, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/db/container_environment/tests.rs b/crates/core/src/db/container_environment/tests.rs new file mode 100644 index 00000000000..45a91d21cfe --- /dev/null +++ b/crates/core/src/db/container_environment/tests.rs @@ -0,0 +1,427 @@ +use super::*; +use crate::db::deployment::{ + install_container_fence, install_publication_fence, record_deployment_commit, DeploymentCommit, +}; +use crate::db::relational_db::tests_utils::TestDB; +use crate::host::container_environment as host; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_lib::container::*; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; +use spacetimedb_lib::{hash_bytes, Identity, Timestamp}; +use std::sync::{Arc, Barrier}; + +fn uuid() -> Uuid { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) +} + +fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: keys, + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } + .normalize(&Default::default()) + .unwrap(); + let request = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: db.owner_identity(), + expected_revision: None, + prepared_manifest_hash: hash_bytes(b"prepared"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: Some(spec.clone()), + }), + }; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + record_deployment_commit(tx, &request, Timestamp::now(), &Default::default())?; + install_container_fence(db, tx, &self_fence(db, 1, true))?; + for key in &spec.env_keys { + environment::set(db, tx, key, "before")?; + } + Ok(()) + }) + .unwrap(); + EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: 1, + database_identity: db.database_identity(), + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: request.deployment.revision().unwrap(), + start_request: request.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + } +} + +fn self_fence(db: &RelationalDB, generation: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed, + } +} + +fn tx( + db: &RelationalDB, + action: impl FnOnce(&mut MutTx) -> Result, +) -> Result { + db.with_auto_commit(Workload::ForTests, action) +} + +#[test] +fn container_environment_capture_retry_read_and_durable_reopen() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + let captured = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), scope.clone())) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "A", "changed")?; + environment::delete(&db, tx, "B")?; + Ok(()) + }) + .unwrap(); + let db = db.reopen().unwrap(); + let retried = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), scope)) + .unwrap(); + assert_eq!(retried.receipt, captured.receipt); + assert!(retried.durable_through >= captured.durable_through); + let values = db + .runtime() + .unwrap() + .block_on(host::read(db.db.clone(), retried.receipt)) + .unwrap() + .receipt; + assert_eq!( + values.selected_values, + BTreeMap::from([("A".into(), "before".into()), ("B".into(), "before".into())]) + ); + assert!(!format!("{values:?}").contains("before")); +} + +#[test] +fn container_environment_capture_is_atomic_against_concurrent_environment_mutation() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + let barrier = Arc::new(Barrier::new(2)); + let writer = { + let db = db.db.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "A", "after")?; + environment::set(&db, tx, "B", "after")?; + Ok(()) + }) + .unwrap(); + }) + }; + barrier.wait(); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + writer.join().unwrap(); + let values = db + .with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap(); + assert_eq!(values.selected_values["A"], values.selected_values["B"]); +} + +#[test] +fn container_environment_closure_fences_delayed_capture_and_new_instance_observes_new_values() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let old = setup(&db, vec!["A".into()]); + let captured = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), old.clone())) + .unwrap() + .receipt; + assert_eq!( + tx(&db, |tx| close(&db, tx, &old, 1)), + Err(EnvironmentSnapshotError::Fenced) + ); + assert_eq!( + tx(&db, |tx| close(&db, tx, &old, 2)), + Err(EnvironmentSnapshotError::Fenced) + ); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 2, true))?; + environment::set(&db, tx, "A", "new-boot")?; + Ok(()) + }) + .unwrap(); + db.runtime() + .unwrap() + .block_on(host::close(db.db.clone(), old.clone(), 2)) + .unwrap(); + let db = db.reopen().unwrap(); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &old)), + Err(EnvironmentSnapshotError::Fenced) + ); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &captured)), + Err(EnvironmentSnapshotError::Fenced) + )); + db.runtime() + .unwrap() + .block_on(host::close(db.db.clone(), old.clone(), 2)) + .unwrap(); + let new = EnvironmentSnapshotScope { + generation: 2, + env_generation: uuid(), + ..old + }; + let receipt = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), new)) + .unwrap() + .receipt; + assert_ne!(receipt.capture_receipt, captured.capture_receipt); + assert_eq!( + db.runtime() + .unwrap() + .block_on(host::read(db.db.clone(), receipt)) + .unwrap() + .receipt + .selected_values["A"], + "new-boot" + ); +} + +#[test] +fn container_environment_full_scope_conflicts_and_missing_ready_record_fail_closed() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let mut variants = Vec::new(); + let mut changed = scope.clone(); + changed.cluster = "other".into(); + variants.push(changed); + let mut changed = scope.clone(); + changed.database_id += 1; + variants.push(changed); + let mut changed = scope.clone(); + changed.node_id += 1; + variants.push(changed); + let mut changed = scope.clone(); + changed.node_incarnation = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.start_request = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.env_generation = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.env_keys.clear(); + variants.push(changed); + for changed in variants { + assert_eq!( + tx(&db, |tx| capture(&db, tx, &changed)), + Err(EnvironmentSnapshotError::ScopeConflict) + ); + } + let mut changed = scope.clone(); + changed.deployment_revision = hash_bytes(b"other"); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &changed)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + let mut fork = scope; + fork.database_identity = Identity::from_u256(99u64.into()); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &fork)), + Err(EnvironmentSnapshotError::InvalidScope) + ); + // Simulate a restored/lost Ready record. Resolve may not reinterpret its UUID. + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + tx.clear_table(ST_CONTAINER_ENVIRONMENT_ID)?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)), + Err(EnvironmentSnapshotError::NotCaptured) + )); +} + +#[test] +fn container_environment_missing_empty_invalid_and_capacity_are_atomic_and_redacted() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::delete(&db, tx, "B")?; + Ok(()) + }) + .unwrap(); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::MissingKeys(vec!["B".into()])) + ); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "B", "secret\0value")?; + Ok(()) + }) + .unwrap(); + let failure = tx(&db, |tx| capture(&db, tx, &scope)).unwrap_err(); + assert_eq!(failure, EnvironmentSnapshotError::InvalidEnvironment); + assert!(!format!("{failure:?}: {failure}").contains("secret")); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "B", "")?; + Ok(()) + }) + .unwrap(); + for generation in 1..=MAX_RETAINED_SNAPSHOTS { + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, generation, true))?; + Ok(()) + }) + .unwrap(); + let request = EnvironmentSnapshotScope { + generation, + env_generation: uuid(), + ..scope.clone() + }; + let receipt = tx(&db, |tx| capture(&db, tx, &request)).unwrap(); + assert_eq!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap() + .selected_values["B"], + "" + ); + } + let generation = MAX_RETAINED_SNAPSHOTS + 1; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, generation, true))?; + Ok(()) + }) + .unwrap(); + assert_eq!( + tx(&db, |tx| capture( + &db, + tx, + &EnvironmentSnapshotScope { generation, ..scope } + )), + Err(EnvironmentSnapshotError::Capacity) + ); +} + +#[test] +fn container_environment_history_is_hidden_from_privileged_sql_and_subscriptions() { + use crate::sql::ast::SchemaViewer; + use spacetimedb_expr::check::SchemaView; + use spacetimedb_lib::identity::AuthCtx; + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let auth = AuthCtx::for_current(db.owner_identity()); + db.with_read_only(Workload::ForTests, |state| { + let schema = SchemaViewer::new(state, &auth); + assert!(schema.schema_for_table(ST_CONTAINER_ENVIRONMENT_ID).is_none()); + assert!(schema.table_id("st_container_environment").is_none()); + for sql in [ + "SELECT * FROM st_container_environment", + "SELECT h.* FROM st_container_environment h JOIN st_env e ON h.generation = 1", + "DELETE FROM st_container_environment", + "UPDATE st_container_environment SET generation = 2", + ] { + assert!( + spacetimedb_query::compile_sql_stmt(sql, &schema, &auth).is_err(), + "{sql}" + ); + } + assert!(spacetimedb_query::compile_sql_stmt("SELECT * FROM st_env", &schema, &auth).is_ok()); + assert!(crate::subscription::query::compile_read_only_query( + &auth, + state, + "SELECT * FROM st_container_environment" + ) + .is_err()); + assert!(crate::subscription::subscription::get_all( + |db, tx| db.get_all_tables(tx).map(Vec::into_iter), + &db, + state, + &auth + ) + .unwrap() + .is_empty()); + }); +} + +#[test] +fn container_environment_host_api_requires_durability() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec![]); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + assert!(matches!( + rt.block_on(host::capture(db.db.clone(), scope)), + Err(EnvironmentSnapshotError::DurabilityUnavailable) + )); +} + +#[test] +fn container_environment_concurrent_closure_cannot_reopen_collected_generation() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + let barrier = Arc::new(Barrier::new(2)); + let capture_thread = { + let db = db.db.clone(); + let scope = scope.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + tx(&db, |tx| capture(&db, tx, &scope)) + }) + }; + barrier.wait(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 2, false))?; + close(&db, tx, &scope, 2)?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + capture_thread.join().unwrap(), + Ok(_) | Err(EnvironmentSnapshotError::Fenced) + )); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::Fenced) + ); + db.with_read_only(Workload::ForTests, |state| { + assert_eq!(state.table_row_count(ST_CONTAINER_ENVIRONMENT_ID), Some(0)) + }); +} diff --git a/crates/core/src/db/deployment.rs b/crates/core/src/db/deployment.rs new file mode 100644 index 00000000000..8ee66d385c8 --- /dev/null +++ b/crates/core/src/db/deployment.rs @@ -0,0 +1,586 @@ +//! Transactional deployment and hosted-client fences. +//! +//! Only authenticated host operations may call the mutation functions here. +//! Callers retain the same serializable transaction through module migration, +//! deployment recording, and commit. These functions do not contact control, +//! perform process IO, or turn an unverified client Identity into host authority. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + ConnectionIdViaU128, StConnectionAuthRow, StContainerFenceRow, StDeploymentOperationRow, StDeploymentRow, + StPublishFenceRow, ST_CONNECTION_AUTH_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, ST_DEPLOYMENT_OPERATION_ID, + ST_PUBLISH_FENCE_ID, +}; +use spacetimedb_lib::container::ContainerSpecLimits; +use spacetimedb_lib::deployment::{operation_expiry_ms, DeploymentSpec, DeploymentValidationError}; +use spacetimedb_lib::{bsatn, hash_bytes, ConnectionId, Hash, Identity, SpacetimeType, Timestamp, Uuid}; +use spacetimedb_primitives::{ColId, TableId}; +use spacetimedb_sats::AlgebraicValue; + +#[derive(Debug, thiserror::Error)] +pub enum DeploymentError { + #[error("this database requires the deployment publication protocol")] + CoordinatorRequired, + #[error("the supplied module does not match the prepared deployment")] + ProgramMismatch, + #[error("the prepared module must advertise hosted_auth_v1 to attach a container")] + UnsupportedHostedModule, + #[error("the publication coordinator no longer owns the database fence")] + PublicationFenced, + #[error("the expected deployment revision does not match the database")] + RevisionConflict, + #[error("operation ID is already bound to a different publication")] + OperationConflict, + #[error("container generation or grant is not authorized by this database")] + ContainerFenced, + #[error("a conflicting or older container fence cannot replace current authority")] + FenceConflict, + #[error("the receiving host fence revision is exhausted")] + FenceRevisionExhausted, + #[error("deployment metadata is inconsistent")] + CorruptMetadata, + #[error(transparent)] + Validation(#[from] DeploymentValidationError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Prepared by the authorized coordinator after recording durable intent. +/// The publisher is its verified original caller, never a guest-selected claim. +#[derive(Clone, Debug)] +pub struct DeploymentCommit { + pub operation_id: Uuid, + pub publication_epoch: u64, + pub publisher: Identity, + pub expected_revision: Option, + pub prepared_manifest_hash: Hash, + pub deployment: DeploymentSpec, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishResult { + #[serde(with = "spacetimedb_lib::deployment::uuid_json")] + pub operation_id: Uuid, + pub previous_revision: Option, + pub revision: Hash, +} + +#[derive(Clone, Debug, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +enum CommitReceipt { + V1(CommitReceiptV1), +} + +#[derive(Clone, Debug, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct CommitReceiptV1 { + request_hash: Hash, + publisher: Identity, + result: PublishResult, +} + +#[derive(Clone, Debug)] +pub enum CommitAdmission { + /// Return this result without repeating module init/update or any effects. + AlreadyCommitted(PublishResult), + Ready, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AbortResult { + /// The commit won the race. Recovery must converge on this deployment. + AlreadyCommitted(PublishResult), + /// This epoch can no longer admit a commit, and the prior revision remains. + Aborted { previous_revision: Option }, +} + +/// Check the actual program selected by the host, not a caller-provided +/// capability bit. The program bytes are hashed here because `Program` also +/// has a public constructor which accepts a previously computed hash. +pub fn validate_deployment_program( + request: &DeploymentCommit, + program: &spacetimedb_datastore::traits::Program, + module: &spacetimedb_schema::def::ModuleDef, +) -> Result<(), DeploymentError> { + use spacetimedb_datastore::system_tables::ModuleKind; + use spacetimedb_lib::deployment::{ModuleComponent, UserModuleKind}; + if hash_bytes(&program.bytes) != program.hash { + return Err(DeploymentError::ProgramMismatch); + } + match &request.deployment.current().module { + ModuleComponent::User(expected) + if expected.program_hash == program.hash + && matches!( + (expected.kind, program.kind), + (UserModuleKind::Wasm, ModuleKind::WASM) | (UserModuleKind::Js, ModuleKind::JS) + ) => {} + ModuleComponent::SystemEmpty(version) if crate::host::empty_module::matches_program(*version, program) => {} + _ => return Err(DeploymentError::ProgramMismatch), + } + if request.deployment.current().container.is_some() && !module.supports_hosted_auth_v1() { + return Err(DeploymentError::UnsupportedHostedModule); + } + Ok(()) +} + +/// Legacy raw module publication must be routed through the coordinator once +/// a database has deployment metadata or a publication fence, including after +/// an attempt aborts or its container is removed. In particular a legacy +/// request cannot race the first prepared container publication. +/// Call while holding the transaction that changes the program/schema. +pub fn require_unmanaged_publication(tx: &MutTx) -> Result<(), DeploymentError> { + if current_deployment(tx)?.is_some() || singleton(tx, ST_PUBLISH_FENCE_ID)?.is_some() { + return Err(DeploymentError::CoordinatorRequired); + } + Ok(()) +} + +/// Foreign callers depend on the receiving module's bindings just as self +/// callers do. A module replacement cannot silently erase that capability +/// while an admitted generation can still address the database. +pub fn validate_active_hosted_grants( + tx: &MutTx, + module: &spacetimedb_schema::def::ModuleDef, +) -> Result<(), DeploymentError> { + if !module.supports_hosted_auth_v1() { + for row in tx.iter(ST_CONTAINER_FENCE_ID)? { + if StContainerFenceRow::try_from(row)?.allowed { + return Err(DeploymentError::UnsupportedHostedModule); + } + } + } + Ok(()) +} + +fn singleton( + state: &S, + table: TableId, +) -> Result>, DeploymentError> { + Ok(state.iter_by_col_eq(table, ColId(0), &AlgebraicValue::U8(0))?.next()) +} + +pub fn current_deployment(state: &S) -> Result, DeploymentError> { + let Some(row) = singleton(state, ST_DEPLOYMENT_ID)? else { + return Ok(None); + }; + let row = StDeploymentRow::try_from(row)?; + let spec = DeploymentSpec::decode(&row.payload)?; + if spec.revision()? != row.revision { + return Err(DeploymentError::CorruptMetadata); + } + Ok(Some((row.revision, spec))) +} + +/// Monotonic compare-and-set, serialized against user-database commits. +/// Advancing this epoch does not itself quiesce a container or authorize launch. +pub fn install_publication_fence( + tx: &mut MutTx, + publication_epoch: u64, + operation_id: Uuid, +) -> Result<(), DeploymentError> { + if publication_epoch == 0 || operation_id == Uuid::NIL { + return Err(DeploymentError::PublicationFenced); + } + let next = StPublishFenceRow { + key: 0, + publication_epoch, + operation_id: operation_id.as_u128(), + }; + if let Some(current) = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()? + { + if current == next { + return Ok(()); + } + if current.publication_epoch >= publication_epoch { + return Err(DeploymentError::PublicationFenced); + } + } + tx.clear_table(ST_PUBLISH_FENCE_ID)?; + tx.insert_via_serialize_bsatn(ST_PUBLISH_FENCE_ID, &next)?; + Ok(()) +} + +/// Close an admitted publication before reporting an abort to control. Run in +/// one serializable transaction and await its durability before releasing the +/// control operation or resuming the previous container under a fresh generation. +/// A delayed commit and this transaction serialize on the same database fence. +/// +/// The nil operation ID is a closed-epoch marker, never a publish operation. +/// Keeping the epoch makes closure irreversible at that epoch while allowing +/// the next control-allocated epoch to install its own operation normally. +pub fn abort_deployment_commit(tx: &mut MutTx, request: &DeploymentCommit) -> Result { + if let Some(result) = committed_deployment_operation(tx, request)? { + return Ok(AbortResult::AlreadyCommitted(result)); + } + // Recovery must be able to close an expired operation too. Its expiry + // prevents new commits, but cannot substitute for a durable abort fence. + // Control owns the immutable epoch-to-operation mapping; authenticate and + // resolve that recorded operation before calling this function. + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + let fence = fence + .filter(|row| { + row.publication_epoch == request.publication_epoch + && (row.operation_id == request.operation_id.as_u128() || row.operation_id == 0) + }) + .ok_or(DeploymentError::PublicationFenced)?; + if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + if fence.operation_id == 0 { + return Ok(AbortResult::Aborted { + previous_revision: request.expected_revision, + }); + } + tx.clear_table(ST_PUBLISH_FENCE_ID)?; + tx.insert_via_serialize_bsatn( + ST_PUBLISH_FENCE_ID, + &StPublishFenceRow { + key: 0, + publication_epoch: request.publication_epoch, + operation_id: 0, + }, + )?; + Ok(AbortResult::Aborted { + previous_revision: request.expected_revision, + }) +} + +fn normalized_request( + request: &DeploymentCommit, + limits: &ContainerSpecLimits, +) -> Result<(DeploymentSpec, Hash, Hash), DeploymentError> { + let spec = request.deployment.clone().normalize(limits)?; + if spec != request.deployment { + return Err(DeploymentValidationError::InvalidEncoding.into()); + } + let (revision, request_hash) = request_identity(request)?; + Ok((spec, revision, request_hash)) +} + +fn request_identity(request: &DeploymentCommit) -> Result<(Hash, Hash), DeploymentError> { + if request.operation_id.get_version() != Some(spacetimedb_sats::uuid::Version::V7) { + return Err(DeploymentValidationError::InvalidOperationId.into()); + } + if request.publication_epoch == 0 { + return Err(DeploymentError::PublicationFenced); + } + // These bytes were normalized at admission. Do not re-apply today's + // resource eligibility when inspecting yesterday's committed outcome. + let revision = request.deployment.revision()?; + let mut bytes = b"spacetimedb/deployment-operation\0".to_vec(); + bytes.extend_from_slice( + &bsatn::to_vec(&( + request.operation_id, + request.publication_epoch, + request.publisher, + request.expected_revision, + request.prepared_manifest_hash, + revision, + )) + .map_err(|_| DeploymentError::CorruptMetadata)?, + ); + Ok((revision, hash_bytes(bytes))) +} + +/// Call before module execution, while holding the transaction later used for +/// migration. A cache or pre-enqueue check cannot replace this admission check. +pub fn check_deployment_commit( + tx: &MutTx, + request: &DeploymentCommit, + now: Timestamp, + limits: &ContainerSpecLimits, +) -> Result { + let now_ms = u64::try_from(now.to_micros_since_unix_epoch()).map_err(|_| DeploymentError::CorruptMetadata)? / 1000; + operation_expiry_ms(request.operation_id, now_ms)?; + if let Some(result) = committed_deployment_operation(tx, request)? { + return Ok(CommitAdmission::AlreadyCommitted(result)); + } + normalized_request(request, limits)?; + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + if !fence.is_some_and(|f| { + f.publication_epoch == request.publication_epoch && f.operation_id == request.operation_id.as_u128() + }) { + return Err(DeploymentError::PublicationFenced); + } + if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + Ok(CommitAdmission::Ready) +} + +/// Inspect the exact retained commit receipt during host recovery. Unlike +/// admitting a client retry, inspecting an existing outcome does not expire. +/// This never authorizes module execution. An absent receipt is not proof of +/// abort: close the epoch atomically before reporting an abort to control. +/// Retain active operations' receipts until their control recovery completes. +pub fn committed_deployment_operation( + state: &S, + request: &DeploymentCommit, +) -> Result, DeploymentError> { + let (revision, request_hash) = request_identity(request)?; + let operation_key = AlgebraicValue::U128(request.operation_id.as_u128().into()); + if let Some(row) = state + .iter_by_col_eq(ST_DEPLOYMENT_OPERATION_ID, ColId(0), &operation_key)? + .next() + { + let row = StDeploymentOperationRow::try_from(row)?; + let CommitReceipt::V1(receipt) = + bsatn::from_slice(&row.commit_result).map_err(|_| DeploymentError::CorruptMetadata)?; + if receipt.request_hash != request_hash || receipt.publisher != request.publisher { + return Err(DeploymentError::OperationConflict); + } + if receipt.result.revision != revision + || row.committed_revision != revision + || row.previous_revision != receipt.result.previous_revision + || receipt.result.operation_id != request.operation_id + { + return Err(DeploymentError::CorruptMetadata); + } + return Ok(Some(receipt.result)); + } + Ok(None) +} + +/// Recognize the durable closed marker when recovering a control operation +/// whose abort report was lost. The authenticated caller must resolve control's +/// immutable epoch-to-operation binding before using this host-only API. +pub fn deployment_publication_aborted( + state: &S, + request: &DeploymentCommit, +) -> Result { + request_identity(request)?; + let fence = singleton(state, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + if !fence.is_some_and(|row| row.publication_epoch == request.publication_epoch && row.operation_id == 0) { + return Ok(false); + } + if current_deployment(state)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + Ok(true) +} + +/// Record after successful module initialization/migration in that same +/// transaction. An error must roll back the entire transaction, including the +/// module changes. The caller waits for durability before reporting acceptance. +pub fn record_deployment_commit( + tx: &mut MutTx, + request: &DeploymentCommit, + now: Timestamp, + limits: &ContainerSpecLimits, +) -> Result { + if let CommitAdmission::AlreadyCommitted(result) = check_deployment_commit(tx, request, now, limits)? { + return Ok(result); + } + let (spec, revision, request_hash) = normalized_request(request, limits)?; + let result = PublishResult { + operation_id: request.operation_id, + previous_revision: request.expected_revision, + revision, + }; + let receipt = CommitReceipt::V1(CommitReceiptV1 { + request_hash, + publisher: request.publisher, + result: result.clone(), + }); + let receipt = bsatn::to_vec(&receipt) + .map_err(|_| DeploymentError::CorruptMetadata)? + .into_boxed_slice(); + let expires_ms = operation_expiry_ms(request.operation_id, (now.to_micros_since_unix_epoch() / 1000) as u64)?; + let expires_us = i64::try_from(expires_ms * 1000).map_err(|_| DeploymentError::CorruptMetadata)?; + let row = StDeploymentRow { + key: 0, + revision, + last_operation_id: request.operation_id.as_u128(), + payload: spec.encode()?, + }; + tx.clear_table(ST_DEPLOYMENT_ID)?; + tx.insert_via_serialize_bsatn(ST_DEPLOYMENT_ID, &row)?; + tx.insert_via_serialize_bsatn( + ST_DEPLOYMENT_OPERATION_ID, + &StDeploymentOperationRow { + operation_id: request.operation_id.as_u128(), + previous_revision: request.expected_revision, + committed_revision: revision, + commit_result: receipt, + expires_at: Timestamp::from_micros_since_unix_epoch(expires_us).into(), + }, + )?; + Ok(result) +} + +/// Install the frozen generation/grant tuple. Changes at the same generation +/// cannot reopen a revoked grant, even if a stale coordinator changes only the +/// target-set hash. Control must allocate a new generation for every barrier. +pub fn install_container_fence( + db: &RelationalDB, + tx: &mut MutTx, + next: &StContainerFenceRow, +) -> Result<(), DeploymentError> { + if next.generation == 0 { + return Err(DeploymentError::FenceConflict); + } + let key: AlgebraicValue = next.source_identity.into(); + let previous = tx + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(|row| StContainerFenceRow::try_from(row).map(|value| (row.pointer(), value))) + .transpose()?; + if let Some((pointer, previous)) = previous { + if previous == *next { + return Ok(()); + } + if next.generation <= previous.generation || next.target_grant_revision < previous.target_grant_revision { + return Err(DeploymentError::FenceConflict); + } + db.hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + } else { + db.hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + } + tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, next)?; + Ok(()) +} + +/// An exact denial of a previously observed allowed fence. Revision counters +/// belong to this live database open, not to the durable generation namespace. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FenceDenial { + pub row: StContainerFenceRow, + pub revision_before: u64, + pub revision_after: u64, +} + +/// Deny an orphan without lowering or inventing a control generation. +/// +/// The trusted coordinator must establish the source's absence from current +/// control inventory, then recheck that inventory revision while holding this +/// same serializable transaction. This function does not confer authority to +/// any external caller. Only the exact allowed tuple, or its already-denied +/// form, is accepted. Ordinary installation cannot reopen this generation. +pub fn deny_container_fence( + db: &RelationalDB, + tx: &mut MutTx, + expected: &StContainerFenceRow, +) -> Result { + if !expected.allowed || expected.generation == 0 { + return Err(DeploymentError::FenceConflict); + } + let key: AlgebraicValue = expected.source_identity.into(); + let (pointer, current) = tx + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(|row| StContainerFenceRow::try_from(row).map(|value| (row.pointer(), value))) + .transpose()? + .ok_or(DeploymentError::FenceConflict)?; + let denied = StContainerFenceRow { + allowed: false, + ..expected.clone() + }; + let (revision_before, revision_after) = if current == denied { + let revision = db.hosted_admission().fence_revision(); + (revision, revision) + } else { + if current != *expected { + return Err(DeploymentError::FenceConflict); + } + // The gate is closed before the row changes. A rollback still + // conservatively invalidates any concurrently progressing page scan. + let revisions = db + .hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, &denied)?; + revisions + }; + Ok(FenceDenial { + row: denied, + revision_before, + revision_after, + }) +} + +/// Called with verified hosted credentials inside every admitted transaction, +/// including each later transaction of a procedure. Signature, audience, expiry, +/// capability, and interface checks are additional receiving-host requirements. +pub fn check_container_fence( + state: &S, + source: Identity, + generation: u64, + target_grant_revision: u64, +) -> Result<(), DeploymentError> { + let key = AlgebraicValue::U256(source.to_u256().into()); + let fence = state + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(StContainerFenceRow::try_from) + .transpose()?; + if !fence + .is_some_and(|f| f.allowed && f.generation == generation && f.target_grant_revision == target_grant_revision) + { + return Err(DeploymentError::ContainerFenced); + } + Ok(()) +} + +/// Capture validated hosted connection authority in the transaction inserting st_client. +/// This is host-only metadata; ordinary connections leave no row and retain flags zero. +pub(crate) fn record_connection_auth( + tx: &mut MutTx, + connection_id: ConnectionId, + sender: Identity, + call_auth_flags: u32, +) -> Result<(), DeploymentError> { + tx.insert_via_serialize_bsatn( + ST_CONNECTION_AUTH_ID, + &StConnectionAuthRow { + connection_id: connection_id.into(), + sender_identity: sender.into(), + call_auth_flags, + }, + )?; + Ok(()) +} + +/// Recover captured authority for a host-dispatched disconnect event. It is not +/// a new container admission and does not require a still-valid credential/lease. +pub(crate) fn connection_auth_flags( + state: &S, + connection_id: ConnectionId, + sender: Identity, +) -> Result { + let key: AlgebraicValue = ConnectionIdViaU128::from(connection_id).into(); + let row = state + .iter_by_col_eq(ST_CONNECTION_AUTH_ID, ColId(0), &key)? + .next() + .map(StConnectionAuthRow::try_from) + .transpose()?; + let Some(row) = row else { return Ok(0) }; + if row.sender_identity.0 != sender { + return Err(DeploymentError::CorruptMetadata); + } + Ok(row.call_auth_flags) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/db/deployment/tests.rs b/crates/core/src/db/deployment/tests.rs new file mode 100644 index 00000000000..c027d017136 --- /dev/null +++ b/crates/core/src/db/deployment/tests.rs @@ -0,0 +1,633 @@ +use super::*; +use crate::db::relational_db::tests_utils::{begin_mut_tx, TestDB}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::{ + StEnvRow, ST_CLIENT_ID, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, ST_ENV_ID, +}; +use spacetimedb_durability::Durability; +use spacetimedb_lib::deployment::{ + DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind, PUBLISH_RETRY_WINDOW_MS, +}; + +fn request(sequence: u64, previous: Option) -> DeploymentCommit { + DeploymentCommit { + operation_id: Uuid::from_u128(0x01991ec4000070008000000000000000 | u128::from(sequence)), + publication_epoch: sequence, + publisher: Identity::from_u256(55u64.into()), + expected_revision: previous, + prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(sequence.to_le_bytes()), + }), + container: None, + }), + } +} + +fn now() -> Timestamp { + Timestamp::from_micros_since_unix_epoch(((request(1, None).operation_id.as_u128() >> 80) as i64) * 1000) +} + +fn transact( + db: &RelationalDB, + f: impl FnOnce(&mut MutTx) -> Result, +) -> Result { + db.with_auto_commit(Workload::ForTests, f) +} + +#[test] +fn deployment_retry_returns_original_result_after_later_publish_without_mutation() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let accepted = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let second = request(2, Some(accepted.revision)); + let later = transact(&db, |tx| { + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!(check_deployment_commit(tx, &first, now(), &limits)?, CommitAdmission::AlreadyCommitted(ref r) if r == &accepted)); + assert_eq!(record_deployment_commit(tx, &first, now(), &limits)?, accepted); + assert_eq!(current_deployment(tx)?.unwrap().0, later.revision); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(2)); + Ok(()) + }).unwrap(); +} + +#[test] +fn deployment_fence_and_revision_conflicts_fail_before_module_execution() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let mut second = request(2, Some(hash_bytes(b"not current"))); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, second.publication_epoch, second.operation_id) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits), + Err(DeploymentError::RevisionConflict) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + second.expected_revision = None; + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn deployment_and_module_effects_roll_back_together() { + let db = TestDB::in_memory().unwrap(); + let request = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); + let failed: Result<(), DeploymentError> = transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + // A second persistent table stands in for migration effects in the same + // transaction. Integration must additionally execute Wasm/JS migrations. + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: "MIGRATED".into(), + value: "yes".into(), + }, + )?; + record_deployment_commit(tx, &request, now(), &limits)?; + Err(DeploymentError::Database(DBError::Other(anyhow::anyhow!( + "injected failure before commit" + )))) + }); + assert!(failed.is_err()); + transact(&db, |tx| { + assert!(current_deployment(tx)?.is_none()); + assert_eq!(tx.table_row_count(ST_ENV_ID), Some(0)); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(0)); + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_closes_delayed_commits_and_cannot_reopen_its_epoch() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id) + }) + .unwrap(); + let aborted = AbortResult::Aborted { + previous_revision: None, + }; + assert_eq!( + transact(&db, |tx| abort_deployment_commit(tx, &first)).unwrap(), + aborted + ); + transact(&db, |tx| { + assert_eq!(abort_deployment_commit(tx, &first)?, aborted); + assert!(matches!( + record_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch + 1, Uuid::NIL), + Err(DeploymentError::PublicationFenced) + )); + assert!(current_deployment(tx)?.is_none()); + let second = request(2, None); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + let result = record_deployment_commit(tx, &second, now(), &limits)?; + assert!(matches!( + abort_deployment_commit(tx, &first), + Err(DeploymentError::PublicationFenced) + )); + assert_eq!(current_deployment(tx)?.unwrap().0, result.revision); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_commit_winning_abort_race_is_never_reported_as_aborted() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let second = request(2, Some(committed.revision)); + transact(&db, |tx| { + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed.clone()) + ); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits)?; + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed) + ); + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits)?, + CommitAdmission::AlreadyCommitted(_) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_rollback_does_not_report_a_closed_fence() { + let db = TestDB::in_memory().unwrap(); + let request = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); + let failed: Result<(), DeploymentError> = transact(&db, |tx| { + abort_deployment_commit(tx, &request)?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(failed.is_err()); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn recovery_of_expired_publication_preserves_commits_and_closes_uncommitted_epochs() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let expired = Timestamp::from_micros_since_unix_epoch( + now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS * 1000) as i64, + ); + let second = request(2, Some(committed.revision)); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &first, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed.clone()) + ); + let mut wrong_publisher = first.clone(); + wrong_publisher.publisher = Identity::ZERO; + assert!(matches!( + committed_deployment_operation(tx, &wrong_publisher), + Err(DeploymentError::OperationConflict) + )); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + assert!(matches!( + check_deployment_commit(tx, &second, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + assert_eq!( + abort_deployment_commit(tx, &second)?, + AbortResult::Aborted { + previous_revision: Some(committed.revision) + } + ); + // Clock rollback cannot make a delayed old commit admissible again. + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_and_commit_receipts_survive_commitlog_replay() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + abort_deployment_commit(tx, &first) + }) + .unwrap(); + let db = db.reopen().unwrap(); + let second = request(2, None); + let committed = transact(&db, |tx| { + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::Aborted { + previous_revision: None + } + ); + assert!(matches!( + record_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits) + }) + .unwrap(); + let db = db.reopen().unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!( + committed_deployment_operation(tx, &second).unwrap(), + Some(committed.clone()) + ); + assert_eq!(current_deployment(tx).unwrap().unwrap().0, committed.revision); + }); + assert_eq!( + transact(&db, |tx| abort_deployment_commit(tx, &second)).unwrap(), + AbortResult::AlreadyCommitted(committed) + ); +} + +#[test] +fn publication_recovery_does_not_reapply_tightened_resource_admission() { + use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, + }; + let db = TestDB::in_memory().unwrap(); + let mut first = request(1, None); + let DeploymentSpec::V1(spec) = &mut first.deployment; + spec.container = Some(ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 10_000, + }); + let original_limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &original_limits) + }) + .unwrap(); + let mut tightened = original_limits; + tightened.resources.cpu_millicores = 500; + let mut second = request(2, Some(committed.revision)); + second.deployment = first.deployment.clone(); + transact(&db, |tx| { + assert!(first.deployment.clone().normalize(&tightened).is_err()); + assert!(matches!(check_deployment_commit(tx, &first, now(), &tightened)?, CommitAdmission::AlreadyCommitted(ref result) if result == &committed)); + assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!(abort_deployment_commit(tx, &first)?, AbortResult::AlreadyCommitted(committed.clone())); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + assert!(check_deployment_commit(tx, &second, now(), &tightened).is_err()); + assert_eq!(abort_deployment_commit(tx, &second)?, AbortResult::Aborted { previous_revision: Some(committed.revision) }); + assert!(deployment_publication_aborted(tx, &second)?); + Ok(()) + }).unwrap(); +} + +#[test] +fn deployment_operation_cannot_be_reused_by_another_publisher_or_changed_request() { + let db = TestDB::in_memory().unwrap(); + let original = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, original.publication_epoch, original.operation_id)?; + record_deployment_commit(tx, &original, now(), &limits) + }) + .unwrap(); + transact(&db, |tx| { + let mut changed = original.clone(); + changed.publisher = Identity::from_u256(99u64.into()); + assert!(matches!( + check_deployment_commit(tx, &changed, now(), &limits), + Err(DeploymentError::OperationConflict) + )); + changed = original.clone(); + changed.deployment = DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(1), + container: None, + }); + assert!(matches!( + check_deployment_commit(tx, &changed, now(), &limits), + Err(DeploymentError::OperationConflict) + )); + // A retained row does not extend the advertised retry window. + let expired = Timestamp::from_micros_since_unix_epoch( + now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS as i64) * 1000, + ); + assert!(matches!( + check_deployment_commit(tx, &original, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn container_fence_revokes_copied_credentials_and_does_not_reopen_at_same_generation() { + let db = TestDB::in_memory().unwrap(); + let source = Identity::from_u256(777u64.into()); + let first = StContainerFenceRow { + source_identity: source.into(), + generation: 1, + target_grant_revision: 3, + target_set_hash: hash_bytes(b"targets1"), + allowed: true, + }; + transact(&db, |tx| { + assert!(matches!( + check_container_fence(tx, source, 1, 3), + Err(DeploymentError::ContainerFenced) + )); + install_container_fence(&db, tx, &first)?; + install_container_fence(&db, tx, &first)?; + check_container_fence(tx, source, 1, 3)?; + assert!(matches!( + check_container_fence(tx, source, 1, 4), + Err(DeploymentError::ContainerFenced) + )); + Ok(()) + }) + .unwrap(); + let revoked = StContainerFenceRow { + generation: 2, + target_grant_revision: 4, + target_set_hash: hash_bytes(b"targets2"), + allowed: false, + ..first.clone() + }; + transact(&db, |tx| { + install_container_fence(&db, tx, &revoked)?; + assert!(matches!( + check_container_fence(tx, source, 1, 3), + Err(DeploymentError::ContainerFenced) + )); + assert!(matches!( + check_container_fence(tx, source, 2, 4), + Err(DeploymentError::ContainerFenced) + )); + let reopen = StContainerFenceRow { + allowed: true, + ..revoked.clone() + }; + assert!(matches!( + install_container_fence(&db, tx, &reopen), + Err(DeploymentError::FenceConflict) + )); + assert!(matches!( + install_container_fence(&db, tx, &first), + Err(DeploymentError::FenceConflict) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn captured_connection_authority_survives_replay_without_inferring_sender_authority() { + let db = TestDB::durable().unwrap(); + let self_sender = db.database_identity(); + let foreign_sender = Identity::ONE; + let self_connection = ConnectionId::from_u128(41); + let foreign_connection = ConnectionId::from_u128(42); + let ordinary_connection = ConnectionId::from_u128(43); + transact(&db, |tx| { + for (connection, sender, flags) in [ + (self_connection, self_sender, 1), + (foreign_connection, foreign_sender, 0), + ] { + tx.insert_st_client( + sender, + connection, + r#"{"iss":"platform","sub":"previously-admitted","exp":1}"#, + )?; + record_connection_auth(tx, connection, sender, flags)?; + } + tx.insert_st_client(self_sender, ordinary_connection, "ordinary JWT")?; + Ok(()) + }) + .unwrap(); + // TestDB::reopen expects zero connected clients. Reopen the same committed + // log explicitly to exercise crash recovery with outstanding connections. + let (db, durability, runtime, directory) = db.into_parts(); + let runtime = runtime.unwrap(); + let directory = directory.unwrap(); + let durability = durability.unwrap(); + runtime.block_on(db.shutdown()); + drop(db); + runtime.block_on(durability.close()); + drop(durability); + let _runtime_guard = runtime.enter(); + let (db, durability) = TestDB::open_existing_durable( + &directory, + runtime.handle().clone(), + 0, + TestDB::DATABASE_IDENTITY, + TestDB::OWNER, + true, + ) + .unwrap(); + transact(&db, |tx| { + assert_eq!(connection_auth_flags(tx, self_connection, self_sender)?, 1); + assert_eq!(connection_auth_flags(tx, foreign_connection, foreign_sender)?, 0); + // Equal sender/database identities do not invent internal authority. + assert_eq!(connection_auth_flags(tx, ordinary_connection, self_sender)?, 0); + assert_eq!(tx.table_row_count(ST_CONNECTION_AUTH_ID), Some(2)); + assert!(matches!( + connection_auth_flags(tx, self_connection, foreign_sender), + Err(DeploymentError::CorruptMetadata) + )); + Ok(()) + }) + .unwrap(); + db.clear_all_clients().unwrap(); + transact(&db, |tx| { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(0)); + } + Ok(()) + }) + .unwrap(); + runtime.block_on(db.shutdown()); + drop(db); + runtime.block_on(durability.close()); +} + +#[test] +fn connection_auth_and_client_rows_share_connect_and_cleanup_transactions() { + let db = TestDB::in_memory().unwrap(); + let sender = db.database_identity(); + let connection = ConnectionId::from_u128(77); + let rejected: Result<(), DeploymentError> = transact(&db, |tx| { + tx.insert_st_client(sender, connection, "JWT")?; + record_connection_auth(tx, connection, sender, 1)?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(rejected.is_err()); + transact(&db, |tx| { + assert!(tx.st_client_row(sender, connection).is_none()); + assert_eq!(connection_auth_flags(tx, connection, sender)?, 0); + tx.insert_st_client(sender, connection, "JWT")?; + record_connection_auth(tx, connection, sender, 1)?; + Ok(()) + }) + .unwrap(); + // A failed callback transaction cannot partially delete its captured auth. + let failed_callback: Result<(), DeploymentError> = transact(&db, |tx| { + tx.delete_st_client(sender, connection, db.database_identity())?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(failed_callback.is_err()); + transact(&db, |tx| { + assert!(tx.st_client_row(sender, connection).is_some()); + assert_eq!(connection_auth_flags(tx, connection, sender)?, 1); + // Both successful callbacks and fallback cleanup use this deletion path. + tx.delete_st_client(sender, connection, db.database_identity())?; + Ok(()) + }) + .unwrap(); + transact(&db, |tx| { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(0)); + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn container_fence_installation_serializes_with_admitted_transactions() { + let db = TestDB::in_memory().unwrap(); + let source = Identity::from_u256(778u64.into()); + let first = StContainerFenceRow { + source_identity: source.into(), + generation: 1, + target_grant_revision: 0, + target_set_hash: hash_bytes(b"self"), + allowed: true, + }; + transact(&db, |tx| install_container_fence(&db, tx, &first)).unwrap(); + let admitted = begin_mut_tx(&db); + check_container_fence(&admitted, source, 1, 0).unwrap(); + // A fence cannot commit midway through an already admitted transaction. + assert!(db + .try_begin_mut_tx( + spacetimedb_datastore::traits::IsolationLevel::Serializable, + Workload::ForTests + ) + .is_none()); + let _ = db.rollback_mut_tx(admitted); + transact(&db, |tx| { + install_container_fence(&db, tx, &StContainerFenceRow { generation: 2, ..first }) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!( + check_container_fence(tx, source, 1, 0), + Err(DeploymentError::ContainerFenced) + )); + check_container_fence(tx, source, 2, 0) + }) + .unwrap(); +} diff --git a/crates/core/src/db/durability.rs b/crates/core/src/db/durability.rs index c17a10e9f63..ead4ca83e4e 100644 --- a/crates/core/src/db/durability.rs +++ b/crates/core/src/db/durability.rs @@ -1,15 +1,12 @@ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; -use log::{error, info}; use spacetimedb_commitlog::payload::{ txdata::{Mutations, Ops}, Txdata, }; use spacetimedb_datastore::{execution_context::ReducerContext, traits::TxData}; use spacetimedb_durability::Transaction; -use spacetimedb_lib::Identity; use spacetimedb_sats::ProductValue; -use tokio::{runtime, time::timeout}; use crate::db::persistence::Durability; @@ -32,21 +29,6 @@ pub(super) fn request_durability( })); } -pub(super) fn spawn_close(durability: Arc, runtime: &runtime::Handle, database_identity: Identity) { - let rt = runtime.clone(); - rt.spawn(async move { - let label = format!("[{database_identity}]"); - match timeout(Duration::from_secs(10), durability.close()).await { - Err(_elapsed) => { - error!("{label} timeout waiting for durability shutdown"); - } - Ok(offset) => { - info!("{label} durability shut down at tx offset: {offset:?}"); - } - } - }); -} - fn prepare_tx_data_for_durability( tx_offset: u64, reducer_context: Option, diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs new file mode 100644 index 00000000000..87b7a7231f7 --- /dev/null +++ b/crates/core/src/db/environment.rs @@ -0,0 +1,131 @@ +//! Dedicated access to the private environment store. +//! +//! Mutation callers must authorize owner/admin access before calling these +//! helpers and commit through the normal module transaction machinery so +//! dependent views refresh. Helpers never acquire a second transaction. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; +use spacetimedb_lib::environment::{validate_key, validate_value, EnvironmentValidationError, MAX_ENV_VARS}; +use spacetimedb_sats::AlgebraicValue; +use std::collections::BTreeMap; + +#[derive(Debug, thiserror::Error)] +pub enum EnvironmentError { + #[error(transparent)] + Validation(#[from] EnvironmentValidationError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Read from exactly the caller's snapshot, preserving missing versus empty. +pub fn get(state: &impl StateView, key: &str) -> Result, EnvironmentError> { + validate_key(key)?; + state + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| Ok(StEnvRow::try_from(row)?.value)) + .transpose() +} + +pub fn snapshot(state: &impl StateView) -> Result, EnvironmentError> { + state + .iter(ST_ENV_ID)? + .map(|row| { + let row = StEnvRow::try_from(row)?; + Ok((row.key, row.value)) + }) + .collect() +} + +/// Insert or replace one key. Validation occurs before any mutation. +pub fn set(db: &RelationalDB, tx: &mut MutTx, key: &str, value: &str) -> Result<(), EnvironmentError> { + validate_key(key)?; + validate_value(value)?; + let previous = get(tx, key)?; + if previous.is_none() && tx.table_row_count(ST_ENV_ID).unwrap_or(0) >= MAX_ENV_VARS as u64 { + return Err(EnvironmentValidationError::TooManyVariables.into()); + } + if previous.as_deref() == Some(value) { + return Ok(()); + } + delete(db, tx, key)?; + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.into(), + value: value.into(), + }, + )?; + Ok(()) +} + +pub fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { + validate_key(key)?; + let pointer = tx + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| row.pointer()); + if let Some(pointer) = pointer { + db.delete(tx, ST_ENV_ID, [pointer]); + return Ok(true); + } + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use spacetimedb_datastore::execution_context::Workload; + + #[test] + fn missing_empty_nul_update_and_rollback() { + let db = TestDB::in_memory().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + assert_eq!(get(tx, "EMPTY")?, None); + set(&db, tx, "EMPTY", "")?; + set(&db, tx, "NUL", "a\0b")?; + assert_eq!(get(tx, "EMPTY")?, Some(String::new())); + assert_eq!(get(tx, "NUL")?, Some("a\0b".into())); + Ok(()) + }) + .unwrap(); + let result = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + set(&db, tx, "EMPTY", "changed")?; + delete(&db, tx, "NUL")?; + Err(EnvironmentValidationError::InvalidKey.into()) + }); + assert!(result.is_err()); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!( + snapshot(tx).unwrap(), + BTreeMap::from([("EMPTY".into(), "".into()), ("NUL".into(), "a\0b".into())]) + ); + }); + } + + #[test] + fn capacity_and_value_limits_precede_mutation() { + let db = TestDB::in_memory().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + for i in 0..MAX_ENV_VARS { + set(&db, tx, &format!("K{i}"), "")?; + } + assert!(set(&db, tx, "EXTRA", "").is_err()); + set(&db, tx, "K0", "updated")?; + assert!(set(&db, tx, "K0", &"x".repeat(8193)).is_err()); + assert_eq!(get(tx, "K0")?.as_deref(), Some("updated")); + assert!(delete(&db, tx, "K1")?); + assert!(!delete(&db, tx, "MISSING")?); + set(&db, tx, "EXTRA", "")?; + Ok(()) + }) + .unwrap(); + } +} diff --git a/crates/core/src/db/hosted_admission.rs b/crates/core/src/db/hosted_admission.rs new file mode 100644 index 00000000000..59b6f30ef1c --- /dev/null +++ b/crates/core/src/db/hosted_admission.rs @@ -0,0 +1,282 @@ +//! Transient receiving-host admission, separate from durable generation fences. +//! +//! A replayed fence can be older than current control authority. Every database +//! open starts closed, including cold maintenance opens and restored databases. +//! Trusted platform code must reconcile the complete incoming fence inventory +//! before completing a sweep. This state is never serialized or restored. + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +use anyhow::ensure; +use parking_lot::Mutex; + +#[derive(Default)] +struct State { + revision: u64, + fence_revision: u64, + sweeping: bool, +} + +#[derive(Default)] +struct Inner { + open: AtomicBool, + state: Mutex, +} + +/// Owned by one live RelationalDB. Opening another copy of the same database +/// Identity creates an independent, closed gate. +#[derive(Default)] +pub struct HostedAdmission(Arc); + +impl HostedAdmission { + pub fn is_open(&self) -> bool { + self.0.open.load(Ordering::Acquire) + } + + /// Start one bounded reconciliation. A competing sweep fails immediately. + /// Dropping its ticket keeps admission closed and permits a later retry. + pub fn begin(&self) -> anyhow::Result { + let mut state = self.0.state.lock(); + ensure!(!state.sweeping, "hosted admission reconciliation is already running"); + self.0.open.store(false, Ordering::Release); + let revision = state + .revision + .checked_add(1) + .filter(|value| *value != u64::MAX) + .ok_or_else(|| anyhow::anyhow!("hosted admission revision exhausted"))?; + state.revision = revision; + state.sweeping = true; + Ok(HostedAdmissionSweep { + inner: self.0.clone(), + revision, + fence_revision: state.fence_revision, + }) + } + + /// Read while holding the database transaction that observes fence rows. + pub fn fence_revision(&self) -> u64 { + self.0.state.lock().fence_revision + } + + /// Call before changing a durable fence, within its mutation transaction. + /// Even a later rollback conservatively invalidates an inventory scan. A + /// delayed installation after startup closes admission again, so it cannot + /// insert an unexamined allowed row behind a completed scan's cursor. + pub fn fences_changed(&self) -> anyhow::Result<(u64, u64)> { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + let previous = state.fence_revision; + let Some(next) = previous.checked_add(1).filter(|next| *next != u64::MAX) else { + state.fence_revision = u64::MAX; + state.revision = u64::MAX; + state.sweeping = false; + anyhow::bail!("hosted fence revision exhausted"); + }; + state.fence_revision = next; + Ok((previous, next)) + } + + /// Invalidate any outstanding sweep. This prevents new admission; it is + /// not a substitute for a transactional fence and positive actor drainage. + pub fn close(&self) { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + state.revision = state.revision.saturating_add(1); + state.sweeping = false; + } + + /// Permanently close a database whose storage writer is being shut down. + /// A retained handle cannot start a new sweep on that obsolete object. + pub fn seal(&self) { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + state.revision = u64::MAX; + state.sweeping = false; + } +} + +/// A ticket is tied to the exact database-open state that issued it. Completing +/// one cannot open a replacement database, or undo a later close operation. +#[must_use = "dropping the sweep leaves hosted admission closed"] +pub struct HostedAdmissionSweep { + inner: Arc, + revision: u64, + fence_revision: u64, +} + +impl HostedAdmissionSweep { + /// Retain this guard in the async owner when moving the ticket to physical + /// blocking work. Cancelling that owner invalidates this exact sweep, even + /// when dropping the blocking task's JoinHandle cannot stop its execution. + pub fn cancellation_guard(&self) -> HostedAdmissionCancellation { + HostedAdmissionCancellation { + inner: Some(self.inner.clone()), + revision: self.revision, + } + } + + /// Trusted platform code calls this only after durable fence installation, + /// actor completion and current control confirmation of the entire sweep. + pub fn complete(self) -> anyhow::Result<()> { + let revision = self.fence_revision; + self.complete_with_fence_revision(revision) + } + + /// Complete a scan which made its own confirmed fence mutations. The + /// caller holds the actual DB transaction and has verified every stored + /// fence against current authority at this exact physical revision. + pub fn complete_with_fence_revision(self, fence_revision: u64) -> anyhow::Result<()> { + let mut state = self.inner.state.lock(); + ensure!( + state.sweeping && state.revision == self.revision, + "hosted admission reconciliation was invalidated" + ); + ensure!( + fence_revision != u64::MAX && state.fence_revision == fence_revision, + "hosted fence inventory changed during reconciliation" + ); + state.sweeping = false; + self.inner.open.store(true, Ordering::Release); + Ok(()) + } +} + +/// Cancellation belongs to one sweep revision. A late guard cannot close a +/// newer retry, and completion and cancellation serialize on the same mutex. +#[must_use = "retain until physical completion and disarm only after success"] +pub struct HostedAdmissionCancellation { + inner: Option>, + revision: u64, +} + +impl HostedAdmissionCancellation { + pub fn disarm(mut self) { + self.inner = None; + } +} + +impl Drop for HostedAdmissionCancellation { + fn drop(&mut self) { + let Some(inner) = &self.inner else { return }; + let mut state = inner.state.lock(); + if state.revision == self.revision { + inner.open.store(false, Ordering::Release); + state.revision = state.revision.saturating_add(1); + state.sweeping = false; + } + } +} + +impl Drop for HostedAdmissionSweep { + fn drop(&mut self) { + let mut state = self.inner.state.lock(); + if state.revision == self.revision { + state.sweeping = false; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cancelled_sweep_retries_and_cannot_open_another_database() { + let original = HostedAdmission::default(); + let replacement = HostedAdmission::default(); + assert!(!original.is_open()); + let sweep = original.begin().unwrap(); + assert!(original.begin().is_err()); + drop(sweep); + assert!(!original.is_open()); + original.begin().unwrap().complete().unwrap(); + assert!(original.is_open()); + assert!(!replacement.is_open()); + } + + #[test] + fn stale_completion_and_drop_cannot_open_or_cancel_a_newer_sweep() { + let gate = HostedAdmission::default(); + let stale = gate.begin().unwrap(); + gate.close(); + let current = gate.begin().unwrap(); + assert!(stale.complete().is_err()); + assert!(!gate.is_open()); + assert!(gate.begin().is_err()); + current.complete().unwrap(); + assert!(gate.is_open()); + gate.close(); + assert!(!gate.is_open()); + } + + #[test] + fn revision_exhaustion_remains_closed() { + let gate = HostedAdmission::default(); + gate.0.state.lock().revision = u64::MAX - 2; + gate.begin().unwrap().complete().unwrap(); + assert!(gate.is_open()); + assert!(gate.begin().is_err()); + assert!(!gate.is_open()); + gate.close(); + assert!(gate.begin().is_err()); + assert!(!gate.is_open()); + } + + #[test] + fn owner_cancellation_invalidates_detached_completion_without_closing_a_newer_retry() { + let gate = HostedAdmission::default(); + let stale = gate.begin().unwrap(); + let cancellation = stale.cancellation_guard(); + drop(cancellation); + let current = gate.begin().unwrap(); + assert!(stale.complete().is_err()); + current.complete().unwrap(); + assert!(gate.is_open()); + + gate.close(); + let stale = gate.begin().unwrap(); + let cancellation = stale.cancellation_guard(); + gate.close(); + gate.begin().unwrap().complete().unwrap(); + drop(cancellation); + assert!(gate.is_open()); + assert!(stale.complete().is_err()); + } + + #[test] + fn only_an_acknowledged_completion_survives_owner_drop() { + let gate = HostedAdmission::default(); + let ticket = gate.begin().unwrap(); + let cancellation = ticket.cancellation_guard(); + ticket.complete().unwrap(); + drop(cancellation); + assert!(!gate.is_open()); + let ticket = gate.begin().unwrap(); + let cancellation = ticket.cancellation_guard(); + ticket.complete().unwrap(); + cancellation.disarm(); + assert!(gate.is_open()); + } + + #[test] + fn changed_fences_invalidate_scans_and_close_a_previously_open_gate() { + let gate = HostedAdmission::default(); + let ticket = gate.begin().unwrap(); + assert_eq!(gate.fences_changed().unwrap(), (0, 1)); + assert!(ticket.complete().is_err()); + let ticket = gate.begin().unwrap(); + assert_eq!(gate.fences_changed().unwrap(), (1, 2)); + ticket.complete_with_fence_revision(2).unwrap(); + assert!(gate.is_open()); + gate.fences_changed().unwrap(); + assert!(!gate.is_open()); + let ticket = gate.begin().unwrap(); + gate.0.state.lock().fence_revision = u64::MAX - 1; + assert!(gate.fences_changed().is_err()); + assert!(ticket.complete_with_fence_revision(u64::MAX).is_err()); + assert!(gate.begin().is_err()); + } +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 45777fbd612..d2f06a453b5 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -9,7 +9,11 @@ use crate::subscription::ExecutionCounters; use spacetimedb_datastore::execution_context::WorkloadType; use spacetimedb_datastore::{locking_tx_datastore::datastore::TxMetrics, traits::TxData}; +pub mod container_environment; +pub mod deployment; mod durability; +pub mod environment; +pub mod hosted_admission; pub mod persistence; pub mod relational_db; pub mod snapshot; diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index c8b81b90fea..e192175883f 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -1,4 +1,4 @@ -use crate::db::durability::{request_durability, spawn_close as spawn_durability_close}; +use crate::db::durability::request_durability; use crate::db::MetricsRecorderQueue; use crate::error::{DBError, RestoreSnapshotError}; use crate::subscription::ExecutionCounters; @@ -6,6 +6,8 @@ use crate::util::asyncify; use crate::worker_metrics::WORKER_METRICS; use anyhow::{anyhow, Context}; use enum_map::EnumMap; +use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; use spacetimedb_commitlog::repo::OnNewSegmentFn; use spacetimedb_commitlog::{self as commitlog, Commitlog, SizeOnDisk}; use spacetimedb_data_structures::map::HashSet; @@ -58,7 +60,9 @@ use spacetimedb_table::table_index::IndexKey; use std::borrow::Cow; use std::io; use std::ops::RangeBounds; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::sync::OnceLock; use tokio::sync::watch; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; @@ -95,8 +99,11 @@ pub type ConnectedClients = HashSet<(Identity, ConnectionId)>; pub struct RelationalDB { database_identity: Identity, owner_identity: Identity, + hosted_admission: super::hosted_admission::HostedAdmission, inner: Locking, + commits_closed: Arc, + shutdown: OnceLock>>>, durability: Option>, durability_runtime: Option, snapshot_worker: Option, @@ -132,9 +139,12 @@ impl std::fmt::Debug for RelationalDB { impl Drop for RelationalDB { fn drop(&mut self) { - // Attempt to flush the outstanding transactions. - if let (Some(durability), Some(runtime)) = (self.durability.take(), self.durability_runtime.take()) { - spawn_durability_close(durability, &runtime, self.database_identity); + self.hosted_admission.seal(); + if let Some(runtime) = &self.durability_runtime { + // Join the same owned close when a cancelled shutdown waiter was the + // last database owner. Never start a second provider close early. + let close = self.start_shutdown(runtime); + runtime.spawn(close); } } } @@ -154,12 +164,15 @@ impl RelationalDB { Self { inner, + commits_closed: Default::default(), + shutdown: Default::default(), durability, durability_runtime, snapshot_worker, database_identity, owner_identity, + hosted_admission: Default::default(), row_count_fn: default_row_count_fn(database_identity), disk_size_fn, @@ -169,6 +182,12 @@ impl RelationalDB { } } + /// Transient container admission for this exact database open. The trusted + /// receiving host opens it only after reconciling current incoming fences. + pub fn hosted_admission(&self) -> &super::hosted_admission::HostedAdmission { + &self.hosted_admission + } + /// Open a database, which may or may not already exist. /// /// # Initialization @@ -339,23 +358,41 @@ impl RelationalDB { /// Shut down the database, without dropping it. /// - /// If the database is in-memory only, this does nothing. - /// Otherwise, it instructs the durability layer to shut down - /// and waits until all outstanding transactions are reported as durable. - /// - /// After calling this method, calling [Self::commit_tx_downgrade] or - /// [Self::commit_tx] will panic. - /// - /// Returns `None` if the database is in-memory only, - /// or nothing has been durably persisted yet. - /// - /// Returns the durable [TxOffset] in a `Some` otherwise. + /// Permanently closes hosted and commit admission, then joins the configured + /// durability writer. Transactions already holding the exclusive transaction + /// lock can finish before closure; later commits roll back with `DatabaseClosed`. + /// Caller cancellation does not cancel the physical close. pub async fn shutdown(&self) -> Option { - if let Some(durability) = &self.durability { - return durability.close().await; - } - - None + self.hosted_admission.seal(); + self.start_shutdown(&tokio::runtime::Handle::current()).await + } + + fn start_shutdown(&self, runtime: &tokio::runtime::Handle) -> Shared>> { + self.shutdown + .get_or_init(|| { + let inner = self.inner.clone(); + let closed = self.commits_closed.clone(); + let durability = self.durability.clone(); + // A shutdown owns this task through both lock acquisition and actual + // writer completion. The blocking lock never occupies a runtime worker. + let task = runtime.spawn(async move { + tokio::task::spawn_blocking(move || { + let tx = inner.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + closed.store(true, Ordering::Relaxed); + let _ = inner.rollback_mut_tx(tx); + }) + .await + .expect("database close transaction worker panicked"); + match durability { + Some(durability) => durability.close().await, + None => None, + } + }); + async move { task.await.expect("database writer close task panicked") } + .boxed() + .shared() + }) + .clone() } /// Create any system tables that are missing from the datastore. @@ -816,6 +853,13 @@ impl RelationalDB { tx: MutTx, ) -> Result, TxMetrics, Option)>, DBError> { log::trace!("COMMIT MUT TX"); + // `tx` holds the same exclusive lock used by start_shutdown. The flag + // cannot change between this check and the durability append. + if self.commits_closed.load(Ordering::Relaxed) { + let (_, metrics, reducer) = self.rollback_mut_tx(tx); + self.report_tx_metrics(reducer, None, Some(metrics), None); + return Err(DBError::DatabaseClosed); + } let reducer_context = tx.ctx.reducer_context().cloned(); // TODO: Never returns `None` -- should it? @@ -832,8 +876,15 @@ impl RelationalDB { } #[tracing::instrument(level = "trace", skip_all)] - pub fn commit_tx_downgrade(&self, tx: MutTx, workload: Workload) -> (Arc, TxMetrics, Tx) { + pub fn commit_tx_downgrade(&self, tx: MutTx, workload: Workload) -> Result<(Arc, TxMetrics, Tx), DBError> { log::trace!("COMMIT MUT TX"); + // `tx` holds the same exclusive lock used by start_shutdown. The flag + // cannot change between this check and the durability append. + if self.commits_closed.load(Ordering::Relaxed) { + let (_, metrics, reducer) = self.rollback_mut_tx(tx); + self.report_tx_metrics(reducer, None, Some(metrics), None); + return Err(DBError::DatabaseClosed); + } let reducer_context = tx.ctx.reducer_context().cloned(); let (tx_data, tx_metrics, tx) = self.inner.commit_mut_tx_downgrade_and_then(tx, workload, |tx_data| { @@ -842,7 +893,7 @@ impl RelationalDB { self.maybe_do_snapshot(&tx_data); - (tx_data, tx_metrics, tx) + Ok((tx_data, tx_metrics, tx)) } /// Get the [`DurableOffset`] of this database, or `None` if this is an @@ -1489,6 +1540,7 @@ impl RelationalDB { self.with_auto_commit(Workload::Internal, |mut_tx| { self.clear_all_views(mut_tx)?; self.clear_table(mut_tx, ST_CONNECTION_CREDENTIALS_ID)?; + self.clear_table(mut_tx, spacetimedb_datastore::system_tables::ST_CONNECTION_AUTH_ID)?; self.clear_table(mut_tx, ST_CLIENT_ID)?; self.clear_table(mut_tx, ST_VIEW_SUB_ID)?; Ok(()) @@ -2286,6 +2338,9 @@ pub mod tests_utils { } } +#[cfg(test)] +mod shutdown_tests; + #[cfg(test)] mod tests { #![allow(clippy::disallowed_macros)] diff --git a/crates/core/src/db/relational_db/shutdown_tests.rs b/crates/core/src/db/relational_db/shutdown_tests.rs new file mode 100644 index 00000000000..67a3c0d4fd8 --- /dev/null +++ b/crates/core/src/db/relational_db/shutdown_tests.rs @@ -0,0 +1,94 @@ +use super::*; +use crate::db::environment; +use crate::host::module_host::{DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall}; +use crate::subscription::module_subscription_actor::ModuleSubscriptions; +use tests_utils::TestDB; + +#[test] +fn operation_drain_shutdown_serializes_transactions_and_survives_cancelled_waiter() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let fixture = TestDB::durable().unwrap(); + let db = fixture.db.clone(); + runtime.block_on(async { + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql); + environment::set(&db, &mut tx, "BEFORE", "committed").unwrap(); + let closing = tokio::spawn({ + let db = db.clone(); + async move { db.shutdown().await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while db.shutdown.get().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + closing.abort(); + assert!(closing.await.unwrap_err().is_cancelled()); + assert!(!db.commits_closed.load(Ordering::Relaxed)); + // This SQL/view transaction acquired the exclusive lock before shutdown. + let (_, _, read) = db.commit_tx_downgrade(tx, Workload::Sql).unwrap(); + let _ = db.release_tx(read); + assert!(db.shutdown().await.is_some()); + assert!(db.commits_closed.load(Ordering::Relaxed)); + for downgrade in [false, true] { + let mut late = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Subscribe); + environment::set(&db, &mut late, "LATE", "must-roll-back").unwrap(); + let error = if downgrade { + db.commit_tx_downgrade(late, Workload::Subscribe).err().unwrap() + } else { + db.commit_tx(late).err().unwrap() + }; + assert!(matches!(error, DBError::DatabaseClosed)); + } + let read = db.begin_tx(Workload::Internal); + assert_eq!(environment::get(&read, "BEFORE").unwrap().as_deref(), Some("committed")); + assert_eq!(environment::get(&read, "LATE").unwrap(), None); + let _ = db.release_tx(read); + }); +} + +#[test] +fn operation_drain_retained_sql_and_subscription_handles_reject_after_writer_close() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let fixture = TestDB::durable().unwrap(); + let db = fixture.db.clone(); + runtime.block_on(async { + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.clone()); + db.shutdown().await; + let auth = spacetimedb_lib::identity::AuthCtx::new(db.owner_identity(), db.owner_identity()); + for statement in ["SELECT * FROM st_client", "SET env.LATE = 'denied'"] { + let error = crate::sql::execute::run( + db.clone(), + statement.into(), + auth.clone(), + Some(subscriptions.clone()), + None, + &mut vec![], + ) + .await + .err() + .unwrap(); + assert!(matches!(error, DBError::DatabaseClosed), "{error}"); + } + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Unsubscribe); + environment::set(&db, &mut tx, "LATE", "denied").unwrap(); + let event = ModuleEvent { + timestamp: spacetimedb_lib::Timestamp::now(), + caller_identity: db.owner_identity(), + caller_connection_id: None, + function_call: ModuleFunctionCall::update(), + status: EventStatus::Committed(DatabaseUpdate::default()), + reducer_return_value: None, + energy_quanta_used: crate::energy::EnergyQuanta::ZERO, + host_execution_duration: std::time::Duration::ZERO, + request_id: None, + timer: None, + }; + let error = subscriptions.commit_and_broadcast_event(None, event, tx).err().unwrap(); + assert!(matches!(error, DBError::DatabaseClosed)); + let tx = db.begin_tx(Workload::Internal); + assert_eq!(environment::get(&tx, "LATE").unwrap(), None); + let _ = db.release_tx(tx); + }); +} diff --git a/crates/core/src/db/snapshot.rs b/crates/core/src/db/snapshot.rs index 26e3d8373cf..93e17b79abd 100644 --- a/crates/core/src/db/snapshot.rs +++ b/crates/core/src/db/snapshot.rs @@ -7,7 +7,11 @@ use std::{ }; use anyhow::Context as _; -use futures::{channel::mpsc, StreamExt as _}; +use futures::{ + channel::mpsc, + future::{BoxFuture, Shared}, + FutureExt as _, StreamExt as _, +}; use log::{info, warn}; use parking_lot::RwLock; use prometheus::{Histogram, IntGauge}; @@ -61,6 +65,7 @@ pub struct SnapshotWorker { snapshot_created: watch::Sender, request_snapshot: mpsc::UnboundedSender, snapshot_repository: Arc, + completion: Shared>>>, } impl SnapshotWorker { @@ -86,15 +91,27 @@ impl SnapshotWorker { stats: <_>::default(), }), }; - tokio::spawn(actor.run()); + let task = tokio::spawn(actor.run()); + let completion = async move { task.await.map_err(|error| Arc::::from(error.to_string())) } + .boxed() + .shared(); Self { snapshot_created, request_snapshot: request_tx, snapshot_repository, + completion, } } + /// Permanently close snapshot admission and join all previously accepted + /// snapshot and compression I/O. All clones observe the same completion. + /// Cancelling a waiter does not cancel the worker or its blocking I/O. + pub async fn shutdown(&self) -> Result<(), Arc> { + self.request_snapshot.close_channel(); + self.completion.clone().await + } + /// Finish the initialization of [Self] by passing a [SnapshotDatabaseState], /// or replace the current [SnapshotDatabaseState] with a new one. /// @@ -370,3 +387,63 @@ impl Compressor { } } } + +#[cfg(test)] +mod shutdown_tests { + use super::*; + use spacetimedb_datastore::{ + execution_context::Workload, + system_tables::{StEnvRow, ST_ENV_ID}, + traits::{IsolationLevel, MutTx as _}, + }; + use spacetimedb_paths::{server::SnapshotsPath, FromPathUnchecked}; + use spacetimedb_snapshot::SnapshotRepository; + use spacetimedb_table::page_pool::PagePool; + + #[tokio::test(flavor = "multi_thread")] + async fn terminal_shutdown_drains_accepted_snapshot_after_waiter_cancellation() { + let dir = tempfile::tempdir().unwrap(); + let repository = Arc::new( + SnapshotRepository::open(SnapshotsPath::from_path_unchecked(dir.path()), Identity::ONE, 1).unwrap(), + ); + let datastore = Locking::bootstrap(Identity::ONE, PagePool::new_for_test()).unwrap(); + let mut tx = datastore.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: "PERSISTED".into(), + value: "snapshot drain".into(), + }, + ) + .unwrap(); + datastore.commit_mut_tx(tx).unwrap(); + let worker = SnapshotWorker::new(repository.clone(), Compression::Enabled); + worker.set_state(datastore.committed_state.clone()); + let state = datastore.committed_state.write_arc(); + worker.request_snapshot(); + let mut first = tokio::spawn({ + let worker = worker.clone(); + async move { worker.shutdown().await } + }); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut first) + .await + .is_err()); + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + let mut second = tokio::spawn({ + let worker = worker.clone(); + async move { worker.shutdown().await } + }); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut second) + .await + .is_err()); + assert!(worker.request_snapshot.unbounded_send(Request::TakeSnapshot).is_err()); + assert_eq!(repository.latest_snapshot().unwrap(), None); + drop(state); + second.await.unwrap().unwrap(); + worker.shutdown().await.unwrap(); + assert_eq!(repository.latest_snapshot().unwrap(), Some(0)); + let snapshot = repository.read_snapshot(0, &PagePool::new_for_test()).unwrap(); + assert_eq!(snapshot.tx_offset, 0); + } +} diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 4ba103979ec..53f7d1f852f 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -89,6 +89,8 @@ impl From for DatabaseError { #[derive(Error, Debug)] pub enum DBError { + #[error("database is closed")] + DatabaseClosed, #[error("LibError: {0}")] Lib(#[from] LibError), #[error("BufferError: {0}")] @@ -132,7 +134,7 @@ pub enum DBError { #[error("Error reading a value from a table through BSATN: {0}")] ReadViaBsatnError(#[from] ReadViaBsatnError), #[error("Module validation errors: {0}")] - ModuleValidationErrors(#[from] ValidationErrors), + ModuleValidationErrors(#[from] Box), #[error(transparent)] Other(#[from] anyhow::Error), #[error(transparent)] @@ -151,6 +153,12 @@ pub enum DBError { View(#[from] ViewCallError), } +impl From for DBError { + fn from(errors: ValidationErrors) -> Self { + Self::ModuleValidationErrors(Box::new(errors)) + } +} + impl From for DBError { fn from(value: InvalidFieldError) -> Self { LibError::from(value).into() @@ -237,6 +245,12 @@ pub enum LogReplayError { #[derive(Error, Debug)] pub enum NodesError { + #[error("invalid environment variable name")] + InvalidEnvironmentKey, + #[error("too many outstanding byte sources for environment read")] + EnvironmentSourceLimit, + #[error("hosted invocation rejected: {0}")] + HostedInvocationRejected(String), #[error("Failed to decode row: {0}")] DecodeRow(#[source] DecodeError), #[error("Failed to decode value: {0}")] diff --git a/crates/core/src/host/container_environment.rs b/crates/core/src/host/container_environment.rs new file mode 100644 index 00000000000..45535df1a09 --- /dev/null +++ b/crates/core/src/host/container_environment.rs @@ -0,0 +1,188 @@ +//! Bounded, durable host operations for immutable container environments. +//! +//! The caller must confirm current control authority and authoritative leader +//! before calling. These local host APIs are not an external authentication +//! interface. Historical restore must keep admission closed until current +//! operational fences have been reconciled. No in-memory production fallback. + +use crate::db::container_environment::{ + self as storage, EnvironmentClosedReceipt, EnvironmentSnapshotError, SecretEnvironment, +}; +use crate::db::relational_db::{MutTx, RelationalDB}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::container_environment::{EnvironmentSnapshotReceipt, EnvironmentSnapshotScope}; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Semaphore; + +/// Bounds queued/blocked transactions even when the async caller is cancelled. +static OPERATIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(8))); + +/// The barrier is specific to this proof. It can advance on an exact retry. +#[derive(Debug)] +pub struct Durable { + pub receipt: T, + pub durable_through: u64, +} + +pub type DurableSnapshotReceipt = Durable; +pub type DurableEnvironmentValues = Durable; +pub type DurableClosedReceipt = Durable; + +pub async fn capture( + db: Arc, + scope: EnvironmentSnapshotScope, +) -> Result { + let action_db = db.clone(); + mutate(db, move |tx| storage::capture(&action_db, tx, &scope)).await +} + +pub async fn read( + db: Arc, + receipt: EnvironmentSnapshotReceipt, +) -> Result { + read_with_capacity(db, receipt, OPERATIONS.clone()).await +} + +async fn read_with_capacity( + db: Arc, + receipt: EnvironmentSnapshotReceipt, + capacity: Arc, +) -> Result { + let mut durability = db + .durable_tx_offset() + .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| EnvironmentSnapshotError::Capacity)?; + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_tx(Workload::Internal); + let result = storage::read(&action_db, &tx, &receipt); + let (offset, metrics, reducer) = action_db.release_tx(tx); + action_db.report_read_tx_metrics(reducer, metrics); + result.map(|result| (offset, result)) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)? +} + +pub async fn close( + db: Arc, + scope: EnvironmentSnapshotScope, + closed_through_generation: u64, +) -> Result { + let action_db = db.clone(); + mutate(db, move |tx| { + storage::close(&action_db, tx, &scope, closed_through_generation) + }) + .await +} + +async fn mutate( + db: Arc, + action: impl FnOnce(&mut MutTx) -> Result + Send + 'static, +) -> Result, EnvironmentSnapshotError> { + mutate_with_capacity(db, OPERATIONS.clone(), action).await +} + +async fn mutate_with_capacity( + db: Arc, + capacity: Arc, + action: impl FnOnce(&mut MutTx) -> Result + Send + 'static, +) -> Result, EnvironmentSnapshotError> { + let mut durability = db + .durable_tx_offset() + .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| EnvironmentSnapshotError::Capacity)?; + // A cancelled waiter drops only this JoinHandle. The actual operation + // keeps its database and finite slot until commit and durability finish. + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, result) = action_db.with_auto_rollback(tx, action)?; + let (offset, data, metrics, reducer) = action_db + .commit_tx(tx) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .ok_or(EnvironmentSnapshotError::Storage)?; + action_db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, EnvironmentSnapshotError>((offset, result)) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)? +} + +#[cfg(test)] +#[path = "container_environment/durability_tests.rs"] +mod durability_tests; + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use std::time::Duration; + + #[test] + fn container_environment_cancelled_transaction_retains_capacity_until_worker_finishes() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let (entered, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release, release_rx) = std::sync::mpsc::sync_channel(1); + let blocked = db + .runtime() + .unwrap() + .spawn(mutate_with_capacity(db.db.clone(), capacity.clone(), move |_| { + entered.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Err::<(), _>(EnvironmentSnapshotError::InvalidEnvironment) + })); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + blocked.abort(); + db.runtime().unwrap().block_on(async { + assert!(blocked.await.unwrap_err().is_cancelled()); + let retry = mutate_with_capacity(db.db.clone(), capacity.clone(), |_| Ok(())).await; + assert!(matches!(retry, Err(EnvironmentSnapshotError::Capacity))); + }); + release.send(()).unwrap(); + db.runtime().unwrap().block_on(async { + // This waits for the actual cancelled caller's blocking worker to + // return, not merely for cancellation of its async JoinHandle. + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + mutate_with_capacity(db.db.clone(), capacity, |_| Ok(())).await.unwrap(); + }); + } +} diff --git a/crates/core/src/host/container_environment/durability_tests.rs b/crates/core/src/host/container_environment/durability_tests.rs new file mode 100644 index 00000000000..c72ab9435db --- /dev/null +++ b/crates/core/src/host/container_environment/durability_tests.rs @@ -0,0 +1,288 @@ +//! Real local writes with only their durability acknowledgment delayed. +use super::*; +use crate::db::deployment::{ + install_container_fence, install_publication_fence, record_deployment_commit, DeploymentCommit, +}; +use crate::db::relational_db::{ + local_durability, + tests_utils::{TempReplicaDir, TestDB}, + LocalDurability, +}; +use crate::db::{environment, persistence::Persistence}; +use spacetimedb_datastore::system_tables::StContainerFenceRow; +use spacetimedb_durability::{Close, Durability, DurableOffset, PreparedTx}; +use spacetimedb_lib::container::*; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; +use spacetimedb_lib::{hash_bytes, Timestamp, Uuid}; +use std::time::Duration; + +fn uuid() -> Uuid { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) +} + +fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: keys, + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } + .normalize(&Default::default()) + .unwrap(); + let request = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: db.owner_identity(), + expected_revision: None, + prepared_manifest_hash: hash_bytes(b"prepared"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: Some(spec.clone()), + }), + }; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + record_deployment_commit(tx, &request, Timestamp::now(), &Default::default())?; + install_container_fence(db, tx, &self_fence(db, 1, true))?; + for key in &spec.env_keys { + environment::set(db, tx, key, "before")?; + } + Ok(()) + }) + .unwrap(); + EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: 1, + database_identity: db.database_identity(), + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: request.deployment.revision().unwrap(), + start_request: request.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + } +} + +fn self_fence(db: &RelationalDB, generation: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed, + } +} + +struct DelayedAcknowledgment { + writer: LocalDurability, + acknowledged: DurableOffset, +} +impl Durability for DelayedAcknowledgment { + type TxData = crate::db::relational_db::Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.writer.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.acknowledged.clone() + } + fn close(&self) -> Close { + self.writer.close() + } +} + +struct Fixture { + db: Arc, + writer: LocalDurability, + acknowledge: tokio::sync::watch::Sender>, + _directory: TempReplicaDir, +} +impl Fixture { + async fn new() -> Self { + let directory = TempReplicaDir::new().unwrap(); + let (writer, disk_size) = local_durability((*directory).clone(), None).await.unwrap(); + let (acknowledge, acknowledged) = tokio::sync::watch::channel(None); + let db = Arc::new( + TestDB::open_db( + writer.as_history(), + Some(Persistence { + durability: Arc::new(DelayedAcknowledgment { + writer: writer.clone(), + acknowledged: acknowledged.into(), + }), + disk_size, + snapshots: None, + runtime: tokio::runtime::Handle::current(), + }), + None, + 0, + ) + .unwrap(), + ); + Self { + db, + writer, + acknowledge, + _directory: directory, + } + } + async fn acknowledge_current(&self) -> u64 { + let tx = self.db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = self.db.release_tx(tx); + self.db.report_read_tx_metrics(reducer, metrics); + let mut actual = self.writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + self.acknowledge.send_replace(Some(durable)); + offset + } + async fn finish(self) { + self.db.shutdown().await; + drop(self.db); + self.writer.close().await; + } +} + +async fn wait_for_slot(capacity: &Arc) { + tokio::time::timeout(Duration::from_secs(5), async { + while capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +async fn released(capacity: &Arc) { + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_capture_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let capacity = Arc::new(Semaphore::new(1)); + let action_db = db.clone(); + let (captured, result) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(mutate_with_capacity(db.clone(), capacity.clone(), move |tx| { + let receipt = storage::capture(&action_db, tx, &scope)?; + captured.send(receipt.clone()).unwrap(); + Ok(receipt) + })); + let captured = tokio::time::timeout(Duration::from_secs(5), result) + .await + .unwrap() + .unwrap(); + // This read waits for the actual capture transaction to commit. + db.with_read_only(Workload::ForTests, |tx| storage::read(&db, tx, &captured)) + .unwrap(); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + mutate_with_capacity(db.clone(), capacity.clone(), |_| Ok(())).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + let values = read(db.clone(), captured).await.unwrap(); + assert_eq!(values.receipt.selected_values["SECRET"], "before"); + drop(db); + fixture.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_read_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let receipt = db + .with_auto_commit(Workload::ForTests, |tx| storage::capture(&db, tx, &scope)) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(read_with_capacity(db.clone(), receipt.clone(), capacity.clone())); + wait_for_slot(&capacity).await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + read_with_capacity(db.clone(), receipt.clone(), capacity.clone()).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + let values = read_with_capacity(db.clone(), receipt, capacity).await.unwrap(); + assert_eq!(values.receipt.selected_values["SECRET"], "before"); + drop(db); + fixture.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_close_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let receipt = db + .with_auto_commit(Workload::ForTests, |tx| storage::capture(&db, tx, &scope)) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &self_fence(&db, 2, false)) + }) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let action_db = db.clone(); + let (closed, result) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(mutate_with_capacity(db.clone(), capacity.clone(), move |tx| { + let receipt = storage::close(&action_db, tx, &scope, 2)?; + closed.send(()).unwrap(); + Ok(receipt) + })); + tokio::time::timeout(Duration::from_secs(5), result) + .await + .unwrap() + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + assert_eq!( + tx.table_row_count(spacetimedb_datastore::system_tables::ST_CONTAINER_ENVIRONMENT_ID), + Some(0) + ); + }); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + mutate_with_capacity(db.clone(), capacity.clone(), |_| Ok(())).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + assert!(matches!( + read(db.clone(), receipt).await, + Err(EnvironmentSnapshotError::Fenced) + )); + drop(db); + fixture.finish().await; +} diff --git a/crates/core/src/host/container_fence.rs b/crates/core/src/host/container_fence.rs new file mode 100644 index 00000000000..4b872fbcea1 --- /dev/null +++ b/crates/core/src/host/container_fence.rs @@ -0,0 +1,204 @@ +//! Bounded host-only inspection and durable denial of retained source fences. +//! +//! The authenticated platform coordinator owns control membership checks and +//! actor drainage. These APIs do not authorize an external client or expose the +//! protected fence table through SQL, subscriptions, or module syscalls. + +use crate::db::deployment::{deny_container_fence, DeploymentError, FenceDenial}; +use crate::db::relational_db::RelationalDB; +use crate::host::container_environment::Durable; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::ScanOrIndex; +use spacetimedb_datastore::system_tables::{StContainerFenceRow, ST_CONTAINER_FENCE_ID}; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::Identity; +use spacetimedb_primitives::ColId; +use spacetimedb_sats::AlgebraicValue; +use std::ops::Bound; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Semaphore; + +pub const FENCE_PAGE_SIZE: usize = 64; +static OPERATIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(8))); + +#[derive(Debug, thiserror::Error)] +pub enum FenceOperationError { + #[error("receiving host fence operation capacity exhausted")] + Capacity, + #[error("receiving host fence storage is unavailable")] + Storage, + #[error("receiving host fence durability is unavailable")] + DurabilityUnavailable, + #[error("receiving host fence durability failed")] + DurabilityFailed, + #[error("receiving host fence index is unavailable")] + IndexUnavailable, + #[error("receiving host fence inventory changed")] + RevisionChanged, + #[error(transparent)] + Deployment(#[from] DeploymentError), +} + +#[derive(Debug)] +pub struct FencePage { + /// Includes denied rows so every bounded physical page advances its cursor. + pub rows: Vec, + /// Last source in this page, or the input cursor if the page is empty. + pub next_source: Option, + pub complete: bool, + /// Read under the same database transaction as the rows. Restart the scan + /// if this differs from the previous page or a known local denial result. + pub fence_revision: u64, +} + +/// Read at most 64 rows using the protected source Identity B-tree. We refuse +/// the datastore's scan fallback: row order and physical work must be bounded. +pub async fn page(db: Arc, after: Option) -> Result { + let permit = OPERATIONS + .clone() + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let tx = db.begin_tx(Workload::Internal); + let result = (|| { + let lower = after.map_or(Bound::Unbounded, |identity| { + Bound::Excluded(AlgebraicValue::U256(identity.to_u256().into())) + }); + let range = db + .iter_by_col_range(&tx, ST_CONTAINER_FENCE_ID, ColId(0), (lower, Bound::Unbounded)) + .map_err(|_| FenceOperationError::Storage)?; + let ScanOrIndex::Index(mut range) = range else { + return Err(FenceOperationError::IndexUnavailable); + }; + let rows = range + .by_ref() + .take(FENCE_PAGE_SIZE) + .map(|row| StContainerFenceRow::try_from(row).map_err(|_| FenceOperationError::Storage)) + .collect::, _>>()?; + let complete = range.next().is_none(); + let next_source = rows.last().map(|row| row.source_identity.into()).or(after); + Ok(FencePage { + rows, + next_source, + complete, + fence_revision: db.hosted_admission().fence_revision(), + }) + })(); + let (_, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + result + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +/// The coordinator must have excluded this exact source from current control +/// authority. For a check that must serialize with an inventory lock, use the +/// synchronous `deployment::deny_container_fence` inside the caller's own +/// serializable transaction instead. Neither API performs that control check. +pub async fn deny_orphan( + db: Arc, + expected: StContainerFenceRow, +) -> Result, FenceOperationError> { + deny_with_capacity(db, expected, OPERATIONS.clone()).await +} + +/// Confirm durability of all fences visible at one exact physical revision. +/// In particular, a retry that sees a previous owner's committed denial must +/// still wait for that denial's storage acknowledgment. The coordinator must +/// recheck the revision under its final database transaction before admission; +/// this receipt does not freeze future mutations or confer control authority. +pub async fn confirm_revision( + db: Arc, + expected_physical: u64, +) -> Result, FenceOperationError> { + confirm_with_capacity(db, expected_physical, OPERATIONS.clone()).await +} + +async fn confirm_with_capacity( + db: Arc, + expected_physical: u64, + capacity: Arc, +) -> Result, FenceOperationError> { + let mut durability = db + .durable_tx_offset() + .ok_or(FenceOperationError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let durable_through = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_tx(Workload::Internal); + let physical = action_db.hosted_admission().fence_revision(); + let (offset, metrics, reducer) = action_db.release_tx(tx); + action_db.report_read_tx_metrics(reducer, metrics); + if physical != expected_physical || physical == u64::MAX { + return Err(FenceOperationError::RevisionChanged); + } + Ok(offset) + }) + .await + .map_err(|_| FenceOperationError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| FenceOperationError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt: (), + durable_through, + }) + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +async fn deny_with_capacity( + db: Arc, + expected: StContainerFenceRow, + capacity: Arc, +) -> Result, FenceOperationError> { + let mut durability = db + .durable_tx_offset() + .ok_or(FenceOperationError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + // A caller's cancellation drops only this JoinHandle. The owned task keeps + // both the permit and database alive through physical commit and durability. + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, receipt) = + action_db.with_auto_rollback(tx, |tx| deny_container_fence(&action_db, tx, &expected))?; + let (offset, data, metrics, reducer) = action_db + .commit_tx(tx) + .map_err(|_| FenceOperationError::Storage)? + .ok_or(FenceOperationError::Storage)?; + action_db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, FenceOperationError>((offset, receipt)) + }) + .await + .map_err(|_| FenceOperationError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| FenceOperationError::DurabilityFailed)?; + // Do not drop the physical database before the owned durability wait. + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/host/container_fence/tests.rs b/crates/core/src/host/container_fence/tests.rs new file mode 100644 index 00000000000..fdd641fef88 --- /dev/null +++ b/crates/core/src/host/container_fence/tests.rs @@ -0,0 +1,389 @@ +use super::*; +use crate::db::deployment::{check_container_fence, install_container_fence}; +use crate::db::relational_db::tests_utils::TestDB; +use spacetimedb_lib::hash_bytes; +use std::time::Duration; + +fn fence(source: u64) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: Identity::from_u256(source.into()).into(), + generation: 7, + target_grant_revision: 3, + target_set_hash: hash_bytes(b"retained target set"), + allowed: true, + } +} + +fn install(db: &RelationalDB, row: &StContainerFenceRow) { + db.with_auto_commit(Workload::ForTests, |tx| install_container_fence(db, tx, row)) + .unwrap(); +} + +#[tokio::test] +async fn container_fence_pages_follow_identity_index_and_include_denied_rows() { + let db = TestDB::in_memory().unwrap(); + // Reverse insertion order deliberately differs from source index order. + for source in (0..130).rev() { + install( + &db, + &StContainerFenceRow { + allowed: source % 3 == 0, + ..fence(source) + }, + ); + } + let revision = db.hosted_admission().fence_revision(); + let mut after = None; + let mut identities = Vec::new(); + for expected_size in [64, 64, 2] { + let page = page(db.db.clone(), after).await.unwrap(); + assert_eq!(page.rows.len(), expected_size); + assert_eq!(page.complete, expected_size == 2); + assert_eq!(page.fence_revision, revision); + identities.extend(page.rows.iter().map(|row| Identity::from(row.source_identity))); + after = page.next_source; + assert_eq!(after, identities.last().copied()); + } + assert_eq!( + identities, + (0..130u64).map(|n| Identity::from_u256(n.into())).collect::>() + ); + let empty = page(db.db.clone(), after).await.unwrap(); + assert!(empty.rows.is_empty() && empty.complete); + assert_eq!(empty.next_source, after); +} + +#[tokio::test] +async fn container_fence_missing_index_refuses_unbounded_scan_and_memory_refuses_denial() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(17); + install(&db, &expected); + assert!(matches!( + deny_orphan(db.db.clone(), expected).await, + Err(FenceOperationError::DurabilityUnavailable) + )); + db.with_auto_commit(Workload::ForTests, |tx| { + db.drop_index(tx, spacetimedb_primitives::IndexId(34)) + }) + .unwrap(); + assert!(matches!( + page(db.db.clone(), None).await, + Err(FenceOperationError::IndexUnavailable) + )); +} + +#[test] +fn container_fence_orphan_denial_is_durable_exact_and_cannot_reopen_generation() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let expected = fence(19); + install(&db, &expected); + db.hosted_admission().begin().unwrap().complete().unwrap(); + let before = db.hosted_admission().fence_revision(); + let first = db + .runtime() + .unwrap() + .block_on(deny_orphan(db.db.clone(), expected.clone())) + .unwrap(); + assert_eq!(first.receipt.revision_before, before); + assert_eq!(first.receipt.revision_after, before + 1); + assert!(!db.hosted_admission().is_open()); + assert_eq!( + first.receipt.row, + StContainerFenceRow { + allowed: false, + ..expected.clone() + } + ); + assert!(db.durable_tx_offset().unwrap().get().unwrap().unwrap() >= first.durable_through); + let retry = db + .runtime() + .unwrap() + .block_on(deny_orphan(db.db.clone(), expected.clone())) + .unwrap(); + assert_eq!(retry.receipt.revision_before, before + 1); + assert_eq!(retry.receipt.revision_after, before + 1); + let db = db.reopen().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + assert!(matches!( + check_container_fence( + tx, + expected.source_identity.into(), + expected.generation, + expected.target_grant_revision + ), + Err(DeploymentError::ContainerFenced) + )); + assert!(matches!( + install_container_fence(&db, tx, &expected), + Err(DeploymentError::FenceConflict) + )); + let retry = deny_container_fence(&db, tx, &expected)?; + assert_eq!(retry.row, first.receipt.row); + assert_eq!(retry.revision_before, retry.revision_after); + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + install( + &db, + &StContainerFenceRow { + generation: expected.generation + 1, + ..expected + }, + ); +} + +#[test] +fn container_fence_orphan_cas_rejects_missing_changed_and_invalid_observations() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(20); + db.with_auto_commit(Workload::ForTests, |tx| { + assert!(matches!( + deny_container_fence(&db, tx, &expected), + Err(DeploymentError::FenceConflict) + )); + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + install(&db, &expected); + let next = StContainerFenceRow { + generation: expected.generation + 1, + ..expected.clone() + }; + install(&db, &next); + let revision = db.hosted_admission().fence_revision(); + db.with_auto_commit(Workload::ForTests, |tx| { + for stale in [ + expected, + StContainerFenceRow { + allowed: false, + ..next.clone() + }, + StContainerFenceRow { + target_set_hash: hash_bytes(b"different"), + ..next.clone() + }, + StContainerFenceRow { + target_grant_revision: 4, + ..next.clone() + }, + ] { + assert!(matches!( + deny_container_fence(&db, tx, &stale), + Err(DeploymentError::FenceConflict) + )); + } + check_container_fence( + tx, + next.source_identity.into(), + next.generation, + next.target_grant_revision, + )?; + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + assert_eq!(db.hosted_admission().fence_revision(), revision); +} + +#[tokio::test] +async fn container_fence_physical_change_behind_cursor_invalidates_completion() { + let db = TestDB::in_memory().unwrap(); + install(&db, &fence(100)); + let ticket = db.hosted_admission().begin().unwrap(); + let first = page(db.db.clone(), None).await.unwrap(); + assert!(first.complete); + install(&db, &fence(1)); + assert!(ticket.complete_with_fence_revision(first.fence_revision).is_err()); + assert!(!db.hosted_admission().is_open()); + let restarted = page(db.db.clone(), None).await.unwrap(); + assert_eq!(restarted.rows.len(), 2); + assert_ne!(first.fence_revision, restarted.fence_revision); +} + +#[test] +fn container_fence_orphan_rollback_keeps_row_and_invalidates_physical_scan() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(21); + install(&db, &expected); + let before = db.hosted_admission().fence_revision(); + let ticket = db.hosted_admission().begin().unwrap(); + let result = db.with_auto_commit(Workload::ForTests, |tx| { + deny_container_fence(&db, tx, &expected)?; + Err::<(), _>(DeploymentError::CorruptMetadata) + }); + assert!(result.is_err()); + assert!(ticket.complete_with_fence_revision(before).is_err()); + db.with_auto_commit(Workload::ForTests, |tx| { + check_container_fence( + tx, + expected.source_identity.into(), + expected.generation, + expected.target_grant_revision, + ) + }) + .unwrap(); +} + +/// Real local storage with its durable acknowledgment deliberately withheld. +/// This models a lagging quorum without replacing the transaction or writer. +struct DelayedAcknowledgment { + writer: Arc>, + acknowledged: spacetimedb_durability::DurableOffset, +} + +impl spacetimedb_durability::Durability for DelayedAcknowledgment { + type TxData = crate::db::relational_db::Txdata; + fn append_tx(&self, tx: spacetimedb_durability::PreparedTx) { + self.writer.append_tx(tx); + } + fn durable_tx_offset(&self) -> spacetimedb_durability::DurableOffset { + self.acknowledged.clone() + } + fn close(&self) -> spacetimedb_durability::Close { + self.writer.close() + } +} + +async fn delayed_storage() -> ( + crate::db::relational_db::tests_utils::TempReplicaDir, + Arc, + crate::db::relational_db::LocalDurability, + tokio::sync::watch::Sender>, +) { + use crate::db::persistence::Persistence; + use crate::db::relational_db::{local_durability, tests_utils::TempReplicaDir}; + let directory = TempReplicaDir::new().unwrap(); + let (writer, disk_size) = local_durability((*directory).clone(), None).await.unwrap(); + let (acknowledge, acknowledged) = tokio::sync::watch::channel(None); + let db = Arc::new( + TestDB::open_db( + writer.as_history(), + Some(Persistence { + durability: Arc::new(DelayedAcknowledgment { + writer: writer.clone(), + acknowledged: acknowledged.into(), + }), + disk_size, + snapshots: None, + runtime: tokio::runtime::Handle::current(), + }), + None, + 0, + ) + .unwrap(), + ); + (directory, db, writer, acknowledge) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_fence_cancelled_denial_retains_capacity_through_actual_durability_wait() { + use spacetimedb_durability::Durability; + let (_directory, db, writer, acknowledge) = delayed_storage().await; + let expected = fence(22); + install(&db, &expected); + let capacity = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(deny_with_capacity(db.clone(), expected.clone(), capacity.clone())); + let row = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let page = page(db.clone(), None).await.unwrap(); + if let Some(row) = page.rows.first().filter(|row| !row.allowed) { + break row.clone(); + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!row.allowed); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + deny_with_capacity(db.clone(), expected.clone(), capacity.clone()).await, + Err(FenceOperationError::Capacity) + )); + let tx = db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + let mut actual = writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + acknowledge.send_replace(Some(durable)); + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + // The detached owner finished only once its physical acknowledgment arrived. + db.shutdown().await; + drop(db); + writer.close().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_fence_retry_barrier_waits_for_prior_commit_and_retains_cancelled_capacity() { + use spacetimedb_durability::Durability; + let (_directory, db, writer, acknowledge) = delayed_storage().await; + let expected = fence(23); + install(&db, &expected); + // A previous coordinator has committed but never confirmed durability. + let denied = db + .with_auto_commit(Workload::ForTests, |tx| deny_container_fence(&db, tx, &expected)) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + // Changed inventory fails immediately, even while acknowledgments lag. + assert!(matches!( + tokio::time::timeout( + Duration::from_secs(5), + confirm_with_capacity(db.clone(), denied.revision_before, capacity.clone()) + ) + .await + .unwrap(), + Err(FenceOperationError::RevisionChanged) + )); + let caller = tokio::spawn(confirm_with_capacity( + db.clone(), + denied.revision_after, + capacity.clone(), + )); + tokio::time::timeout(Duration::from_secs(5), async { + while capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!caller.is_finished()); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert!(matches!( + confirm_with_capacity(db.clone(), denied.revision_after, capacity.clone()).await, + Err(FenceOperationError::Capacity) + )); + let tx = db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + let mut actual = writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + acknowledge.send_replace(Some(durable)); + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + let proof = confirm_revision(db.clone(), denied.revision_after).await.unwrap(); + assert_eq!(proof.durable_through, offset); + // The proof is only a barrier, so finalization still checks current revision. + install(&db, &fence(24)); + assert!(matches!( + confirm_revision(db.clone(), denied.revision_after).await, + Err(FenceOperationError::RevisionChanged) + )); + db.shutdown().await; + drop(db); + writer.close().await; +} diff --git a/crates/core/src/host/empty_module.rs b/crates/core/src/host/empty_module.rs new file mode 100644 index 00000000000..c0310ac8628 --- /dev/null +++ b/crates/core/src/host/empty_module.rs @@ -0,0 +1,47 @@ +//! Versioned built-in module for a database published with only a container. +//! +//! This is a real Wasm program, not an empty byte string. It declares an empty +//! V10 user schema and hosted_auth_v1, so normal database initialization, system +//! tables, subscriptions, and later migration use the existing host machinery. +//! Its required reducer ABI entry point traps because no reducer is declared. +//! +//! Version 1 bytes are immutable. Reproduce them with +//! `python3 crates/core/src/host/empty_module/generate.py --check` from public/. +//! New program bytes require a new version, preserving old deployment replay. + +use spacetimedb_datastore::{system_tables::ModuleKind, traits::Program}; +use spacetimedb_lib::{hash_bytes, Hash}; +use std::sync::OnceLock; + +pub const VERSION_1: u32 = 1; +pub use spacetimedb_lib::deployment::SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH as VERSION_1_PROGRAM_HASH; + +pub use spacetimedb_lib::deployment::SYSTEM_EMPTY_MODULE_V1_BYTES as VERSION_1_BYTES; + +/// Return the exact bundled program for a recognized system module version. +/// Unknown versions fail closed instead of silently selecting the latest one. +pub fn program(version: u32) -> Option { + (version == VERSION_1 && v1_hash() == VERSION_1_PROGRAM_HASH).then(|| Program { + hash: v1_hash(), + bytes: VERSION_1_BYTES.into(), + kind: ModuleKind::WASM, + }) +} + +/// Validate a system-empty deployment against immutable platform bytes, not +/// against an empty-looking schema supplied by a publisher or a claimed hash. +pub fn matches_program(version: u32, candidate: &Program) -> bool { + version == VERSION_1 + && candidate.kind == ModuleKind::WASM + && candidate.hash == VERSION_1_PROGRAM_HASH + && v1_hash() == VERSION_1_PROGRAM_HASH + && candidate.bytes.as_ref() == VERSION_1_BYTES +} + +fn v1_hash() -> Hash { + static HASH: OnceLock = OnceLock::new(); + *HASH.get_or_init(|| hash_bytes(VERSION_1_BYTES)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/host/empty_module/README.md b/crates/core/src/host/empty_module/README.md new file mode 100644 index 00000000000..4db85e2fdd1 --- /dev/null +++ b/crates/core/src/host/empty_module/README.md @@ -0,0 +1,23 @@ +# System empty module, version 1 + +The host uses this real Wasm module for a database published with a container and no user module. It declares V10 with an empty typespace and `hosted_auth_v1`; there are no user tables, reducers, procedures, views, schedules, or HTTP handlers. + +`generate.py` is the complete source. It uses only the Python standard library and explicitly encodes the Wasm sections and BSATN description. It produces a 250-byte Wasm binary with one fixed memory page, a schema describer, and the required reducer ABI entry point. The reducer entry point always traps because no reducer is declared. The `spacetime_10.6::get_call_auth_flags` import requires the host ABI corresponding to the advertised capability; the empty module has no invocation contexts to construct. + +From the public repository, reproduce and verify the checked-in files with: + +```sh +python3 crates/core/src/host/empty_module/generate.py --check +``` + +Omit `--check` to regenerate the three files. This does not require a Rust module compiler, WASI toolchain, WAT compiler, or container image builder. + +The SHA-256 checksum of `v1.wasm` is recorded in `v1.sha256`. SpacetimeDB's separate Keccak-256 program identity is: + +```text +08365097d1ef202653615f02cc46be16088044c6b9cd9660e3a0f336e47918f0 +``` + +Version 1 is immutable. A change to its schema or Wasm requires a new version and new files. Keep version 1 available for existing deployments and replay. The Rust helper checks the known program hash as well as the exact bytes, kind, and version, so a different publisher-supplied module with an empty-looking schema cannot qualify as the system empty module. `Program::empty` is also rejected. + +Core tests compare the BSATN fixture against the current V10 Rust wire types, load the Wasm through the actual host, and initialize a real database through `HostController`. They verify that initialization stores the bundled program and the database's metadata without invoking a user reducer. diff --git a/crates/core/src/host/empty_module/generate.py b/crates/core/src/host/empty_module/generate.py new file mode 100644 index 00000000000..6400d46db9d --- /dev/null +++ b/crates/core/src/host/empty_module/generate.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Reproduce the version-1 system empty module using only Python's standard library. + +The binary has one page of memory, a V10 schema describer, and a reducer entry +point which always traps. There are no declared user tables or functions. + +Equivalent code (the data payload below is BSATN, not WebAssembly encoding): + (import "spacetime_10.0" "bytes_sink_write" (func (param i32 i32 i32) (result i32))) + (import "spacetime_10.6" "get_call_auth_flags" (func (result i32))) + (memory (export "memory") 1 1) + (func (export "__describe_module__") (param i32) + local.get 0 i32.const 32 i32.const 16 call 0 + if unreachable end) + (func (export "__call_reducer__") + (param i32 i64 i64 i64 i64 i64 i64 i64 i32 i32) (result i32) + unreachable) + +The unused 10.6 import explicitly requires the host ABI which supports captured +invocation flags. No function or invocation context exists in this module. +""" + +import argparse +import hashlib +from pathlib import Path +import struct + + +def leb(value): + encoded = bytearray() + while value >= 128: + encoded.append((value & 127) | 128) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def string(value): + value = value.encode("utf-8") + return leb(len(value)) + value + + +def vector(items): + return leb(len(items)) + b"".join(items) + + +def section(tag, payload): + return bytes([tag]) + leb(len(payload)) + payload + + +def function_type(params, results): + return b"\x60" + vector(params) + vector(results) + + +def generate(): + u32 = lambda value: struct.pack(">>; -/// The registry of all running hosts. -type Hosts = Arc>>; +mod lifecycle; +mod registry; +use registry::{Hosts, Registration}; + +#[cfg(test)] +mod lifecycle_tests; + +#[cfg(test)] +mod deployment_tests; + +#[cfg(test)] +mod execution_deadline_tests; + +#[cfg(test)] +static FAIL_NEXT_DEPLOYMENT_ACTIVATION: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); pub type ExternalDurability = (Arc>, DiskSizeFn); #[async_trait] pub trait ExternalStorage: Send + Sync + 'static { async fn lookup(&self, program_hash: Hash) -> anyhow::Result>>; + + /// Resolve the authorized initial deployment from the same control + /// transaction that created the database. A failed lookup must return an + /// error, never None: None is reserved for legacy module-only databases. + /// Called only when no initialized program exists in the user database. + async fn initial_deployment(&self, _database: &Database) -> anyhow::Result> { + Ok(None) + } } #[async_trait] impl ExternalStorage for F @@ -333,10 +357,10 @@ impl HostController { // Note that `tokio::spawn` only cancels its tasks when the runtime shuts down, // at which point we won't be calling `try_init_host` again anyways. let rx = tokio::spawn(async move { - let host = this.try_init_host(database, replica_id).await?; + let host = this.try_init_host(database, replica_id, &guard).await?; let rx = host.module.subscribe(); - *guard = Some(host); + guard.install(host); Ok::<_, anyhow::Error>(rx) }) @@ -395,6 +419,23 @@ impl HostController { replica_id: u64, program_bytes: Box<[u8]>, policy: MigrationPolicy, + ) -> anyhow::Result { + self.update_module_host_with_deployment(database, host_type, replica_id, program_bytes, policy, None) + .await + } + + /// Authorized coordinator entry point. The deployment and its operation + /// receipt are committed in the module migration transaction. Process + /// startup and the control database mirror remain separate reconciliation. + #[allow(clippy::too_many_arguments)] + pub async fn update_module_host_with_deployment( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { let program = Program::from_bytes(host_type.into(), program_bytes); trace!( @@ -431,27 +472,57 @@ impl HostController { let mut host = match guard.take() { None => { trace!("host not running, try_init"); - this.try_init_host(database, replica_id).await? + this.try_init_host(database, replica_id, &guard).await? } Some(host) => { trace!("host found, updating"); host } }; - let update_result = host - .update_module( - this.runtimes.clone(), - program, - policy, - this.energy_monitor.clone(), - this.unregister_fn(replica_id), - this.db_cores.take(), - ) - .await?; - - *guard = Some(host); - - Ok::<_, anyhow::Error>(update_result) + let mut database_committed = false; + let registration = guard.registration(); + let update_result = std::panic::AssertUnwindSafe(host.update_module( + this.runtimes.clone(), + program, + policy, + deployment, + this.energy_monitor.clone(), + this.unregister_fn(registration.clone()), + registration, + this.db_cores.take(), + &mut database_committed, + )) + .catch_unwind() + .await; + let update_result = match update_result { + Ok(result) => result, + Err(panic) => { + if let Err(error) = lifecycle::close_host(host).await { + if matches!(error, lifecycle::CloseFailure::WriterUnconfirmed) { + guard.quarantine(); + } + return Err(error.into()); + } + std::panic::resume_unwind(panic); + } + }; + + if update_result.is_err() && database_committed { + // Schema/program/receipt already committed. The previous + // executable cannot be retained after activation failure. + // Close clients/scheduler and reconstruct from stored program + // on the next leader lookup or reconciliation attempt. + if let Err(error) = lifecycle::close_host(host).await { + if matches!(error, lifecycle::CloseFailure::WriterUnconfirmed) { + guard.quarantine(); + } + return Err(error.into()); + } + } else { + guard.install(host); + } + + update_result }) .await??; @@ -496,58 +567,22 @@ impl HostController { /// Release all resources of the [`ModuleHost`] identified by `replica_id`, /// and deregister it from the controller. + /// + /// A timeout returns an error while the owned close continues. Only success + /// confirms the configured storage writer has completed shutdown. #[tracing::instrument(level = "trace", skip_all)] pub async fn exit_module_host(&self, replica_id: u64, timeout: Duration) -> Result<(), anyhow::Error> { - let Some(lock) = self.hosts.lock().remove(&replica_id) else { + let Some(request) = registry::close(&self.hosts, replica_id) else { return Ok(()); }; - // To debug the potential deadlock issue reported in - // https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - // we'll log a warning every 5s if we can't acquire an exclusive lock. - let start = Instant::now(); - let mut t = interval_at(start + Duration::from_secs(5), Duration::from_secs(5)); - let warn_blocked = tokio::spawn(async move { - loop { - t.tick().await; - warn!( - "blocked waiting to exit module for replica {} since {}s", - replica_id, - start.elapsed().as_secs_f32() - ); - } - }); - defer!(warn_blocked.abort()); - - let shutdown = tokio::time::timeout(timeout, async { - let mut guard = lock.write_owned().await; - let Some(host) = guard.take() else { - return; - }; - let module = host.module.borrow().clone(); - let info = module.info(); - - let database_identity = info.database_identity; - let table_names = info.module_def.tables().map(|t| t.name.deref()); - - // Ensure we clear the metrics even if the future is cancelled. - defer!(remove_database_gauges(&database_identity, table_names)); - - info!("replica={replica_id} database={database_identity} exiting module"); - module.exit().await; - info!("replica={replica_id} database={database_identity} exiting database"); - module.relational_db().shutdown().await; - info!("replica={replica_id} database={database_identity} module host exited"); - }) - .await; - - if shutdown.is_err() { - warn!( - "replica={replica_id} shutdown timed out after {}s", - start.elapsed().as_secs_f32() - ); + if let Some(owner) = request.owner { + // The close owns its registry pin independently of this waiter. + // Cancellation or timeout cannot release a still-running writer. + tokio::spawn(owner.run()); } - - Ok(()) + tokio::time::timeout(timeout, request.completion.wait()) + .await + .map_err(|_| anyhow!("replica {replica_id} shutdown is still pending after {timeout:?}"))? } /// Get the [`ModuleHost`] identified by `replica_id` or return an error @@ -604,41 +639,40 @@ impl HostController { self.hosts.lock().keys().copied().collect() } - /// On-panic callback passed to [`ModuleHost`]s created by this controller. - /// - /// Removes the module with the given `replica_id` from this controller. - fn unregister_fn(&self, replica_id: u64) -> impl Fn() + Send + Sync + 'static + use<> { - let hosts = Arc::downgrade(&self.hosts); + /// On-panic callbacks are scoped to one installed executable and cell. + /// A stale callback cannot unregister an updated or reopened database. + fn unregister_fn(&self, registration: Registration) -> impl Fn() + Send + Sync + 'static + use<> { + let runtime = tokio::runtime::Handle::current(); move || { - if let Some(hosts) = hosts.upgrade() { - hosts.lock().remove(&replica_id); + if let Some(request) = registration.close_if_current() + && let Some(owner) = request.owner + { + runtime.spawn(owner.run()); } } } - /// Acquire a write lock on the [HostCell] for `replica_id`. - /// - /// This will time out after 5s to aid debugging of - /// https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - async fn acquire_write_lock(&self, replica_id: u64) -> Result>, Elapsed> { - let lock = self.hosts.lock().entry(replica_id).or_default().clone(); - timeout(Duration::from_secs(5), lock.write_owned()).await + /// The total wait includes any ongoing close and cell reacquisition. + async fn acquire_write_lock(&self, replica_id: u64) -> anyhow::Result { + timeout(Duration::from_secs(5), registry::Pin::write(&self.hosts, replica_id)).await? } - /// Acquire a read lock on the [HostCell] for `replica_id`. - /// - /// This will time out after 5s to aid debugging of - /// https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - async fn acquire_read_lock(&self, replica_id: u64) -> Result>, Elapsed> { - let lock = self.hosts.lock().entry(replica_id).or_default().clone(); - timeout(Duration::from_secs(5), lock.read_owned()).await + async fn acquire_read_lock(&self, replica_id: u64) -> anyhow::Result { + timeout(Duration::from_secs(5), registry::Pin::read(&self.hosts, replica_id)).await? } - async fn try_init_host(&self, database: Database, replica_id: u64) -> anyhow::Result { + async fn try_init_host( + &self, + database: Database, + replica_id: u64, + guard: ®istry::WriteGuard, + ) -> anyhow::Result { let database_identity = database.database_identity; - Host::try_init(self, database, replica_id) - .await - .with_context(|| format!("failed to init replica {} for {}", replica_id, database_identity)) + let result = Host::try_init(self, database, replica_id, guard.registration()).await; + if result.as_ref().is_err_and(lifecycle::writer_unconfirmed) { + guard.quarantine(); + } + result.with_context(|| format!("failed to init replica {} for {}", replica_id, database_identity)) } } @@ -813,36 +847,31 @@ impl ModuleLauncher { /// If the `db` is not initialized yet (i.e. its program hash is `None`), /// return an error. /// -/// Otherwise, if `db.program_hash` matches the given `program_hash`, do -/// nothing and return an empty `UpdateDatabaseResult`. -/// -/// Otherwise, invoke `module.update_database` and return the result. +/// Admission, including unchanged programs and idempotent deployment retries, +/// is serialized inside the migration transaction. async fn update_module( db: &RelationalDB, module: &ModuleHost, program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { let addr = db.database_identity(); match stored_program_hash(db)? { None => Err(anyhow!("database `{addr}` not yet initialized")), Some(stored) => { - let res = if stored == program.hash { - info!("database `{}` up to date with program `{}`", addr, program.hash); - UpdateDatabaseResult::NoUpdateNeeded - } else { - info!("updating `{}` from {} to {}", addr, stored, program.hash); - module.update_database(program, old_module_info, policy).await? - }; - - Ok(res) + info!("publishing `{}` from {} to {}", addr, stored, program.hash); + module + .update_database_with_deployment(program, old_module_info, policy, deployment) + .await } } } /// Encapsulates a database, associated module, and auxiliary state. struct Host { + registration: Registration, /// The [`ModuleHost`], providing the callable reducer API. /// /// Modules may be updated via [`Host::update_module`]. @@ -873,230 +902,239 @@ struct Host { impl Host { /// Attempt to instantiate a [`Host`] from persistent storage. /// - /// Note that this does **not** run module initialization routines, but may - /// create on-disk artifacts if the host / database did not exist. + /// This executes the stored module and initializes a new database when + /// necessary. It may create on-disk artifacts. Retained cleanup uses the + /// separate metadata-only open path instead. #[tracing::instrument(level = "debug", skip_all)] - async fn try_init(host_controller: &HostController, database: Database, replica_id: u64) -> anyhow::Result { + async fn try_init( + host_controller: &HostController, + database: Database, + replica_id: u64, + registration: Registration, + ) -> anyhow::Result { let HostController { data_dir, default_config: config, program_storage, energy_monitor, runtimes, - persistence, - page_pool, bsatn_rlb_pool, .. } = host_controller; let replica_dir = data_dir.replica(replica_id); let (tx_metrics_queue, tx_metrics_recorder_task) = spawn_tx_metrics_recorder(); - let (db, connected_clients) = match config.storage { - db::Storage::Memory => RelationalDB::open( - database.database_identity, - database.owner_identity, - EmptyHistory::new(), - None, - Some(tx_metrics_queue), - page_pool.clone(), - )?, - db::Storage::Disk => { - // Replay from the local state. - let history = relational_db::local_history(&replica_dir).await?; - let persistence = persistence.persistence(&database, replica_id).await?; - // Loading a database from persistent storage involves heavy - // blocking I/O. `asyncify` to avoid blocking the async worker. - let (db, clients) = asyncify({ - let database_identity = database.database_identity; - let owner_identity = database.owner_identity; - let page_pool = page_pool.clone(); - move || { - RelationalDB::open( - database_identity, - owner_identity, - history, - Some(persistence), - Some(tx_metrics_queue), - page_pool, - ) - } - }) - .await - // Make sure we log the source chain of the error - // as a single line, with the help of `anyhow`. - .map_err(anyhow::Error::from) - .inspect_err(|e| { - tracing::error!( - database = %database.database_identity, - replica = replica_id, - "Failed to open database: {e:#}" + let metrics_cleanup = scopeguard::guard(tx_metrics_recorder_task.clone(), |task| task.abort()); + let (db, connected_clients, joined) = + lifecycle::open_database(host_controller, &database, replica_id, Some(tx_metrics_queue)).await?; + let initialized = std::panic::AssertUnwindSafe(async { + let (mut program, program_needs_init, initial_deployment) = match db.program()? { + // Launch module with program from existing database. + Some(program) => { + info!( + "loaded program {} from the database host-type={}", + program.hash, + HostType::from(program.kind) ); - })?; - - (db, clients) - } - }; - let (mut program, program_needs_init) = match db.program()? { - // Launch module with program from existing database. - Some(program) => { - info!( - "loaded program {} from the database host-type={}", - program.hash, - HostType::from(program.kind) - ); - (program, false) - } - // Database is empty, load program from external storage and run - // initialization. - None => { - info!( - "loading program {} from external storage host-type={}", - database.initial_program, database.host_type - ); - let program_bytes = load_program(program_storage, database.initial_program).await?; - let program = Program { - hash: database.initial_program, - bytes: program_bytes, - kind: database.host_type.into(), - }; - (program, true) - } - }; - - let relational_db = Arc::new(db); - let (program, launched) = match HostType::from(program.kind) { - HostType::Js => { - ModuleLauncher { - database, - replica_id, - program, - on_panic: host_controller.unregister_fn(replica_id), - relational_db, - energy_monitor: energy_monitor.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), + (program, false, None) } - .launch_module() - .await? - } - HostType::Wasm => { - // Prior to https://github.com/clockworklabs/SpacetimeDB/pull/4549 - // the host type in `st_module` was always set to wasm. - // We now correctly use the host type from the database, but the - // module may in fact be a JS module. - // So if launching it as a wasm module fails, try JS instead. - // If this succeeds, the module is definitely a JS module, so - // attempt to repair `st_module` in this case. - // - // TODO: This code should eventually be removed once all - // databases have been repaired. - let launch_wasm_result = ModuleLauncher { - database: database.clone(), - replica_id, - program: program.clone(), - on_panic: host_controller.unregister_fn(replica_id), - relational_db: relational_db.clone(), - energy_monitor: energy_monitor.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.clone().module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), + // Database is empty, load program from external storage and run + // initialization. + None => { + info!( + "loading program {} from external storage host-type={}", + database.initial_program, database.host_type + ); + let initial_deployment = program_storage.initial_deployment(&database).await?; + let program_bytes = load_program(program_storage, database.initial_program).await?; + let program = Program { + hash: database.initial_program, + bytes: program_bytes, + kind: database.host_type.into(), + }; + (program, true, initial_deployment) } - .launch_module() - .await; - match launch_wasm_result { - Ok(program_and_module_host) => program_and_module_host, - Err(e) => { - warn!("failed to launch wasm module, trying js: {e:#}"); - - program.kind = ModuleKind::JS; - let res = ModuleLauncher { - database, - replica_id, - program: program.clone(), - on_panic: host_controller.unregister_fn(replica_id), - relational_db: relational_db.clone(), - energy_monitor: energy_monitor.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), - } - .launch_module() - .await; + }; - if res.is_ok() { - let _ = relational_db - .with_auto_commit(Workload::Internal, |tx| relational_db.update_program(tx, program)); + let relational_db = db.clone(); + let (program, launched) = match HostType::from(program.kind) { + HostType::Js => { + ModuleLauncher { + database, + replica_id, + program, + on_panic: host_controller.unregister_fn(registration.clone()), + relational_db, + energy_monitor: energy_monitor.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await? + } + HostType::Wasm => { + // Prior to https://github.com/clockworklabs/SpacetimeDB/pull/4549 + // the host type in `st_module` was always set to wasm. + // We now correctly use the host type from the database, but the + // module may in fact be a JS module. + // Retry JS only for an existing stored module whose database + // declaration explicitly identifies it as JS. A new publication + // or declared Wasm module must preserve its Wasm validation error. + // If the legacy retry succeeds, repair `st_module`. + // + // TODO: This code should eventually be removed once all + // databases have been repaired. + let launch_wasm_result = ModuleLauncher { + database: database.clone(), + replica_id, + program: program.clone(), + on_panic: host_controller.unregister_fn(registration.clone()), + relational_db: relational_db.clone(), + energy_monitor: energy_monitor.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.clone().module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await; + match launch_wasm_result { + Ok(program_and_module_host) => program_and_module_host, + Err(e) => { + if program_needs_init || database.host_type != HostType::Js { + return Err(e); + } + warn!("failed to launch wasm module, trying js: {e:#}"); + + program.kind = ModuleKind::JS; + let res = ModuleLauncher { + database, + replica_id, + program: program.clone(), + on_panic: host_controller.unregister_fn(registration.clone()), + relational_db: relational_db.clone(), + energy_monitor: energy_monitor.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await; + + if res.is_ok() { + let _ = relational_db.with_auto_commit(Workload::Internal, |tx| { + relational_db.update_program(tx, program) + }); + } + + res.map_err(|js_error| { + e.context(format!("legacy JS host-type repair also failed: {js_error:#}")) + })? } - - res? } } + }; + + if program_needs_init { + let call_result = launched + .module_host + .init_database_with_deployment(program, initial_deployment) + .await?; + if let Some(call_result) = call_result { + Result::from(call_result)?; + } + } else { + drop(program) } - }; - if program_needs_init { - let call_result = launched.module_host.init_database(program).await?; - if let Some(call_result) = call_result { - Result::from(call_result)?; + let LaunchedModule { + replica_ctx, + module_host, + scheduler, + scheduler_starter, + } = launched; + + // Disconnect dangling clients. + // No need to clear view tables here since we do it in `clear_all_clients`. + for (identity, connection_id) in connected_clients { + module_host + .call_identity_disconnected(identity, connection_id) + .await + .with_context(|| { + format!( + "Error calling disconnect for {} {} on {}", + identity, connection_id, replica_ctx.database_identity + ) + })?; } - } else { - drop(program) - } + // We should have no clients left, but we do this just in case. + // This should only matter if we crashed with something in st_client_credentials, + // then restarted with an older version of the code that doesn't use st_client_credentials. + // That case would cause some permanently dangling st_client_credentials. + // Since we have no clients on startup, this should be safe to do regardless. + module_host.clear_all_clients().await?; + + // Scheduled work can run before this Host is installed in its cell. + // Its callback must already identify this executable; the close + // owner will wait for our pinned guard before taking the Host. + registration.activate(); + scheduler_starter.start(&module_host)?; + #[cfg(test)] + if lifecycle::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + .lock() + .remove(&replica_ctx.database_identity) + { + host_controller.unregister_fn(registration.clone())(); + } + let disk_metrics_recorder_task = tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle(); + let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone()); - let LaunchedModule { - replica_ctx, - module_host, - scheduler, - scheduler_starter, - } = launched; - - // Disconnect dangling clients. - // No need to clear view tables here since we do it in `clear_all_clients`. - for (identity, connection_id) in connected_clients { - module_host - .call_identity_disconnected(identity, connection_id) - .await - .with_context(|| { - format!( - "Error calling disconnect for {} {} on {}", - identity, connection_id, replica_ctx.database_identity - ) - })?; - } - // We should have no clients left, but we do this just in case. - // This should only matter if we crashed with something in st_client_credentials, - // then restarted with an older version of the code that doesn't use st_client_credentials. - // That case would cause some permanently dangling st_client_credentials. - // Since we have no clients on startup, this should be safe to do regardless. - module_host.clear_all_clients().await?; - - scheduler_starter.start(&module_host)?; - let disk_metrics_recorder_task = tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle(); - let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone()); - - let module = watch::Sender::new(module_host); - - Ok(Host { - module, - replica_ctx, - scheduler, - disk_metrics_recorder_task, - tx_metrics_recorder_task, - view_cleanup_task, + let module = watch::Sender::new(module_host); + + Ok(Host { + registration, + module, + replica_ctx, + scheduler, + disk_metrics_recorder_task, + tx_metrics_recorder_task, + view_cleanup_task, + }) }) + .catch_unwind() + .await; + match initialized { + Ok(Ok(host)) => { + let _ = scopeguard::ScopeGuard::into_inner(metrics_cleanup); + Ok(host) + } + Ok(Err(error)) => { + if let Some(joined) = joined { + joined.join().await?; + } + drop(db); + Err(error) + } + Err(panic) => { + if let Some(joined) = joined { + joined.join().await?; + } + drop(db); + std::panic::resume_unwind(panic) + } + } } /// Construct an in-memory instance of `database` running `program`. @@ -1162,9 +1200,12 @@ impl Host { runtimes: Arc, program: Program, policy: MigrationPolicy, + deployment: Option, energy_monitor: Arc, on_panic: impl Fn() + Send + Sync + 'static, + registration: Registration, core: AllocatedJobCore, + database_committed: &mut bool, ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); @@ -1183,43 +1224,72 @@ impl Host { // Get the old module info to diff against when building a migration plan. let old_module_info = self.module.borrow().info.clone(); - let update_result = - update_module(replica_ctx.relational_db(), &module, program, old_module_info, policy).await?; + let update_result = update_module( + replica_ctx.relational_db(), + &module, + program, + old_module_info, + policy, + deployment, + ) + .await?; + + *database_committed = matches!( + update_result, + UpdateDatabaseResult::UpdatePerformed { .. } + | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } + ); + + #[cfg(test)] + if *database_committed + && FAIL_NEXT_DEPLOYMENT_ACTIVATION + .lock() + .remove(&replica_ctx.database_identity) + { + bail!("injected scheduler activation failure after deployment commit"); + } // Only replace the module + scheduler if the update succeeded. // Otherwise, we want the database to continue running with the old state. match update_result { UpdateDatabaseResult::NoUpdateNeeded | UpdateDatabaseResult::UpdatePerformed { .. } => { - self.scheduler = scheduler; + registration.activate(); scheduler_starter.start(&module)?; + self.scheduler = scheduler; + self.registration = registration.clone(); let old_module = self.module.send_replace(module); old_module.exit().await; } // In this case, we need to disconnect all clients connected to the old module UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } => { + self.registration = registration; + self.registration.activate(); // Replace the module first, so that new clients get the new module. let old_watcher = std::mem::replace(&mut self.module, watch::Sender::new(module.clone())); + let old_module = old_watcher.borrow().clone(); // Disconnect all clients connected to the old module. - let connected_clients = replica_ctx.relational_db().connected_clients()?; - for (identity, connection_id) in connected_clients { - let client_actor_id = ClientActorId { - identity, - connection_id, - name: ClientName(0), - }; - //NOTE: This will call disconnect reducer of the new module, not the old one. - //It makes sense, as relationaldb is already updated to the new module. - module.disconnect_client(client_actor_id).await; + let activation = async { + let connected_clients = replica_ctx.relational_db().connected_clients()?; + for (identity, connection_id) in connected_clients { + let client_actor_id = ClientActorId { + identity, + connection_id, + name: ClientName(0), + }; + // Disconnect uses the newly committed module. + module.disconnect_client(client_actor_id).await; + } + scheduler_starter.start(&module)?; + Ok::<_, anyhow::Error>(()) } - - self.scheduler = scheduler; - scheduler_starter.start(&module)?; + .await; // exit the old module, drop the `old_watcher` afterwards, // which will signal websocket clients that the module is gone. - let old_module = old_watcher.borrow().clone(); old_module.exit().await; + activation?; + self.scheduler = scheduler; } _ => {} } diff --git a/crates/core/src/host/host_controller/deployment_tests.rs b/crates/core/src/host/host_controller/deployment_tests.rs new file mode 100644 index 00000000000..348d90a695d --- /dev/null +++ b/crates/core/src/host/host_controller/deployment_tests.rs @@ -0,0 +1,213 @@ +use super::*; +use crate::db::deployment::{current_deployment, install_publication_fence}; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::empty_module; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ST_DEPLOYMENT_OPERATION_ID, ST_PUBLISH_FENCE_ID}; +use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; +use spacetimedb_lib::{hash_bytes, Uuid}; +use spacetimedb_paths::FromPathUnchecked; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +struct InitialStorage { + program: Program, + request: Option, + initial_lookups: AtomicUsize, +} + +#[async_trait] +impl ExternalStorage for InitialStorage { + async fn lookup(&self, hash: Hash) -> anyhow::Result>> { + Ok((self.program.hash == hash).then(|| self.program.bytes.clone())) + } + async fn initial_deployment(&self, _: &Database) -> anyhow::Result> { + self.initial_lookups.fetch_add(1, Ordering::SeqCst); + Ok(self.request.clone()) + } +} + +fn request(program: &Program, sequence: u64, initial: bool) -> DeploymentCommit { + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + DeploymentCommit { + operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(sequence)), + publication_epoch: sequence, + publisher: Identity::ONE, + expected_revision: None, + prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: if initial { + ModuleComponent::SystemEmpty(1) + } else { + ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: program.hash, + }) + }, + container: Some(ContainerSpec { + image_manifest: OciDigest::sha256([4; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 1000, + }), + }), + } +} + +fn fixture(id: u64, bootstrap: bool) -> (tempfile::TempDir, HostController, Database, Arc) { + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = empty_module::program(1).unwrap(); + let storage = Arc::new(InitialStorage { + request: bootstrap.then(|| request(&program, 1, true)), + program: program.clone(), + initial_lookups: AtomicUsize::new(0), + }); + let controller = HostController::new( + data_dir.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + storage.clone(), + Arc::new(NullEnergyMonitor), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + }; + (directory, controller, database, storage) +} + +#[tokio::test(flavor = "multi_thread")] +async fn bootstrap_commits_fence_program_and_deployment_and_reopens_from_disk() { + let (_directory, controller, database, storage) = fixture(0xdd01, true); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let expected = storage.request.as_ref().unwrap().deployment.revision().unwrap(); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, expected); + assert_eq!(tx.table_row_count(ST_PUBLISH_FENCE_ID), Some(1)); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + assert_eq!( + module.relational_db().program().unwrap().unwrap().hash, + database.initial_program + ); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!( + storage.initial_lookups.load(Ordering::SeqCst), + 1, + "replay must not re-run bootstrap intent" + ); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, expected); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn activation_failure_after_commit_closes_old_host_and_recovers_committed_program() { + let (_directory, controller, database, _storage) = fixture(0xdd02, false); + let old_module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let watcher = controller.watch_module_host(database.id).await.unwrap(); + let mut bytes = empty_module::VERSION_1_BYTES.to_vec(); + bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let newer = Program::from_bytes(ModuleKind::WASM, bytes); + let publication = request(&newer, 1, false); + old_module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, publication.publication_epoch, publication.operation_id) + }) + .unwrap(); + FAIL_NEXT_DEPLOYMENT_ACTIVATION + .lock() + .insert(database.database_identity); + let error = controller + .update_module_host_with_deployment( + database.clone(), + HostType::Wasm, + database.id, + newer.bytes.clone(), + MigrationPolicy::Compatible, + Some(publication.clone()), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("injected scheduler activation failure")); + assert!( + controller.get_module_host(database.id).await.is_err(), + "old executable must not remain available" + ); + assert!(watcher.has_changed().is_err(), "old client watcher must close"); + assert_eq!(old_module.relational_db().program().unwrap().unwrap().hash, newer.hash); + old_module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!( + current_deployment(tx).unwrap().unwrap().0, + publication.deployment.revision().unwrap() + ); + }); + drop(watcher); + drop(old_module); + let recovered = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(recovered.info.module_hash, newer.hash); + recovered.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!( + current_deployment(tx).unwrap().unwrap().0, + publication.deployment.revision().unwrap() + ); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + drop(recovered); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} diff --git a/crates/core/src/host/host_controller/execution_deadline_tests.rs b/crates/core/src/host/host_controller/execution_deadline_tests.rs new file mode 100644 index 00000000000..edc7dfa9ac2 --- /dev/null +++ b/crates/core/src/host/host_controller/execution_deadline_tests.rs @@ -0,0 +1,411 @@ +//! Actual V8 module hosts, transactions and updates. No network or external service. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::FunctionArgs; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_lib::db::raw_def::{v10::RawModuleDefV10Builder, v9::Lifecycle}; +use spacetimedb_paths::FromPathUnchecked; +use spacetimedb_sats::{AlgebraicType, ProductType}; + +fn config() -> HostRuntimeConfig { + HostRuntimeConfig { + v8: V8Config { + execution_timeout: Duration::from_millis(200), + ..V8Config::default() + }, + ..HostRuntimeConfig::default() + } +} + +fn program(loop_in_first_init: bool) -> Program { + program_with_startup_cutoff(loop_in_first_init, u64::MAX) +} + +fn program_with_startup_cutoff(loop_in_first_init: bool, cutoff_millis: u64) -> Program { + let mut schema = RawModuleDefV10Builder::new(); + schema + .build_table_with_new_type("rows", [("value", AlgebraicType::U64)], true) + .finish(); + schema.add_lifecycle_reducer(Lifecycle::Init, "init", ProductType::unit()); + schema.add_reducer("loop", ProductType::unit()); + schema.add_reducer("good", ProductType::unit()); + schema.add_procedure("task", ProductType::unit(), AlgebraicType::U64); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{ register_hooks, table_id_from_name, datastore_insert_bsatn }} from "spacetime:sys@1.0"; + import {{ register_hooks as register_procedure_hooks }} from "spacetime:sys@1.2"; + if (Date.now() > {cutoff_millis}) {{ for (;;) {{}} }} + let initCalls = 0; + let nextValue = 0; + register_hooks({{ + __describe_module__: function() {{ return new Uint8Array({schema:?}); }}, + __call_reducer__: function(id) {{ + const row = new Uint8Array(8); + row[0] = ++nextValue; + datastore_insert_bsatn(table_id_from_name("rows"), row); + if (id === 1 || (id === 0 && ++initCalls === 1 && {loop_in_first_init})) {{ for (;;) {{}} }} + return {{ tag: "ok" }}; + }}, + }}); + let retained; + register_procedure_hooks({{ __call_procedure__: function() {{ + if (Date.now() > {cutoff_millis}) {{ retained = new Array(4 * 1024 * 1024).fill(1); }} + return new Uint8Array(8); + }} }}); + "# + ) + .into_bytes(), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_procedure_startup_failure_keeps_main_module_registered() { + use std::time::{SystemTime, UNIX_EPOCH}; + let cutoff = SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + Duration::from_secs(3); + let program = program_with_startup_cutoff(false, cutoff.as_millis() as u64); + let (_directory, controller, database) = controller_fixture( + 0xed05, + &program, + HostRuntimeConfig { + v8: V8Config { + // One slot makes a leaked admission permit observable on retry. + procedure_instance_pool_size: std::num::NonZeroUsize::new(1).unwrap(), + ..config().v8 + }, + ..config() + }, + ); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert!(call(&module, "good").await.is_ok()); + let until_cutoff = cutoff.saturating_sub(SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + tokio::time::sleep(until_cutoff + Duration::from_millis(50)).await; + // The module's main isolate already ran startup. Only newly created + // procedure isolates encounter the now-hostile startup branch. + for _ in 0..2 { + let result = timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary), + ) + .await + .unwrap(); + let error = result.result.unwrap_err(); + assert!(error.to_string().contains("wall-clock limit"), "{error}"); + let registered = controller.get_module_host(database.id).await.unwrap(); + assert!(call(®istered, "good").await.is_ok()); + } + assert_eq!(row_count(&module), Some(4)); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +fn database(id: u64, program: &Program) -> Database { + Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Js, + initial_program: program.hash, + } +} + +fn controller_fixture( + id: u64, + program: &Program, + config: HostRuntimeConfig, +) -> (tempfile::TempDir, HostController, Database) { + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let initial = program.clone(); + let storage = move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }; + let controller = HostController::new( + data_dir.clone(), + db::Config { + storage: db::Storage::Memory, + page_pool_max_size: None, + }, + config, + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + (directory, controller, database(id, program)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_failed_procedure_recreation_keeps_main_module_registered() { + use std::time::{SystemTime, UNIX_EPOCH}; + let cutoff = SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + Duration::from_secs(3); + let program = program_with_startup_cutoff(false, cutoff.as_millis() as u64); + let (_directory, controller, database) = controller_fixture( + 0xed06, + &program, + HostRuntimeConfig { + v8: V8Config { + procedure_instance_pool_size: std::num::NonZeroUsize::new(1).unwrap(), + heap_policy: crate::config::V8HeapPolicyConfig { + heap_limit_bytes: 64 * 1024 * 1024, + heap_check_request_interval: Some(1), + heap_gc_trigger_fraction: 0.2, + heap_retire_fraction: 0.2, + ..Default::default() + }, + ..config().v8 + }, + ..config() + }, + ); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + // Populate the one-slot procedure pool before the startup branch changes. + assert!(module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + .result + .is_ok()); + let remaining = cutoff.saturating_sub(SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + tokio::time::sleep(remaining + Duration::from_millis(50)).await; + // This invocation runs in the existing isolate and leaves32MiB alive. The + // real post-call heap policy retires it and tries a new, now-looping startup. + assert!(module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + .result + .is_ok()); + let result = timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary), + ) + .await + .unwrap(); + let error = result.result.unwrap_err(); + assert!( + error.to_string().contains("procedure isolate startup failed"), + "{error}" + ); + assert!(error.to_string().contains("wall-clock limit"), "{error}"); + let registered = controller.get_module_host(database.id).await.unwrap(); + assert!(call(®istered, "good").await.is_ok()); + // A subsequent checkout also completes, proving the dead instance's slot + // was released instead of leaked after its replacement failed. + assert!(timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary,) + ) + .await + .unwrap() + .result + .is_err()); + assert!(call(®istered, "good").await.is_ok()); + drop(registered); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +async fn launch(id: u64, program: Program) -> anyhow::Result<(Program, LaunchedModule)> { + timeout( + Duration::from_secs(5), + Host::try_init_in_memory_to_check( + &HostRuntimes::new(None, config()), + PagePool::new(None), + database(id, &program), + program, + AllocatedJobCore::default(), + BsatnRowListBuilderPool::new(), + ), + ) + .await? +} + +fn row_count(module: &ModuleHost) -> Option { + module.relational_db().with_read_only(Workload::Internal, |tx| { + tx.table_id_from_name("rows") + .unwrap() + .and_then(|table| tx.table_row_count(table)) + }) +} + +async fn call(module: &ModuleHost, name: &str) -> ReducerCallResult { + timeout( + Duration::from_secs(5), + module.call_reducer(Identity::ONE, None, None, None, None, name, FunctionArgs::Nullary), + ) + .await + .unwrap() + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_rolls_back_init_and_reducer_and_preserves_isolate() { + let (program, launched) = launch(0xed01, program(true)).await.unwrap(); + let module = launched.module_host; + let result = module.init_database(program.clone()).await.unwrap().unwrap(); + assert!(result.is_err(), "infinite init must fail"); + assert!(module.relational_db().program().unwrap().is_none()); + assert_eq!( + row_count(&module), + None, + "failed init must roll back the schema and inserted row" + ); + + assert!(module.init_database(program).await.unwrap().unwrap().is_ok()); + assert_eq!(row_count(&module), Some(1)); + let failed = call(&module, "loop").await; + assert!(failed.is_err()); + assert_eq!( + row_count(&module), + Some(1), + "timed-out reducer must roll back its write" + ); + + for _ in 0..64 { + assert!(call(&module, "good").await.is_ok()); + } + // Cross the original timer boundary before reusing the same isolate again. + tokio::time::sleep(Duration::from_millis(250)).await; + assert!(call(&module, "good").await.is_ok()); + assert_eq!(row_count(&module), Some(66)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_bounds_startup_description_and_failed_update() { + let valid = program(false); + let looping_startup = Program::from_bytes(ModuleKind::JS, b"for (;;) {}".to_vec()); + let looping_description = Program::from_bytes( + ModuleKind::JS, + br#"import {register_hooks} from "spacetime:sys@1.0"; + register_hooks({__describe_module__: function() {for (;;) {}}, __call_reducer__: function() {}});"# + .to_vec(), + ); + for (id, bad) in [(0xed02, &looping_startup), (0xed03, &looping_description)] { + let error = match launch(id, bad.clone()).await { + Ok(_) => panic!("infinite JavaScript unexpectedly launched"), + Err(error) => error, + }; + assert!(format!("{error:#}").contains("wall-clock limit"), "{error:#}"); + } + + let (_directory, controller, database) = controller_fixture(0xed04, &valid, config()); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(row_count(&module), Some(1)); + for bad in [looping_startup, looping_description] { + assert!(timeout( + Duration::from_secs(5), + controller.update_module_host( + database.clone(), + HostType::Js, + database.id, + bad.bytes, + MigrationPolicy::Compatible, + ) + ) + .await + .unwrap() + .is_err()); + assert_eq!(module.relational_db().program().unwrap().unwrap().hash, valid.hash); + assert!(call(&module, "good").await.is_ok()); + } + let mut newer = valid.bytes.to_vec(); + newer.extend_from_slice(b"\n// Valid replacement after two deadline failures.\n"); + let newer = Program::from_bytes(ModuleKind::JS, newer); + controller + .update_module_host( + database.clone(), + HostType::Js, + database.id, + newer.bytes.clone(), + MigrationPolicy::Compatible, + ) + .await + .unwrap(); + let replacement = controller.get_module_host(database.id).await.unwrap(); + assert_eq!(replacement.relational_db().program().unwrap().unwrap().hash, newer.hash); + assert!(call(&replacement, "good").await.is_ok()); + drop(replacement); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn javascript_logging_handles_direct_startup_and_preserves_wrapped_call_locations() { + use futures::TryStreamExt as _; + + let mut schema = RawModuleDefV10Builder::new(); + schema.add_reducer("log", ProductType::unit()); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + // Direct top-level logging has exactly one JS frame. It previously passed + // index1 to V8's unchecked GetFrame and could crash the entire host process. + let source = format!( + r#"import {{ register_hooks, console_log }} from "spacetime:sys@1.0"; +console_log(2, "direct-startup"); +function wrapped(message) {{ console_log(2, message); }} +wrapped("wrapped-startup"); +register_hooks({{ + __describe_module__: () => new Uint8Array({schema:?}), + __call_reducer__: () => {{ + console_log(2, "direct-reducer"); + wrapped("wrapped-reducer"); + return {{ tag: "ok" }}; + }}, +}}); +"# + ); + let program = Program::from_bytes(ModuleKind::JS, source.into_bytes()); + let (_directory, controller, database) = controller_fixture(0xed08, &program, config()); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert!(call(&module, "log").await.is_ok()); + let chunks: Vec<_> = module + .database_logger() + .tail(None, false) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let bytes = chunks.into_iter().flatten().collect::>(); + let records = String::from_utf8(bytes) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + for (message, line) in [ + ("direct-startup", 2), + ("wrapped-startup", 4), + ("direct-reducer", 8), + ("wrapped-reducer", 9), + ] { + let record = records.iter().find(|record| record["message"] == message).unwrap(); + assert_eq!(record["line_number"], line, "wrong call location for {message}"); + assert!(record["filename"].as_str().is_some_and(|filename| !filename.is_empty())); + } +} diff --git a/crates/core/src/host/host_controller/lifecycle.rs b/crates/core/src/host/host_controller/lifecycle.rs new file mode 100644 index 00000000000..3c1237035b4 --- /dev/null +++ b/crates/core/src/host/host_controller/lifecycle.rs @@ -0,0 +1,145 @@ +//! Shared physical writer completion for ordinary module initialization and +//! shutdown. Provider snapshot and archival services keep their own ownership. + +use super::*; +use crate::db::persistence::Persistence; +use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; +use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; +use std::panic::{resume_unwind, AssertUnwindSafe}; +use std::sync::OnceLock; + +#[derive(Debug, thiserror::Error)] +pub(super) enum CloseFailure { + #[error("module exit panicked; the storage writer was positively closed")] + ModuleExit, + #[error("storage writer close panicked; the replica remains quarantined")] + WriterUnconfirmed, +} + +#[cfg(test)] +pub(super) static FAIL_NEXT_MODULE_EXIT: parking_lot::Mutex> = + parking_lot::Mutex::new(std::collections::BTreeSet::new()); + +#[cfg(test)] +pub(super) static PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START: parking_lot::Mutex> = + parking_lot::Mutex::new(std::collections::BTreeSet::new()); + +pub(super) fn writer_unconfirmed(error: &anyhow::Error) -> bool { + matches!( + error.downcast_ref::(), + Some(CloseFailure::WriterUnconfirmed) + ) +} + +/// Some providers hand the first close caller the writer's JoinHandle and let +/// later calls finish immediately. Share that *first* physical close instead. +/// In particular, partial RelationalDB replay may start close from Drop before +/// our owning task sees its error. That task must join the same close future. +pub(super) struct JoinedDurability { + inner: Arc>, + close: OnceLock>>>, +} + +impl JoinedDurability { + pub fn wrap(persistence: &mut Persistence) -> Arc { + let joined = Arc::new(Self { + inner: persistence.durability.clone(), + close: OnceLock::new(), + }); + persistence.durability = joined.clone(); + joined + } + + pub async fn join(&self) -> Result, CloseFailure> { + AssertUnwindSafe(async { self.close().await }) + .catch_unwind() + .await + .map_err(|_| CloseFailure::WriterUnconfirmed) + } +} + +impl Durability for JoinedDurability { + type TxData = Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.inner.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.inner.durable_tx_offset() + } + fn close(&self) -> Close { + self.close.get_or_init(|| self.inner.close().shared()).clone().boxed() + } +} + +/// A failed replay joins the same physical writer close before another normal +/// initialization can acquire the replica's canonical registry cell. +pub(super) async fn open_database( + controller: &HostController, + database: &Database, + replica_id: u64, + tx_metrics_queue: Option, +) -> anyhow::Result<( + Arc, + relational_db::ConnectedClients, + Option>, +)> { + if matches!(controller.default_config.storage, db::Storage::Memory) { + let (db, clients) = RelationalDB::open( + database.database_identity, + database.owner_identity, + EmptyHistory::new(), + None, + tx_metrics_queue, + controller.page_pool.clone(), + )?; + return Ok((Arc::new(db), clients, None)); + } + + let replica_dir = controller.data_dir.replica(replica_id); + let history = relational_db::local_history(&replica_dir).await?; + let mut persistence = controller.persistence.persistence(database, replica_id).await?; + let joined = JoinedDurability::wrap(&mut persistence); + let identity = database.database_identity; + let owner = database.owner_identity; + let page_pool = controller.page_pool.clone(); + let opened = AssertUnwindSafe(asyncify(move || { + RelationalDB::open(identity, owner, history, Some(persistence), tx_metrics_queue, page_pool) + })) + .catch_unwind() + .await; + let (db, clients) = match opened { + Ok(Ok(opened)) => opened, + Ok(Err(error)) => { + joined.join().await?; + return Err(error.into()); + } + Err(panic) => { + joined.join().await?; + resume_unwind(panic); + } + }; + let db = Arc::new(db); + Ok((db, clients, Some(joined))) +} + +pub(super) async fn close_host(host: Host) -> Result<(), CloseFailure> { + let module = host.module.borrow().clone(); + let info = module.info(); + let identity = info.database_identity; + let table_names = info.module_def.tables().map(|table| table.name.deref()); + defer!(remove_database_gauges(&identity, table_names)); + let exited = AssertUnwindSafe(async { + module.exit().await; + #[cfg(test)] + if FAIL_NEXT_MODULE_EXIT.lock().remove(&identity) { + panic!("injected module exit failure"); + } + }) + .catch_unwind() + .await; + let writer = AssertUnwindSafe(module.relational_db().shutdown()).catch_unwind().await; + drop(host); + writer.map_err(|_| CloseFailure::WriterUnconfirmed)?; + exited.map_err(|_| CloseFailure::ModuleExit) +} diff --git a/crates/core/src/host/host_controller/lifecycle_tests.rs b/crates/core/src/host/host_controller/lifecycle_tests.rs new file mode 100644 index 00000000000..6cacab25172 --- /dev/null +++ b/crates/core/src/host/host_controller/lifecycle_tests.rs @@ -0,0 +1,397 @@ +//! Real local durability writers and module hosts. No network or external +//! configuration. The probe counts writers until their first close completes. +use super::*; +use crate::db::persistence::{LocalPersistenceProvider, Persistence}; +use crate::host::empty_module; +use futures::FutureExt; +use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; +use spacetimedb_paths::FromPathUnchecked; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use tokio::sync::Semaphore; + +struct Probe { + opened: AtomicUsize, + active: AtomicUsize, + maximum: AtomicUsize, + block_close: AtomicBool, + panic_close: AtomicBool, + close_started: Semaphore, + release_close: Semaphore, +} + +impl Default for Probe { + fn default() -> Self { + Self { + opened: AtomicUsize::new(0), + active: AtomicUsize::new(0), + maximum: AtomicUsize::new(0), + block_close: AtomicBool::new(false), + panic_close: AtomicBool::new(false), + close_started: Semaphore::new(0), + release_close: Semaphore::new(0), + } + } +} + +struct ProbedProvider { + local: LocalPersistenceProvider, + probe: Arc, +} + +#[async_trait] +impl PersistenceProvider for ProbedProvider { + async fn persistence(&self, database: &Database, replica: u64) -> anyhow::Result { + let mut persistence = self.local.persistence(database, replica).await?; + let active = self.probe.active.fetch_add(1, Ordering::SeqCst) + 1; + self.probe.maximum.fetch_max(active, Ordering::SeqCst); + self.probe.opened.fetch_add(1, Ordering::SeqCst); + persistence.durability = Arc::new(ProbedWriter { + inner: persistence.durability, + probe: self.probe.clone(), + closing: AtomicBool::new(false), + }); + Ok(persistence) + } +} + +struct ProbedWriter { + inner: Arc>, + probe: Arc, + closing: AtomicBool, +} + +impl Durability for ProbedWriter { + type TxData = Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.inner.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.inner.durable_tx_offset() + } + fn close(&self) -> Close { + // Reproduce the actual first-caller-owns-join behavior deliberately. + // A second close must not let another database open before this one. + if self.closing.swap(true, Ordering::SeqCst) { + let offset = self.inner.durable_tx_offset().last_seen(); + return async move { offset }.boxed(); + } + let close = self.inner.close(); + let probe = self.probe.clone(); + async move { + probe.close_started.add_permits(1); + if probe.block_close.load(Ordering::SeqCst) { + probe.release_close.acquire().await.unwrap().forget(); + } + assert!( + !probe.panic_close.load(Ordering::SeqCst), + "injected writer close failure" + ); + let offset = close.await; + probe.active.fetch_sub(1, Ordering::SeqCst); + offset + } + .boxed() + } +} + +fn fixture( + id: u64, +) -> ( + tempfile::TempDir, + HostController, + Database, + Arc, + Arc, +) { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = empty_module::program(1).unwrap(); + let initial = program.clone(); + let lookups = Arc::new(AtomicUsize::new(0)); + let lookup_count = lookups.clone(); + let storage = move |hash| { + let initial = initial.clone(); + lookup_count.fetch_add(1, Ordering::SeqCst); + async move { Ok((hash == initial.hash).then_some(initial.bytes)) } + }; + let probe = Arc::new(Probe::default()); + let controller = HostController::new( + data.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(ProbedProvider { + local: LocalPersistenceProvider::new(data), + probe: probe.clone(), + }), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + }; + (directory, controller, database, probe, lookups) +} + +async fn closed_seed(controller: &HostController, database: &Database, probe: &Probe) { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + probe.close_started.acquire().await.unwrap().forget(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); +} + +async fn wait_for_close(probe: &Probe) { + timeout(Duration::from_secs(5), probe.close_started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_idle_module_and_queued_relookup_do_not_break_positive_close() { + let (_directory, controller, database, probe, _) = fixture(0xc002); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + probe.block_close.store(true, Ordering::SeqCst); + let exiting = { + let controller = controller.clone(); + tokio::spawn(async move { controller.exit_module_host(0xc002, Duration::from_secs(5)).await }) + }; + wait_for_close(&probe).await; + let lookup = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + }) + }; + assert!( + controller + .exit_module_host(database.id, Duration::from_millis(10)) + .await + .is_err(), + "a timeout is not positive closure" + ); + exiting.abort(); + assert!(exiting.await.unwrap_err().is_cancelled()); + assert_eq!(probe.opened.load(Ordering::SeqCst), 1); + assert_eq!(probe.active.load(Ordering::SeqCst), 1); + probe.block_close.store(false, Ordering::SeqCst); + probe.release_close.add_permits(1); + let current = timeout(Duration::from_secs(5), lookup).await.unwrap().unwrap().unwrap(); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + assert!(!Arc::ptr_eq(idle.relational_db(), current.relational_db())); + let seen_live = controller.get_module_host(database.id).await.unwrap(); + assert!(Arc::ptr_eq(seen_live.relational_db(), current.relational_db())); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); + drop((idle, current)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_failed_normal_initialization_joins_writer_before_retry() { + let (_directory, controller, database, probe, _) = fixture(0xc008); + closed_seed(&controller, &database, &probe).await; + let mut wrong_owner = database.clone(); + wrong_owner.owner_identity = Identity::from_u256(123456_u64.into()); + probe.block_close.store(true, Ordering::SeqCst); + let failed = { + let controller = controller.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(wrong_owner.clone(), wrong_owner.id) + .await + }) + }; + wait_for_close(&probe).await; + let next = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + }) + }; + assert!(!failed.is_finished()); + probe.release_close.add_permits(1); + assert!(failed.await.unwrap().is_err()); + let reopened = next.await.unwrap().unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + probe.block_close.store(false, Ordering::SeqCst); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + drop(reopened); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_module_exit_panic_reports_error_after_positive_writer_close_and_allows_reopen() { + let (_directory, controller, database, probe, _) = fixture(0xc00a); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + lifecycle::FAIL_NEXT_MODULE_EXIT + .lock() + .insert(database.database_identity); + let error = controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(error.to_string().contains("module exit panicked")); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + drop(idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_unconfirmed_writer_close_reports_error_and_quarantines_replica() { + let (_directory, controller, database, probe, _) = fixture(0xc00b); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + probe.panic_close.store(true, Ordering::SeqCst); + let error = controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(error.to_string().contains("writer close panicked")); + let error = timeout( + Duration::from_secs(1), + controller.get_or_launch_module_host(database.clone(), database.id), + ) + .await + .unwrap() + .unwrap_err(); + assert!(error.to_string().contains("unable to lock")); + assert_eq!(probe.opened.load(Ordering::SeqCst), 1); + assert!(controller + .exit_module_host(database.id, Duration::from_secs(1)) + .await + .unwrap_err() + .to_string() + .contains("quarantined")); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + drop(module); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_panic_callback_before_initial_host_install_still_owns_cleanup() { + let (_directory, controller, database, probe, _) = fixture(0xc00c); + lifecycle::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + .lock() + .insert(database.database_identity); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + // No explicit exit is requested until the injected scheduler interleaving + // has independently started the actual storage writer close. + wait_for_close(&probe).await; + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.get_module_host(database.id).await.is_err()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + drop(idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_stale_panic_callback_does_not_unregister_updated_or_reopened_host() { + let (_directory, controller, database, probe, _) = fixture(0xc006); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let old_callback = { + let guard = controller.acquire_read_lock(database.id).await.unwrap(); + controller.unregister_fn(guard.as_ref().unwrap().registration.clone()) + }; + let mut newer = empty_module::VERSION_1_BYTES.to_vec(); + newer.extend_from_slice(&[0, 3, 1, b'x', 1]); + let newer_hash = spacetimedb_lib::hash_bytes(&newer); + controller + .update_module_host( + database.clone(), + HostType::Wasm, + database.id, + newer.into(), + MigrationPolicy::Compatible, + ) + .await + .unwrap(); + old_callback(); + assert_eq!( + controller.get_module_host(database.id).await.unwrap().info.module_hash, + newer_hash + ); + assert_eq!(probe.active.load(Ordering::SeqCst), 1); + let current_callback = { + let guard = controller.acquire_read_lock(database.id).await.unwrap(); + controller.unregister_fn(guard.as_ref().unwrap().registration.clone()) + }; + current_callback(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert!(controller.get_module_host(database.id).await.is_err()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + current_callback(); + assert!(controller.get_module_host(database.id).await.is_ok()); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); +} diff --git a/crates/core/src/host/host_controller/registry.rs b/crates/core/src/host/host_controller/registry.rs new file mode 100644 index 00000000000..030a1d7c3fe --- /dev/null +++ b/crates/core/src/host/host_controller/registry.rs @@ -0,0 +1,333 @@ +//! Canonical host cells, pinned by queued and active operations. +//! +//! Lock order: the registry mutex is never held across an await. A cell guard +//! may briefly acquire the registry mutex to publish its resident-host state. +//! It releases the cell guard before releasing its operation pin. The last pin +//! removes an empty, non-closing entry without waiting for unrelated Arc owners. + +use super::{Host, HostCell}; +use parking_lot::Mutex; +use spacetimedb_data_structures::map::IntMap; +use std::ops::{Deref, DerefMut}; +use std::sync::{Arc, Weak}; +use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard}; + +pub(super) type Hosts = Arc>>; + +pub(super) struct Entry { + cell: HostCell, + pins: usize, + resident: bool, + registration: Option>, + closing: Option>, +} + +/// Completion means the storage writer has actually closed, not that the +/// caller's wait expired. Provider-owned snapshot/archival services are separate. +pub(super) struct Closing { + result: watch::Sender>>>, +} + +impl Closing { + pub async fn wait(&self) -> anyhow::Result<()> { + let mut receiver = self.result.subscribe(); + let result = receiver.wait_for(Option::is_some).await?.clone().unwrap(); + result.map_err(|message| anyhow::anyhow!(message.to_string())) + } +} + +/// A pin counts an operation, including time queued for the cell lock. Idle +/// ModuleHost references are not pins. Raw cells never authorize an operation. +pub(super) struct Pin { + hosts: Weak>>, + replica: u64, + cell: HostCell, +} + +impl Pin { + pub fn acquire(hosts: &Hosts, replica: u64) -> Self { + let mut entries = hosts.lock(); + let entry = entries.entry(replica).or_insert_with(|| Entry { + cell: HostCell::default(), + pins: 0, + resident: false, + registration: None, + closing: None, + }); + entry.pins += 1; + Self { + hosts: Arc::downgrade(hosts), + replica, + cell: entry.cell.clone(), + } + } + + fn closing(&self) -> Option> { + let hosts = self.hosts.upgrade()?; + let entries = hosts.lock(); + let entry = entries.get(&self.replica)?; + debug_assert!(Arc::ptr_eq(&entry.cell, &self.cell)); + entry.closing.clone() + } + + pub async fn read(hosts: &Hosts, replica: u64) -> anyhow::Result { + loop { + let pin = Self::acquire(hosts, replica); + let guard = pin.cell.clone().read_owned().await; + if let Some(closing) = pin.closing() { + drop(guard); + drop(pin); + closing.wait().await?; + continue; + } + return Ok(ReadGuard { + guard: Some(guard), + pin: Some(pin), + }); + } + } + + pub async fn write(hosts: &Hosts, replica: u64) -> anyhow::Result { + loop { + let pin = Self::acquire(hosts, replica); + let guard = pin.cell.clone().write_owned().await; + if let Some(closing) = pin.closing() { + drop(guard); + drop(pin); + closing.wait().await?; + continue; + } + return Ok(WriteGuard { + guard: Some(guard), + pin: Some(pin), + }); + } + } + + fn publish(&self, host: Option<&Host>) { + let Some(hosts) = self.hosts.upgrade() else { return }; + let mut entries = hosts.lock(); + let Some(entry) = entries.get_mut(&self.replica) else { + return; + }; + debug_assert!(Arc::ptr_eq(&entry.cell, &self.cell)); + entry.resident = host.is_some(); + entry.registration = host.map(|host| host.registration.token.clone()); + } + + fn registration(&self) -> Registration { + Registration { + hosts: self.hosts.clone(), + replica: self.replica, + cell: Arc::downgrade(&self.cell), + token: Arc::new(()), + } + } +} + +impl Drop for Pin { + fn drop(&mut self) { + let Some(hosts) = self.hosts.upgrade() else { return }; + let removed = { + let mut entries = hosts.lock(); + let Some(entry) = entries.get_mut(&self.replica) else { + return; + }; + assert!(Arc::ptr_eq(&entry.cell, &self.cell), "replaced a pinned host cell"); + entry.pins -= 1; + if entry.pins == 0 && !entry.resident && entry.closing.is_none() { + entries.remove(&self.replica) + } else { + None + } + }; + drop(removed); + } +} + +pub(super) struct ReadGuard { + guard: Option>>, + pin: Option, +} + +impl Deref for ReadGuard { + type Target = Option; + fn deref(&self) -> &Self::Target { + self.guard.as_ref().unwrap() + } +} + +impl Drop for ReadGuard { + fn drop(&mut self) { + drop(self.guard.take()); + drop(self.pin.take()); + } +} + +pub(super) struct WriteGuard { + guard: Option>>, + pin: Option, +} + +impl WriteGuard { + pub fn registration(&self) -> Registration { + self.pin.as_ref().unwrap().registration() + } + + pub fn install(&mut self, host: Host) { + **self = Some(host); + self.pin.as_ref().unwrap().publish(self.as_ref()); + } + + pub fn quarantine(&self) { + let pin = self.pin.as_ref().unwrap(); + let Some(hosts) = pin.hosts.upgrade() else { return }; + let mut entries = hosts.lock(); + let entry = entries.get_mut(&pin.replica).expect("pinned cell exists"); + if let Some(closing) = &entry.closing { + closing + .result + .send_replace(Some(Err("storage writer close is unconfirmed".into()))); + } + entry.closing = Some(Arc::new(Closing { + result: watch::Sender::new(Some(Err( + "storage writer close is unconfirmed; replica is quarantined".into() + ))), + })); + } +} + +impl Deref for WriteGuard { + type Target = Option; + fn deref(&self) -> &Self::Target { + self.guard.as_ref().unwrap() + } +} +impl DerefMut for WriteGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.guard.as_mut().unwrap() + } +} +impl Drop for WriteGuard { + fn drop(&mut self) { + self.pin.as_ref().unwrap().publish(self.as_ref()); + drop(self.guard.take()); + drop(self.pin.take()); + } +} + +/// The token changes with each installed executable, even when its cell and +/// database stay the same. A late panic from an old executable is not authority +/// to close its replacement. +#[derive(Clone)] +pub(super) struct Registration { + hosts: Weak>>, + replica: u64, + cell: Weak>>, + token: Arc<()>, +} + +impl Registration { + pub fn activate(&self) { + let (Some(hosts), Some(cell)) = (self.hosts.upgrade(), self.cell.upgrade()) else { + return; + }; + let mut entries = hosts.lock(); + if let Some(entry) = entries.get_mut(&self.replica) + && Arc::ptr_eq(&entry.cell, &cell) + { + entry.registration = Some(self.token.clone()); + } + } + + pub fn close_if_current(&self) -> Option { + let (Some(hosts), Some(cell)) = (self.hosts.upgrade(), self.cell.upgrade()) else { + return None; + }; + let mut entries = hosts.lock(); + let entry = entries.get_mut(&self.replica)?; + if !Arc::ptr_eq(&entry.cell, &cell) + || !entry + .registration + .as_ref() + .is_some_and(|token| Arc::ptr_eq(token, &self.token)) + { + return None; + } + Some(request_close(&hosts, self.replica, entry)) + } +} + +pub(super) struct CloseOwner { + pin: Pin, + completion: Arc, +} + +pub(super) struct CloseRequest { + pub completion: Arc, + pub owner: Option, +} + +pub(super) fn close(hosts: &Hosts, replica: u64) -> Option { + let mut entries = hosts.lock(); + let entry = entries.get_mut(&replica)?; + Some(request_close(hosts, replica, entry)) +} + +fn request_close(hosts: &Hosts, replica: u64, entry: &mut Entry) -> CloseRequest { + if let Some(completion) = &entry.closing { + return CloseRequest { + completion: completion.clone(), + owner: None, + }; + } + let completion = Arc::new(Closing { + result: watch::Sender::new(None), + }); + entry.closing = Some(completion.clone()); + entry.pins += 1; + CloseRequest { + completion: completion.clone(), + owner: Some(CloseOwner { + pin: Pin { + hosts: Arc::downgrade(hosts), + replica, + cell: entry.cell.clone(), + }, + completion, + }), + } +} + +impl CloseOwner { + pub async fn run(self) { + let mut guard = self.pin.cell.clone().write_owned().await; + // A prior cold owner can report an unconfirmed writer before this + // queued close obtains its guard. An empty cell is not proof of closure. + if self.completion.result.borrow().is_some() { + return; + } + let result = match guard.take() { + Some(host) => super::lifecycle::close_host(host).await, + None => Ok(()), + }; + let writer_closed = !matches!(result, Err(super::lifecycle::CloseFailure::WriterUnconfirmed)); + // Publish under the cell lock, then release it before final registry + // unpin/removal. Closing stays set during both steps, so reopen waits. + self.pin.publish(None); + drop(guard); + if let Some(hosts) = self.pin.hosts.upgrade() { + let mut entries = hosts.lock(); + if let Some(entry) = entries.get_mut(&self.pin.replica) { + debug_assert!(Arc::ptr_eq(&entry.cell, &self.pin.cell)); + if writer_closed { + entry.closing = None; + } + } + } + self.completion + .result + .send_replace(Some(result.map_err(|error| Arc::from(error.to_string())))); + // self.pin drops last. An unrelated idle Arc cannot delay completion. + } +} diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 6b1c5054cde..c0d90ac2333 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,4 +1,6 @@ use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::check_hosted_admission; use crate::database_logger::{BacktraceFrame, BacktraceProvider, LogLevel, ModuleBacktrace, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; use crate::error::{DBError, DatastoreError, IndexError, NodesError}; @@ -18,6 +20,7 @@ use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, IndexScanPointOrRange, MutTxId}; +use spacetimedb_datastore::system_tables::{is_module_restricted_index, is_module_restricted_table}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{http as st_http, ConnectionId, Identity, Timestamp}; use spacetimedb_primitives::{ColId, ColList, IndexId, TableId}; @@ -49,6 +52,10 @@ pub struct InstanceEnv { pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. pub func_name: Option, + /// Set by trusted host dispatch for this invocation, independently of JWTs + /// and connection IDs. Cleared before each new function call. + call_auth_flags: u32, + hosted_auth: Option>, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. @@ -235,6 +242,8 @@ impl InstanceEnv { // run a function func_type: FuncCallType::Reducer, func_name: None, + call_auth_flags: 0, + hosted_auth: None, in_anon_tx: false, procedure_last_tx_offset: None, } @@ -251,6 +260,20 @@ impl InstanceEnv { self.start_instant = Instant::now(); self.func_type = func_type; self.func_name = Some(name); + self.call_auth_flags = 0; + self.hosted_auth = None; + } + + pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { + self.call_auth_flags = flags; + } + + pub(crate) fn set_hosted_auth(&mut self, auth: Option>) { + self.hosted_auth = auth; + } + + pub(crate) fn get_call_auth_flags(&self) -> u32 { + self.call_auth_flags } /// Returns the name of the most recent reducer to be run in this environment, @@ -281,9 +304,37 @@ impl InstanceEnv { self.replica_ctx.relational_db() } + /// Dedicated read-only environment access. Missing reads also register a + /// dependency so a later insert refreshes a view that observed absence. + pub(crate) fn env_get(&self, key: &str) -> Result, NodesError> { + use crate::db::environment; + use spacetimedb_datastore::system_tables::ST_ENV_ID; + spacetimedb_lib::environment::validate_key(key).map_err(|_| NodesError::InvalidEnvironmentKey)?; + let read = |state: &_| environment::get(state, key).map_err(|err| NodesError::from(DBError::Other(err.into()))); + if let Ok(mut tx) = self.get_tx() { + tx.record_table_scan(&self.func_type, ST_ENV_ID); + return read(&*tx); + } + if !matches!(self.func_type, FuncCallType::Procedure) { + return Err(NodesError::NotInTransaction); + } + self.relational_db().with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, self.relational_db(), self.hosted_auth.as_deref()) + .map_err(|err| NodesError::HostedInvocationRejected(err.to_string()))?; + environment::get(tx, key).map_err(|err| NodesError::from(DBError::Other(err.into()))) + }) + } + pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { - let tx = &mut *self.get_tx()?; - Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?) + if let Ok(tx) = self.get_tx() { + return Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?); + } + // Procedures may inspect authentication before opening their first + // transaction. Use a short read transaction without manufacturing an + // internal caller or dropping the real connection's JWT. + Ok(self.relational_db().with_read_only(Workload::Internal, |tx| { + tx.get_jwt_payload(connection_id).map_err(DBError::from) + })?) } #[tracing::instrument(level = "trace", skip_all)] @@ -355,9 +406,20 @@ impl InstanceEnv { count } + /// Engine-owned deployment/authentication records and environment values + /// are reachable only through their dedicated host interfaces. + fn require_module_table(table_id: TableId) -> Result<(), NodesError> { + if is_module_restricted_table(table_id) { + Err(NodesError::TableNotFound) + } else { + Ok(()) + } + } + pub fn insert(&self, table_id: TableId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, insert_flags) = stdb .insert(tx, table_id, buffer) @@ -436,6 +498,7 @@ impl InstanceEnv { pub fn update(&self, table_id: TableId, index_id: IndexId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, update_flags) = stdb .update(tx, table_id, index_id, buffer) @@ -479,6 +542,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, _, iter) = stdb.index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = iter.map(|row_ref| row_ref.pointer()).collect::>(); @@ -499,6 +563,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, iter) = stdb.index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = match iter { IndexScanPointOrRange::Point(_, iter) => iter.map(|row_ref| row_ref.pointer()).collect(), @@ -540,6 +605,7 @@ impl InstanceEnv { pub fn datastore_delete_all_by_eq_bsatn(&self, table_id: TableId, relation: &[u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Track the number of bytes coming from the caller tx.metrics.bytes_scanned += relation.len(); @@ -563,6 +629,7 @@ impl InstanceEnv { pub fn clear(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let rows_deleted = stdb.clear_table(tx, table_id).map_err(NodesError::from)?; @@ -584,6 +651,7 @@ impl InstanceEnv { // Query the table id from the name. stdb.table_id_from_name_mut(tx, table_name)? + .filter(|id| !is_module_restricted_table(*id)) .ok_or(NodesError::TableNotFound) } @@ -598,6 +666,7 @@ impl InstanceEnv { // Query the index id from the name. stdb.index_id_from_name_mut(tx, index_name)? + .filter(|id| !is_module_restricted_index(*id)) .ok_or(NodesError::IndexNotFound) } @@ -609,6 +678,7 @@ impl InstanceEnv { pub fn datastore_table_row_count(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Query the row count for id. stdb.table_row_count_mut(tx, table_id) @@ -625,6 +695,7 @@ impl InstanceEnv { table_id: TableId, ) -> Result>, NodesError> { let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Open the iterator. let iter = self.relational_db().iter_mut(tx, table_id)?; @@ -652,6 +723,7 @@ impl InstanceEnv { // Open index iterator let (table_id, point, iter) = self.relational_db().index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (chunks, rows_scanned, bytes_scanned) = ChunkedWriter::collect_iter(pool, iter); @@ -682,6 +754,7 @@ impl InstanceEnv { let (table_id, iter) = self.relational_db() .index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (point, (chunks, rows_scanned, bytes_scanned)) = match iter { @@ -741,6 +814,10 @@ impl InstanceEnv { let tx = self .relational_db() .begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + if let Err(err) = check_hosted_admission(&tx, self.relational_db(), self.hosted_auth.as_deref()) { + let _ = tx.rollback(); + return Err(NodesError::HostedInvocationRejected(err.to_string())); + } self.tx.set_raw(tx); self.in_anon_tx = true; @@ -1399,6 +1476,112 @@ mod test { Ok(db) } + #[test] + fn environment_reads_use_active_transaction_and_track_missing_view_dependency() -> Result<()> { + use crate::db::environment; + use spacetimedb_datastore::locking_tx_datastore::ViewCallInfo; + use spacetimedb_datastore::system_tables::ST_ENV_ID; + use spacetimedb_primitives::{ViewFnPtr, ViewId}; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + assert!(matches!(env.env_get("A"), Err(NodesError::NotInTransaction))); + env.func_type = FuncCallType::Procedure; + assert_eq!(env.env_get("A")?, None); + let mut tx = begin_mut_tx(&db); + environment::set(&db, &mut tx, "A", "uncommitted")?; + env.tx.set_raw(tx); + assert_eq!(env.env_get("A")?.as_deref(), Some("uncommitted")); + let view = ViewCallInfo { + view_id: ViewId(88), + table_id: ST_ENV_ID, + fn_ptr: ViewFnPtr(0), + sender: None, + }; + env.func_type = FuncCallType::View(view.clone()); + assert_eq!(env.env_get("MISSING")?, None); + let tx = env.tx.take()?; + db.commit_tx(tx)?; + let mut tx = begin_mut_tx(&db); + environment::set(&db, &mut tx, "MISSING", "")?; + assert!(tx.views_for_refresh().any(|dependency| dependency == &view)); + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + env.func_type = FuncCallType::Procedure; + assert_eq!(env.env_get("MISSING")?, None); + assert!(matches!(env.env_get("A=B"), Err(NodesError::InvalidEnvironmentKey))); + Ok(()) + } + + #[test] + fn procedure_environment_snapshot_rechecks_durable_fence_and_expiry() -> Result<()> { + use crate::auth::{ + hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, + JwtKeys, + }; + use crate::db::{deployment::install_container_fence, environment}; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + use std::time::SystemTime; + let db = relational_db()?; + db.hosted_admission().begin()?.complete()?; + let (mut env, _runtime) = instance_env(db.clone())?; + env.func_type = FuncCallType::Procedure; + let keys = JwtKeys::generate()?; + let validator = HostedTokenValidator::new([("platform.test".into(), keys.public)])?; + let mint = |issued: SystemTime| -> Result<_> { + let binding = HostedTokenBinding { + source_database: db.database_identity(), + target_database: db.database_identity(), + generation: 1, + grant_revision: 1, + lease_expires_at: issued + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "platform.test", + &binding, + issued, + issued + Duration::from_secs(20), + "env-test", + )?; + Ok(Arc::new(validator.validate_token( + &token, + db.database_identity(), + issued, + |_, _, _| Some(binding), + )?)) + }; + let fence = |generation, allowed| StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: generation, + target_set_hash: Hash::ZERO, + allowed, + }; + db.with_auto_commit(Workload::ForTests, |tx| -> Result<()> { + install_container_fence(&db, tx, &fence(1, true))?; + environment::set(&db, tx, "VALUE", "available")?; + Ok(()) + })?; + env.set_hosted_auth(Some(mint(SystemTime::now())?)); + assert_eq!(env.env_get("VALUE")?.as_deref(), Some("available")); + env.set_hosted_auth(Some(mint(SystemTime::now() - Duration::from_secs(60))?)); + assert!(matches!( + env.env_get("VALUE"), + Err(NodesError::HostedInvocationRejected(_)) + )); + env.set_hosted_auth(Some(mint(SystemTime::now())?)); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &fence(2, false)) + })?; + assert!(matches!( + env.env_get("VALUE"), + Err(NodesError::HostedInvocationRejected(_)) + )); + env.set_hosted_auth(None); + assert_eq!(env.env_get("VALUE")?.as_deref(), Some("available")); + Ok(()) + } + /// Generate a `ProductValue` for use in [create_table_with_index] fn product_row(i: usize) -> ProductValue { let str = i.to_string(); @@ -1407,6 +1590,92 @@ mod test { product!(id, str) } + #[test] + fn module_cannot_access_hosted_system_records_by_guessed_ids() -> Result<()> { + use spacetimedb_datastore::system_tables::{ + ST_CONNECTION_AUTH_ID, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, + ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, ST_PUBLISH_FENCE_ID, + }; + let db = relational_db()?; + let (env, _runtime) = instance_env(db.clone())?; + let mut slot = env.tx.clone(); + let protected = [ + (ST_ENV_ID, "st_env", to_vec("TOKEN")?), + (ST_DEPLOYMENT_ID, "st_deployment", to_vec(&0u8)?), + (ST_PUBLISH_FENCE_ID, "st_publish_fence", to_vec(&0u8)?), + (ST_DEPLOYMENT_OPERATION_ID, "st_deployment_operation", to_vec(&0u128)?), + (ST_CONNECTION_AUTH_ID, "st_connection_auth", to_vec(&0u128)?), + (ST_CONTAINER_ENVIRONMENT_ID, "st_container_environment", to_vec(&0u64)?), + ( + ST_CONTAINER_FENCE_ID, + "st_container_fence", + to_vec(&spacetimedb_sats::u256::ZERO)?, + ), + ]; + let tx = begin_mut_tx(&db); + let (tx, result) = slot.set(tx, || -> Result<()> { + for (table, name, point) in &protected { + // Host lookup remains available, independently of module lookup. + let (index, index_name) = { + let tx = env.get_tx()?; + let schema = db.schema_for_table_mut(&tx, *table)?; + let index = &schema.indexes[0]; + (index.index_id, index.index_name.to_string()) + }; + assert!(matches!(env.table_id_from_name(name), Err(NodesError::TableNotFound))); + assert!(matches!( + env.index_id_from_name(&index_name), + Err(NodesError::IndexNotFound) + )); + assert!(matches!(env.insert(*table, &mut []), Err(NodesError::TableNotFound))); + assert!(matches!( + env.update(*table, index, &mut []), + Err(NodesError::TableNotFound) + )); + assert!(matches!(env.clear(*table), Err(NodesError::TableNotFound))); + assert!(matches!( + env.datastore_table_row_count(*table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_table_scan_bsatn_chunks(&mut ChunkPool::default(), *table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_all_by_eq_bsatn(*table, &[]), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_index_scan_point_bsatn_chunks(&mut ChunkPool::default(), index, point), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_point_bsatn(index, point), + Err(NodesError::TableNotFound) + )); + let bound = to_vec(&Bound::::Unbounded)?; + assert!(matches!( + env.datastore_index_scan_range_bsatn_chunks( + &mut ChunkPool::default(), + index, + &[], + 0.into(), + &bound, + &bound + ), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_range_bsatn(index, &[], 0.into(), &bound, &bound), + Err(NodesError::TableNotFound) + )); + } + Ok(()) + }); + let _ = db.rollback_mut_tx(tx); + result + } + /// Generate a BSATN encoded row for use in [create_table_with_index] fn bsatn_row(i: usize) -> Result> { Ok(to_vec(&product_row(i))?) diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index daa506a8cf5..97acb630a86 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -10,7 +10,10 @@ use spacetimedb_lib::ProductValue; use spacetimedb_schema::def::deserialize::{ArgsSeed, FunctionDef}; use spacetimedb_schema::def::ModuleDef; +pub mod container_environment; +pub mod container_fence; mod disk_storage; +pub mod empty_module; mod host_controller; mod module_common; #[allow(clippy::too_many_arguments)] @@ -187,6 +190,8 @@ pub enum AbiCall { Identity, JwtLength, GetJwt, + GetCallAuthFlags, + EnvGet, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/module_common.rs b/crates/core/src/host/module_common.rs index bff864759f3..ebe857c5cb1 100644 --- a/crates/core/src/host/module_common.rs +++ b/crates/core/src/host/module_common.rs @@ -19,7 +19,7 @@ use std::sync::Arc; pub fn build_common_module_from_raw( mcc: ModuleCreationContext, raw_def: RawModuleDef, -) -> Result { +) -> Result> { // Perform a bunch of validation on the raw definition. let def: ModuleDef = raw_def.try_into()?; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 1a8cf3257f9..58969a6da93 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2,9 +2,12 @@ use super::{ ArgsTuple, FunctionArgs, InvalidProcedureArguments, InvalidReducerArguments, ReducerCallResult, ReducerId, ReducerOutcome, Scheduler, }; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::{check_hosted_admission, InvocationCaller, SqlCallAuth}; use crate::client::messages::{OneOffQueryResponseMessage, ProcedureResultMessage, SerializableMessage}; use crate::client::{ClientActorId, ClientConnectionSender, WsVersion}; use crate::database_logger::{DatabaseLogger, LogLevel, Record}; +use crate::db::deployment::{self, CommitAdmission, DeploymentCommit, PublishResult as DeploymentPublishResult}; use crate::db::relational_db::{RelationalDB, Tx}; use crate::energy::EnergyQuanta; use crate::error::DBError; @@ -73,11 +76,16 @@ use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; use std::fmt; use std::num::NonZeroUsize; -use std::sync::atomic::AtomicBool; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; use tokio::sync::{oneshot, OwnedSemaphorePermit, Semaphore}; +#[cfg(test)] +mod drain_tests; +mod operations; +use operations::ModuleOperations; +pub(in crate::host) use operations::OperationLease; + #[derive(Debug, Default, Clone, From)] pub struct DatabaseUpdate { pub tables: SmallVec<[DatabaseTableUpdate; 1]>, @@ -430,6 +438,7 @@ impl WasmtimeModuleHost { label: &str, on_panic: Arc, timer_guard: CallTimerGuard, + operation: OperationLease, arg: A, wasm: impl FnOnce(A, &mut ModuleInstance) + Send + 'static, ) where @@ -437,6 +446,7 @@ impl WasmtimeModuleHost { { let label = label.to_owned(); self.main_executor.enqueue_job(move |state| { + let _operation = operation; scopeguard::defer_on_unwind!({ log::warn!("wasm main operation {label} panicked"); on_panic(); @@ -454,15 +464,20 @@ impl WasmtimeModuleHost { label: &str, on_panic: Arc, timer_guard: CallTimerGuard, + operation: OperationLease, arg: A, wasm: impl AsyncFnOnce(A, &mut ModuleInstance) + Send + 'static, ) where A: Send + 'static, { let instance_manager = self.procedure_instances.clone(); - let ModuleInstanceLease { instance, slot } = instance_manager.get_instance().await; + let ModuleInstanceLease { instance, slot } = instance_manager + .get_instance(Some(operation.clone())) + .await + .unwrap_or_else(|never| match never {}); let label = label.to_owned(); self.procedure_executor.enqueue_job(async move || { + let _operation = operation; scopeguard::defer_on_unwind!({ log::warn!("wasm procedure {label} panicked"); on_panic(); @@ -485,7 +500,8 @@ struct V8ModuleHost { /// A module; used as a bound on `InstanceManager`. trait GenericModule { type Instance: GenericModuleInstance; - async fn create_instance(&self) -> Self::Instance; + type CreationError; + async fn create_instance(&self, operation: Option) -> Result; fn host_type(&self) -> HostType; } @@ -515,8 +531,10 @@ impl GenericModuleInstance for Box { impl GenericModule for Arc { type Instance = Box; - async fn create_instance(&self) -> Self::Instance { - Box::new((**self).create_instance()) + type CreationError = std::convert::Infallible; + async fn create_instance(&self, operation: Option) -> Result { + let _operation = operation; + Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { HostType::Wasm @@ -525,8 +543,10 @@ impl GenericModule for Arc { impl GenericModule for Arc { type Instance = Box; - async fn create_instance(&self) -> Self::Instance { - Box::new((**self).create_instance()) + type CreationError = std::convert::Infallible; + async fn create_instance(&self, operation: Option) -> Result { + let _operation = operation; + Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { HostType::Wasm @@ -535,8 +555,9 @@ impl GenericModule for Arc { impl GenericModule for super::v8::JsModule { type Instance = super::v8::JsProcedureInstance; - async fn create_instance(&self) -> Self::Instance { - self.create_instance().await + type CreationError = anyhow::Error; + async fn create_instance(&self, operation: Option) -> Result { + self.create_instance_for_operation(operation).await } fn host_type(&self) -> HostType { HostType::Js @@ -590,18 +611,33 @@ fn extract_trapped(res: Result<(T, bool), E>) -> (Result, bool) { pub(crate) fn init_database( replica_ctx: &ReplicaContext, module_def: &ModuleDef, + module_hash: Hash, program: Program, + deployment: Option, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResult, bool), ) -> (anyhow::Result>, bool) { - extract_trapped(init_database_inner(replica_ctx, module_def, program, call_reducer)) + extract_trapped(init_database_inner( + replica_ctx, + module_def, + module_hash, + program, + deployment, + call_reducer, + )) } fn init_database_inner( replica_ctx: &ReplicaContext, module_def: &ModuleDef, + module_hash: Hash, program: Program, + deployment: Option, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResult, bool), ) -> anyhow::Result<(Option, bool)> { + anyhow::ensure!( + module_hash == program.hash && spacetimedb_lib::hash_bytes(&program.bytes) == program.hash, + "program does not match the instantiated module" + ); log::debug!("init database"); let timestamp = Timestamp::now(); let stdb = replica_ctx.relational_db(); @@ -610,6 +646,28 @@ fn init_database_inner( let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); let auth_ctx = AuthCtx::for_current(owner_identity); + let (tx, admission) = stdb.with_auto_rollback(tx, |tx| { + if let Some(request) = &deployment { + deployment::validate_deployment_program(request, &program, module_def)?; + // New databases install the bootstrap fence, schema, program and + // receipt in this one transaction. An interrupted initialization + // cannot leave a successful partial deployment behind. + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use spacetimedb_datastore::system_tables::ST_MODULE_ID; + if tx.iter(ST_MODULE_ID)?.next().is_none() { + deployment::install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + } + deployment::check_deployment_commit(tx, request, timestamp, &Default::default()) + } else { + deployment::require_unmanaged_publication(tx)?; + Ok(CommitAdmission::Ready) + } + })?; + if matches!(admission, CommitAdmission::AlreadyCommitted(_)) { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + return Ok((None, false)); + } let (tx, ()) = stdb .with_auto_rollback(tx, |tx| { // Create all in-memory tables defined by the module, @@ -643,6 +701,10 @@ fn init_database_inner( stdb.set_initialized(tx, program)?; + if let Some(request) = &deployment { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default())?; + } + anyhow::Ok(()) }) .inspect_err(|e| log::error!("{e:?}"))?; @@ -690,6 +752,13 @@ pub fn call_identity_connected( stdb.report_mut_tx_metrics(reducer_name, metrics, None); }); + let caller = InvocationCaller::from(&caller_auth); + let flags = caller + .flags_for(module.database_identity, &module.module_def) + .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; + check_hosted_admission(&*mut_tx, stdb, caller.hosted.as_deref()) + .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; + mut_tx .insert_st_client( caller_auth.claims.identity, @@ -699,13 +768,23 @@ pub fn call_identity_connected( .map_err(DBError::from) .map_err(Box::new)?; + if caller.hosted.is_some() { + crate::db::deployment::record_connection_auth( + &mut mut_tx, + caller_connection_id, + caller_auth.claims.identity, + flags, + ) + .map_err(|err| ClientConnectedError::DBError(Box::new(DBError::Other(err.into()))))?; + } + if let Some((reducer_id, reducer_def)) = reducer_lookup { // The module defined a lifecycle reducer to handle new connections. // Call this reducer. // If the call fails (as in, something unexpectedly goes wrong with guest execution), // abort the connection: we can't really recover. let tx = Some(ScopeGuard::into_inner(mut_tx)); - let params = ModuleHost::call_reducer_params( + let mut params = ModuleHost::call_reducer_params( module, caller_auth.claims.identity, Some(caller_connection_id), @@ -717,6 +796,8 @@ pub fn call_identity_connected( FunctionArgs::Nullary, ) .map_err(ReducerCallError::from)?; + params.call_auth_flags = flags; + params.hosted_auth = caller.hosted; let (reducer_outcome, trapped) = call_reducer(tx, params); *trapped_slot = trapped; @@ -761,6 +842,9 @@ pub struct CallReducerParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + /// Verified invocation authority. Bit 0 is internal; never client-decoded. + pub(crate) call_auth_flags: u32, + pub(crate) hosted_auth: Option>, pub client: Option>, pub request_id: Option, pub timer: Option, @@ -781,6 +865,8 @@ impl CallReducerParams { timestamp, caller_identity, caller_connection_id: ConnectionId::ZERO, + call_auth_flags: 1, + hosted_auth: None, client: None, request_id: None, timer: None, @@ -986,7 +1072,7 @@ impl ViewCommandErrorTarget { pub(in crate::host) struct SqlCommand { pub(in crate::host) db: Arc, pub(in crate::host) sql_text: String, - pub(in crate::host) auth: AuthCtx, + pub(in crate::host) auth: SqlCallAuth, pub(in crate::host) subs: Option, } @@ -1093,6 +1179,8 @@ pub struct CallProcedureParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + pub(crate) call_auth_flags: u32, + pub(crate) hosted_auth: Option>, pub timer: Option, pub procedure_id: ProcedureId, pub args: ArgsTuple, @@ -1111,6 +1199,8 @@ impl CallProcedureParams { timestamp, caller_identity, caller_connection_id: ConnectionId::ZERO, + call_auth_flags: 1, + hosted_auth: None, timer: None, procedure_id, args, @@ -1142,7 +1232,7 @@ struct ModuleInstanceManager { struct ModuleInstanceLease { instance: I, - slot: Option, + slot: Option>, } /// Holds the single shared instance used by the JS main execution path. @@ -1315,22 +1405,18 @@ impl ModuleInstanceManager { } } - async fn with_instance(&self, f: impl AsyncFnOnce(M::Instance) -> (R, M::Instance)) -> R { - let ModuleInstanceLease { instance, slot } = self.get_instance().await; - let (res, instance) = f(instance).await; - self.return_instance(ModuleInstanceLease { instance, slot }); - res - } - - async fn get_instance(&self) -> ModuleInstanceLease { + async fn get_instance( + &self, + operation: Option, + ) -> Result, M::CreationError> { let slot = if let Some(instance_slots) = &self.instance_slots { - Some( + Some(Arc::new( instance_slots .clone() .acquire_owned() .await .expect("module instance slot semaphore should not close"), - ) + )) } else { None }; @@ -1343,13 +1429,16 @@ impl ModuleInstanceManager { instance } else { let start_time = std::time::Instant::now(); - let res = self.module.create_instance().await; + let res = self + .module + .create_instance(operation.map(|operation| operation.with_pool_slot(slot.clone()))) + .await?; let elapsed_time = start_time.elapsed(); self.metrics.observe_instance_created(elapsed_time); res }; - ModuleInstanceLease { instance, slot } + Ok(ModuleInstanceLease { instance, slot }) } fn return_instance(&self, lease: ModuleInstanceLease) { @@ -1388,10 +1477,10 @@ pub struct ModuleHost { /// Called whenever a reducer call on this host panics. on_panic: Arc, - /// Marks whether this module has been closed by [`Self::exit`]. - /// - /// When this is true, most operations will fail with [`NoSuchModule`]. - closed: Arc, + /// Shared admission and physical-operation drainage for this module. + /// [`Self::exit`] closes admission, rejects new work with [`NoSuchModule`], + /// and waits for accepted executor jobs, including cancelled callers. + operations: Arc, } impl fmt::Debug for ModuleHost { @@ -1407,12 +1496,19 @@ pub struct WeakModuleHost { info: Arc, inner: Weak, on_panic: Weak, - closed: Weak, + operations: Weak, } #[derive(Debug)] pub enum UpdateDatabaseResult { NoUpdateNeeded, + /// A prior successful commit is returned without replacing the current + /// module, which may already be newer than this retried publication. + DeploymentAlreadyCommitted { + result: DeploymentPublishResult, + tx_offset: TransactionOffset, + durable_offset: Option, + }, UpdatePerformed { /// The transaction offset of the successful database update. tx_offset: TransactionOffset, @@ -1437,6 +1533,7 @@ impl UpdateDatabaseResult { self, UpdateDatabaseResult::UpdatePerformed { .. } | UpdateDatabaseResult::NoUpdateNeeded + | UpdateDatabaseResult::DeploymentAlreadyCommitted { .. } | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } ) } @@ -1446,6 +1543,45 @@ impl UpdateDatabaseResult { #[error("no such module")] pub struct NoSuchModule; +#[derive(thiserror::Error, Debug)] +enum PooledCallError { + #[error(transparent)] + NoSuchModule(#[from] NoSuchModule), + #[error("module instance startup failed: {0}")] + Startup(anyhow::Error), +} + +impl From for ProcedureCallError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => { + Self::InternalError(format!("module instance startup failed: {error:#}")) + } + } + } +} + +impl From for HttpHandlerCallError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => { + Self::InternalError(format!("module instance startup failed: {error:#}")) + } + } + } +} + +impl From for CallScheduledFunctionError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => Self::InstanceStartup(error), + } + } +} + #[derive(thiserror::Error, Debug)] pub enum ReducerCallError { #[error(transparent)] @@ -1714,7 +1850,7 @@ impl ModuleHost { info, inner, on_panic, - closed: Arc::new(AtomicBool::new(false)), + operations: Arc::new(ModuleOperations::default()), } } @@ -1733,20 +1869,6 @@ impl ModuleHost { matches!(&*self.inner, ModuleHostInner::Js(_)) } - fn is_marked_closed(&self) -> bool { - // `self.closed` isn't used for any synchronization, it's just a shared flag, - // so `Ordering::Relaxed` is sufficient. - self.closed.load(std::sync::atomic::Ordering::Relaxed) - } - - fn guard_closed(&self) -> Result<(), NoSuchModule> { - if self.is_marked_closed() { - Err(NoSuchModule) - } else { - Ok(()) - } - } - fn start_call_timer(&self, label: &str) -> CallTimerGuard { // Record the time until our function starts running. let queue_timer = WORKER_METRICS @@ -1784,7 +1906,7 @@ impl ModuleHost { R: Send + 'static, A: Send + 'static, { - self.guard_closed()?; + let operation = self.operations.begin()?; let timer_guard = self.start_call_timer(label); scopeguard::defer_on_unwind!({ @@ -1797,6 +1919,7 @@ impl ModuleHost { let executor = host.main_executor.clone(); executor .run_job(move |state| { + let _operation = operation; state.with_instance(move |inst| { drop(timer_guard); wasm(arg, inst) @@ -1807,7 +1930,7 @@ impl ModuleHost { ModuleHostInner::Js(host) => { drop(timer_guard); host.main_instance - .with_instance(|inst| async move { js(arg, &inst).await }) + .with_instance(|inst| async move { js(arg, &inst.with_operation(operation)).await }) .await } }) @@ -1823,12 +1946,12 @@ impl ModuleHost { arg: A, wasm: impl AsyncFnOnce(A, &mut ModuleInstance) -> R + Send + 'static, js: impl AsyncFnOnce(A, &JsProcedureInstance) -> R, - ) -> Result + ) -> Result where R: Send + 'static, A: Send + 'static, { - self.guard_closed()?; + let operation = self.operations.begin()?; let timer_guard = self.start_call_timer(label); scopeguard::defer_on_unwind!({ @@ -1839,27 +1962,35 @@ impl ModuleHost { Ok(match &*self.inner { ModuleHostInner::Wasm(host) => { let executor = host.procedure_executor.clone(); - let instance_manager = host.procedure_instances.clone(); - instance_manager - .with_instance(async move |mut inst| { - executor - .run_job(async move || { - drop(timer_guard); - let res = wasm(arg, &mut inst).await; - (res, inst) - }) - .await + let manager = host.procedure_instances.clone(); + let ModuleInstanceLease { mut instance, slot } = manager + .get_instance(Some(operation.clone())) + .await + .unwrap_or_else(|never| match never {}); + let operation = operation.with_pool_slot(slot); + executor + .run_job(async move || { + let _operation = operation; + drop(timer_guard); + let result = wasm(arg, &mut instance).await; + manager.return_instance(ModuleInstanceLease { instance, slot: None }); + result }) .await } ModuleHostInner::Js(host) => { - host.procedure_instances - .with_instance(async |inst| { - drop(timer_guard); - let res = js(arg, &inst).await; - (res, inst) - }) + let mut lease = host + .procedure_instances + .get_instance(Some(operation.clone())) .await + .map_err(PooledCallError::Startup)?; + lease + .instance + .set_operation(Some(operation.with_pool_slot(lease.slot.take()))); + drop(timer_guard); + let result = js(arg, &lease.instance).await; + self.return_js_procedure_instance(lease); + result } }) } @@ -1870,7 +2001,7 @@ impl ModuleHost { label: &str, arg: A, js: impl FnOnce(A, JsMainInstance, JsFatalHook) -> JsFut, - wasm: impl FnOnce(A, &WasmtimeModuleHost, JsFatalHook, CallTimerGuard) -> Result<(), NoSuchModule>, + wasm: impl FnOnce(A, &WasmtimeModuleHost, JsFatalHook, CallTimerGuard, OperationLease) -> Result<(), NoSuchModule>, ) -> Result<(), NoSuchModule> where A: Send + 'static, @@ -1882,21 +2013,20 @@ impl ModuleHost { (self.on_panic)(); }); + let operation = self.operations.begin()?; match &*self.inner { ModuleHostInner::Js(js_host) => { - self.guard_closed()?; let on_panic = self.on_panic.clone(); js_host .main_instance - .with_instance(|inst| js(arg, inst, on_panic)) + .with_instance(|inst| js(arg, inst.with_operation(operation), on_panic)) .await; Ok(()) } ModuleHostInner::Wasm(wasm_host) => { - self.guard_closed()?; let timer_guard = self.start_call_timer(label); let on_panic = self.on_panic.clone(); - wasm(arg, wasm_host, on_panic, timer_guard) + wasm(arg, wasm_host, on_panic, timer_guard, operation) } } } @@ -1913,12 +2043,13 @@ impl ModuleHost { label, (cmd, metric), |(cmd, metric), inst, on_panic| async move { inst.enqueue_call_view(cmd, metric, on_panic).await }, - move |(cmd, metric), wasm_host, on_panic, timer_guard| { + move |(cmd, metric), wasm_host, on_panic, timer_guard, operation| { let info = wasm_host.module.info(); wasm_host.enqueue_with_main_instance( label, on_panic, timer_guard, + operation, (cmd, metric), move |(cmd, metric), inst| { let result = inst.call_view(cmd); @@ -2017,7 +2148,7 @@ impl ModuleHost { } /// Invokes the `client_disconnected` reducer, if present, - /// then deletes the client’s rows from `st_client` and `st_connection_credentials`. + /// then deletes the client's rows from `st_client`, `st_connection_credentials`, and `st_connection_auth`. /// If the reducer fails, the rows are still deleted. /// Calling this on an already-disconnected client is a no-op. pub fn call_identity_disconnected_inner( @@ -2081,22 +2212,35 @@ impl ModuleHost { // The module defined a lifecycle reducer to handle disconnects. Call it. // If it succeeds, `WasmModuleInstance::call_reducer_with_tx` has already ensured // that `st_client` is updated appropriately. + let flags = crate::db::deployment::connection_auth_flags(&mut_tx, caller_connection_id, caller_identity) + .map_err(|err| { + InvalidReducerArguments(InvalidFunctionArguments { + err: err.into(), + function_name: reducer_name.clone().into(), + }) + }); let tx = Some(mut_tx); - let result = Self::call_reducer_params( - info, - caller_identity, - Some(caller_connection_id), - None, - None, - None, - reducer_id, - reducer_def, - FunctionArgs::Nullary, - ) - .map(|params| { - let (res, trapped) = call_reducer(tx, params); - *trapped_slot = trapped; - res + let result = flags.and_then(|flags| { + Self::call_reducer_params( + info, + caller_identity, + Some(caller_connection_id), + None, + None, + None, + reducer_id, + reducer_def, + FunctionArgs::Nullary, + ) + .map(|mut params| { + // This host event retains the connection's captured authority, + // including after credential expiry, revocation, or host recovery. + // It must not carry a live hosted proof that could block cleanup. + params.call_auth_flags = flags; + let (res, trapped) = call_reducer(tx, params); + *trapped_slot = trapped; + res + }) }); // If it failed, we still need to update `st_client`: the client's not coming back. @@ -2188,6 +2332,8 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + call_auth_flags: 0, + hosted_auth: None, client, request_id, timer, @@ -2198,7 +2344,7 @@ impl ModuleHost { fn reducer_call_params<'a>( &'a self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2206,6 +2352,10 @@ impl ModuleHost { reducer_name: &str, args: FunctionArgs, ) -> Result<(&'a ReducerDef, CallReducerParams), ReducerCallError> { + let flags = caller + .flags_for(self.info.database_identity, &self.info.module_def) + .map_err(|_| ReducerCallError::NoSuchReducer)?; + let caller_identity = caller.identity; let (reducer_id, reducer_def) = self .info .module_def @@ -2215,24 +2365,27 @@ impl ModuleHost { return Err(ReducerCallError::LifecycleReducer(lifecycle)); } - if reducer_def.visibility.is_private() && !self.is_database_owner(caller_identity) { + if !reducer_def + .visibility + .allows_invocation(flags & 1 != 0, self.is_database_owner(caller_identity)) + { return Err(ReducerCallError::NoSuchReducer); } - Ok(( + let mut params = Self::call_reducer_params( + &self.info, + caller_identity, + caller_connection_id, + client, + request_id, + timer, + reducer_id, reducer_def, - Self::call_reducer_params( - &self.info, - caller_identity, - caller_connection_id, - client, - request_id, - timer, - reducer_id, - reducer_def, - args, - )?, - )) + args, + )?; + params.call_auth_flags = flags; + params.hosted_auth = caller.hosted; + Ok((reducer_def, params)) } async fn call_reducer_with_params( @@ -2260,7 +2413,7 @@ impl ModuleHost { async fn with_reducer_call( &self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2271,7 +2424,7 @@ impl ModuleHost { ) -> Result { let res = async { let (reducer_def, params) = self.reducer_call_params( - caller_identity, + caller, caller_connection_id, client, request_id, @@ -2296,7 +2449,7 @@ impl ModuleHost { pub async fn call_reducer( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2305,7 +2458,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result { self.with_reducer_call( - caller_identity, + caller.into(), caller_connection_id, client, request_id, @@ -2319,7 +2472,7 @@ impl ModuleHost { pub async fn enqueue_reducer( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2328,7 +2481,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result<(), ReducerCallError> { self.with_reducer_call( - caller_identity, + caller.into(), caller_connection_id, client, request_id, @@ -2342,11 +2495,12 @@ impl ModuleHost { reducer_name, call.params, |params, inst, on_panic| async move { inst.enqueue_reducer(params, on_panic).await }, - move |params, wasm_host, on_panic, timer_guard| { + move |params, wasm_host, on_panic, timer_guard, operation| { wasm_host.enqueue_with_main_instance( &reducer_label, on_panic, timer_guard, + operation, params, move |params, inst| { let _ = inst.call_reducer(params); @@ -2471,10 +2625,15 @@ impl ModuleHost { &self, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result { + InvocationCaller { + identity: auth.caller(), + hosted: auth.hosted.clone(), + } + .flags_for(self.info.database_identity, &self.info.module_def)?; let cmd = SqlCommand { db, sql_text, @@ -2490,18 +2649,15 @@ impl ModuleHost { pub async fn call_procedure( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> CallProcedureReturn { let res = async { - let call = - self.prepare_procedure_call(caller_identity, caller_connection_id, timer, procedure_name, args)?; - self.call_procedure_with_params(&call.name, call.params) - .await - .map_err(Into::into) + let call = self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args)?; + self.call_procedure_with_params(&call.name, call.params).await } .await; @@ -2514,7 +2670,7 @@ impl ModuleHost { pub(crate) async fn enqueue_procedure( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, timer: Option, procedure_name: &str, @@ -2522,7 +2678,7 @@ impl ModuleHost { target: ProcedureResultTarget, ) -> Result<(), BroadcastError> { let PreparedProcedureCall { name, params } = - match self.prepare_procedure_call(caller_identity, caller_connection_id, timer, procedure_name, args) { + match self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args) { Ok(value) => value, Err(err) => { return self.send_procedure_error(procedure_name, timer, target, err); @@ -2536,14 +2692,39 @@ impl ModuleHost { (self.on_panic)(); }); - if let Err(err) = self.guard_closed() { - return self.send_procedure_error(&procedure_name, timer, target, err.into()); - } + let operation = match self.operations.begin() { + Ok(operation) => operation, + Err(err) => return self.send_procedure_error(&procedure_name, timer, target, err.into()), + }; match &*self.inner { ModuleHostInner::Js(host) => { - let lease = host.procedure_instances.get_instance().await; - let call = lease.instance.enqueue_procedure(params).await; + let mut lease = match host.procedure_instances.get_instance(Some(operation.clone())).await { + Ok(lease) => lease, + Err(error) => { + return self.send_procedure_error( + &procedure_name, + timer, + target, + PooledCallError::Startup(error).into(), + ); + } + }; + lease + .instance + .set_operation(Some(operation.with_pool_slot(lease.slot.take()))); + let call = match lease.instance.enqueue_procedure(params).await { + Ok(call) => call, + Err(error) => { + self.return_js_procedure_instance(lease); + return self.send_procedure_error( + &procedure_name, + timer, + target, + ProcedureCallError::InternalError(error.to_string()), + ); + } + }; let module = self.clone(); tokio::spawn(async move { match call.receive().await { @@ -2553,6 +2734,16 @@ impl ModuleHost { log::warn!("failed to send procedure result: {err:#}"); } } + JsProcedureCallCompletion::StartupFailed(error) => { + if let Err(error) = module.send_procedure_error( + &procedure_name, + timer, + target, + ProcedureCallError::InternalError(error.to_string()), + ) { + log::warn!("failed to send procedure startup error: {error:#}"); + } + } JsProcedureCallCompletion::Panicked | JsProcedureCallCompletion::WorkerExited => { log::warn!("detached JS procedure worker failed before returning a result"); (module.on_panic)(); @@ -2573,6 +2764,7 @@ impl ModuleHost { &procedure_name, on_panic, timer_guard, + operation, params, async move |params, inst| { let ret = inst.call_procedure(params).await; @@ -2592,10 +2784,11 @@ impl ModuleHost { } } - fn return_js_procedure_instance(&self, lease: ModuleInstanceLease) { + fn return_js_procedure_instance(&self, mut lease: ModuleInstanceLease) { let ModuleHostInner::Js(host) = &*self.inner else { return; }; + lease.instance.set_operation(None); host.procedure_instances.return_instance(lease); } @@ -2697,14 +2890,14 @@ impl ModuleHost { fn prepare_procedure_call( &self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> Result { let (procedure_def, params) = - self.procedure_call_params(caller_identity, caller_connection_id, timer, procedure_name, args)?; + self.procedure_call_params(caller, caller_connection_id, timer, procedure_name, args)?; Ok(PreparedProcedureCall { name: procedure_def.name.to_string(), params, @@ -2713,19 +2906,26 @@ impl ModuleHost { fn procedure_call_params<'a>( &'a self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> Result<(&'a ProcedureDef, CallProcedureParams), ProcedureCallError> { + let flags = caller + .flags_for(self.info.database_identity, &self.info.module_def) + .map_err(|_| ProcedureCallError::NoSuchProcedure)?; + let caller_identity = caller.identity; let (procedure_id, procedure_def) = self .info .module_def .procedure_full(procedure_name) .ok_or(ProcedureCallError::NoSuchProcedure)?; - if procedure_def.visibility.is_private() && !self.is_database_owner(caller_identity) { + if !procedure_def + .visibility + .allows_invocation(flags & 1 != 0, self.is_database_owner(caller_identity)) + { return Err(ProcedureCallError::NoSuchProcedure); } @@ -2740,6 +2940,8 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + call_auth_flags: flags, + hosted_auth: caller.hosted, timer, procedure_id, args, @@ -2756,7 +2958,7 @@ impl ModuleHost { &self, name: &str, params: CallProcedureParams, - ) -> Result { + ) -> Result { call_pooled_instance!( self, name, @@ -2764,6 +2966,7 @@ impl ModuleHost { |params, inst| inst.call_procedure(params).await, |params, inst| inst.call_procedure(params).await, ) + .map_err(Into::into) } pub async fn call_http_handler( @@ -2814,10 +3017,13 @@ impl ModuleHost { self, "scheduled procedure", params, - |params, inst| inst.call_scheduled_procedure(params).await, - |params, inst| inst.call_scheduled_procedure(params).await, + |params, inst| Ok(inst.call_scheduled_procedure(params).await), + |params, inst| inst + .call_scheduled_procedure(params) + .await + .map_err(|error| CallScheduledFunctionError::InstanceStartup(error.into())), ) - .map_err(Into::into) + .map_err(CallScheduledFunctionError::from)? } /// Materializes the views return by the `view_collector`, if not already materialized, @@ -3032,14 +3238,49 @@ impl ModuleHost { } pub async fn init_database(&self, program: Program) -> Result, InitDatabaseError> { - call_instance!( + self.init_database_with_deployment(program, None).await + } + + /// Host-only initialization following an authorized, durable control intent. + /// A new database installs its bootstrap fence in the initialization + /// transaction. Successful deployment initialization waits for durability + /// before its scheduler or container can be activated. + pub async fn init_database_with_deployment( + &self, + program: Program, + deployment: Option, + ) -> Result, InitDatabaseError> { + let confirm_deployment = deployment.is_some(); + let result = call_instance!( self, "", - program, - |p, inst| inst.init_database(p), - |p, inst| inst.init_database(p).await, + (program, deployment), + |(p, d), inst| inst.init_database(p, d), + |(p, d), inst| inst.init_database(p, d).await, )? - .map_err(InitDatabaseError::Other) + .map_err(InitDatabaseError::Other)?; + if confirm_deployment + && result.as_ref().is_none_or(|result| result.is_ok()) + && let Some(mut durability) = self.relational_db().durable_tx_offset() + { + // A read barrier after init includes its receipt even when the + // module has no init reducer. Do not block the async worker on + // the datastore lock while capturing that barrier. + let db = self.relational_db().clone(); + let offset = tokio::task::spawn_blocking(move || { + let tx = db.begin_tx(Workload::Internal); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + offset + }) + .await + .map_err(|error| InitDatabaseError::Other(error.into()))?; + durability + .wait_for(offset) + .await + .map_err(|error| InitDatabaseError::Other(error.into()))?; + } + Ok(result) } pub async fn update_database( @@ -3047,25 +3288,40 @@ impl ModuleHost { program: Program, old_module_info: Arc, policy: MigrationPolicy, + ) -> Result { + self.update_database_with_deployment(program, old_module_info, policy, None) + .await + } + + /// Commit the prepared deployment in the same transaction as migration. + pub async fn update_database_with_deployment( + &self, + program: Program, + old_module_info: Arc, + policy: MigrationPolicy, + deployment: Option, ) -> Result { call_instance!( self, "", - (program, old_module_info, policy), - |(a, b, c), inst| inst.update_database(a, b, c), - |(a, b, c), inst| inst.update_database(a, b, c).await, + (program, old_module_info, policy, deployment), + |(a, b, c, d), inst| inst.update_database(a, b, c, d), + |(a, b, c, d), inst| inst.update_database(a, b, c, d).await, )? } pub async fn exit(&self) { - // As in `Self::marked_closed`, `Relaxed` is sufficient because we're not synchronizing any external state. - self.closed.store(true, std::sync::atomic::Ordering::Relaxed); + // Admission and closure share one lock. Already admitted work retains + // its lease inside the physical executor after cancellation of a caller. + self.operations.close(); self.scheduler().close(); self.exited().await; } pub async fn exited(&self) { self.scheduler().closed().await; + self.operations.close(); + self.operations.drained().await; } pub fn inject_logs(&self, log_level: LogLevel, function_name: &str, message: &str) { @@ -3138,11 +3394,12 @@ impl ModuleHost { label, request, |request, inst, on_panic| async move { inst.enqueue_one_off_query(request, on_panic).await }, - move |request, wasm_host, on_panic, timer_guard| { + move |request, wasm_host, on_panic, timer_guard, operation| { let executor = wasm_host.main_executor.clone(); let info = wasm_host.module.info(); let label = label.to_owned(); executor.enqueue_job(move |_| { + let _operation = operation; scopeguard::defer_on_unwind!({ log::warn!("websocket one-off query operation {label} panicked"); on_panic(); @@ -3294,8 +3551,10 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { - ws_v1::OneOffTable { table_name, rows } + let result = check_hosted_admission(&*tx, &db, client.auth.hosted.as_ref()).and_then(|()| { + Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { + ws_v1::OneOffTable { table_name, rows } + }) }); let total_host_execution_duration = timer.elapsed().into(); @@ -3373,10 +3632,11 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = + let result = check_hosted_admission(&*tx, &db, client.auth.hosted.as_ref()).and_then(|()| { Self::execute_one_off_query::(&db, &tx, &auth, &query, &rlb_pool, |table, rows| { ws_v2::SingleTableRows { table, rows } - }); + }) + }); let (message, metrics) = match result { Ok((rows, metrics)) => { @@ -3414,6 +3674,7 @@ impl ModuleHost { /// for tables without primary keys. It is only used in the benchmarks. /// Note: this doesn't drop the table, it just clears it! pub fn clear_table(&self, table_name: &str) -> Result<(), anyhow::Error> { + let _operation = self.operations.begin()?; let db = self.relational_db(); db.with_auto_commit(Workload::Internal, |tx| { @@ -3435,7 +3696,7 @@ impl ModuleHost { info: self.info.clone(), inner: Arc::downgrade(&self.inner), on_panic: Arc::downgrade(&self.on_panic), - closed: Arc::downgrade(&self.closed), + operations: Arc::downgrade(&self.operations), } } @@ -3474,12 +3735,12 @@ impl WeakModuleHost { pub fn upgrade(&self) -> Option { let inner = self.inner.upgrade()?; let on_panic = self.on_panic.upgrade()?; - let closed = self.closed.upgrade()?; + let operations = self.operations.upgrade()?; Some(ModuleHost { info: self.info.clone(), inner, on_panic, - closed, + operations, }) } } diff --git a/crates/core/src/host/module_host/drain_tests.rs b/crates/core/src/host/module_host/drain_tests.rs new file mode 100644 index 00000000000..50ed2fd2906 --- /dev/null +++ b/crates/core/src/host/module_host/drain_tests.rs @@ -0,0 +1,395 @@ +//! Actual local database writers, Wasm/JS worker queues and cancelled callers. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::host_controller::{HostController, HostRuntimeConfig}; +use crate::util::jobs::JobCores; +use spacetimedb_auth::identity::SpacetimeIdentityClaims; +use spacetimedb_datastore::system_tables::ModuleKind; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; +use spacetimedb_paths::{server::ServerDataDir, FromPathUnchecked}; +use tokio::time::timeout; + +fn javascript() -> Program { + let mut definition = RawModuleDefV10Builder::new(); + definition.add_lifecycle_reducer( + Lifecycle::OnDisconnect, + "disconnected", + spacetimedb_sats::ProductType::unit(), + ); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(definition.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{register_hooks}} from "spacetime:sys@1.0"; + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), + __call_reducer__: () => ({{tag:"ok"}}) }}); + "# + ) + .into_bytes(), + ) +} + +fn fixture(id: u64, program: Program) -> (tempfile::TempDir, HostController, Database) { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let initial = program.clone(); + let controller = HostController::new( + data.clone(), + crate::db::Config { + storage: crate::db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }), + Arc::new(crate::energy::NullEnergyMonitor), + Arc::new(LocalPersistenceProvider::new(data)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: if program.kind == ModuleKind::JS { + HostType::Js + } else { + HostType::Wasm + }, + initial_program: program.hash, + }; + (directory, controller, database) +} + +async fn until(mut condition: impl FnMut() -> bool) { + timeout(Duration::from_secs(5), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +fn client_auth() -> ConnectionAuthCtx { + SpacetimeIdentityClaims { + identity: Identity::ONE, + subject: "local-test".into(), + issuer: "local-test".into(), + audience: Box::new([]), + iat: std::time::SystemTime::now(), + exp: None, + extra: None, + } + .try_into() + .unwrap() +} + +async fn queued_disconnect(js: bool, cancel: bool, id: u64) { + let program = if js { + javascript() + } else { + crate::host::empty_module::program(1).unwrap() + }; + let (_directory, controller, database) = fixture(id, program); + let module = controller + .get_or_launch_module_host(database.clone(), id) + .await + .unwrap(); + let client = ClientActorId::for_test(Identity::ONE); + module + .call_identity_connected(client_auth(), client.connection_id) + .await + .unwrap(); + let db = module.relational_db().clone(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + assert!(tx.st_client_row(client.identity, client.connection_id).is_some()); + let call = tokio::spawn({ + let module = module.clone(); + async move { module.disconnect_client(client).await } + }); + until(|| module.operations.active() == 1).await; + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + if cancel { + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + } else { + drop(call); + } + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + let _ = db.rollback_mut_tx(tx); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + assert!(tx.st_client_row(client.identity, client.connection_id).is_none()); + let _ = db.rollback_mut_tx(tx); + assert!(module.clear_all_clients().await.is_err()); + let result = crate::sql::execute::run( + db.clone(), + "SELECT * FROM st_client".into(), + AuthCtx::new(db.owner_identity(), db.owner_identity()), + None, + None, + &mut Vec::new(), + ) + .await; + assert!(matches!(result, Err(DBError::DatabaseClosed))); + assert!(controller.get_module_host(id).await.is_err()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_wasm_queued_disconnect_survives_cancellation() { + queued_disconnect(false, true, 0xd100).await; + queued_disconnect(false, false, 0xd101).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_js_queued_disconnect_survives_cancellation() { + queued_disconnect(true, true, 0xd102).await; + queued_disconnect(true, false, 0xd103).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_wasm_procedure_owns_external_wait_and_late_commit() { + let id = 0xd104; + let (_directory, controller, database) = fixture(id, crate::host::empty_module::program(1).unwrap()); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let db = module.relational_db().clone(); + let started = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + let call = tokio::spawn({ + let module = module.clone(); + let db = db.clone(); + let started = started.clone(); + let release = release.clone(); + async move { + module + .call_pooled( + "external-wait-test", + (), + async move |_, _| { + started.add_permits(1); + release.acquire().await.unwrap().forget(); + db.with_auto_commit(Workload::Internal, |tx| { + crate::db::environment::set(&db, tx, "AFTER_IO", "committed").map_err(anyhow::Error::from) + }) + .unwrap(); + }, + async |_, _| unreachable!(), + ) + .await + .unwrap(); + } + }); + timeout(Duration::from_secs(5), started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + release.add_permits(1); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let tx = db.begin_tx(Workload::Internal); + assert_eq!( + crate::db::environment::get(&tx, "AFTER_IO").unwrap().as_deref(), + Some("committed") + ); + let _ = db.release_tx(tx); +} + +/// Opt in only for the existing loopback test feature. Production IP filtering +/// stays enabled. Run with proxy variables cleared, as asserted below. +#[cfg(feature = "allow_loopback_http_for_tests")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_js_procedure_owns_actual_http_until_late_commit() { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for variable in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + assert!( + std::env::var_os(variable).is_none(), + "clear {variable} for this disposable loopback test" + ); + } + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let address = listener.local_addr().unwrap(); + assert!(address.ip().is_loopback()); + let started = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + let server = tokio::spawn({ + let started = started.clone(); + let release = release.clone(); + async move { + let (mut socket, peer) = listener.accept().await.unwrap(); + assert!(peer.ip().is_loopback()); + let mut request = [0u8; 4096]; + assert!(socket.read(&mut request).await.unwrap() > 0); + started.add_permits(1); + release.acquire().await.unwrap().forget(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + } + }); + let request = bsatn::to_vec(&spacetimedb_lib::http::Request { + method: spacetimedb_lib::http::Method::Get, + headers: std::iter::empty().collect(), + timeout: None, + uri: format!("http://{address}/owned-test"), + version: spacetimedb_lib::http::Version::Http11, + }) + .unwrap(); + let mut schema = RawModuleDefV10Builder::new(); + schema + .build_table_with_new_type("rows", [("value", AlgebraicType::U64)], true) + .finish(); + schema.add_procedure("task", spacetimedb_sats::ProductType::unit(), AlgebraicType::U64); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + let program = Program::from_bytes(ModuleKind::JS, format!(r#" + import {{register_hooks,table_id_from_name,datastore_insert_bsatn}} from "spacetime:sys@1.0"; + import {{register_hooks as procedures,procedure_http_request,procedure_start_mut_tx,procedure_commit_mut_tx}} from "spacetime:sys@1.2"; + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), __call_reducer__: () => ({{tag:"ok"}})}}); + procedures({{__call_procedure__: () => {{ + procedure_http_request(new Uint8Array({request:?}), ""); + procedure_start_mut_tx(); + datastore_insert_bsatn(table_id_from_name("rows"), new Uint8Array([42,0,0,0,0,0,0,0])); + procedure_commit_mut_tx(); + return new Uint8Array(8); + }} }}); + "#).into_bytes()); + let id = 0xd105; + let (_directory, controller, database) = fixture(id, program); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let db = module.relational_db().clone(); + let call = tokio::spawn({ + let module = module.clone(); + async move { + module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + } + }); + timeout(Duration::from_secs(10), started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(10)).await } + }); + until(|| module.operations.is_closed()).await; + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + release.add_permits(1); + timeout(Duration::from_secs(10), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + server.await.unwrap(); + let tx = db.begin_tx(Workload::Internal); + let table = db.table_id_from_name(&tx, "rows").unwrap().unwrap(); + assert_eq!(tx.table_row_count(table), Some(1)); + let _ = db.release_tx(tx); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_cancelled_js_procedure_startup_retains_physical_slot() { + use futures::StreamExt; + let mut schema = RawModuleDefV10Builder::new(); + schema.add_procedure("task", spacetimedb_sats::ProductType::unit(), AlgebraicType::U64); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + let program = Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{register_hooks,console_log}} from "spacetime:sys@1.0"; + import {{register_hooks as procedures}} from "spacetime:sys@1.2"; + function startup() {{ console_log(2, "physical-startup-entered"); }} + startup(); + const until = Date.now() + 2000; + while (Date.now() < until) {{}} + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), __call_reducer__: () => ({{tag:"ok"}})}}); + procedures({{__call_procedure__: () => new Uint8Array(8)}}); + "# + ) + .into_bytes(), + ); + let id = 0xd106; + let (_directory, controller, database) = fixture(id, program); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let ModuleHostInner::Js(host) = &*module.inner else { + unreachable!() + }; + let slots = host.procedure_instances.instance_slots.as_ref().unwrap(); + let maximum = slots.available_permits(); + let mut logs = module.database_logger().tail(Some(0), true).await.unwrap(); + let call = tokio::spawn({ + let module = module.clone(); + async move { + module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + } + }); + timeout(Duration::from_secs(5), async { + loop { + let log = logs.next().await.unwrap().unwrap(); + if String::from_utf8_lossy(&log).contains("physical-startup-entered") { + break; + } + } + }) + .await + .unwrap(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + assert_eq!(module.operations.active(), 1); + assert_eq!(slots.available_permits(), maximum - 1); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + assert!(!closing.is_finished()); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(slots.available_permits(), maximum); +} diff --git a/crates/core/src/host/module_host/operations.rs b/crates/core/src/host/module_host/operations.rs new file mode 100644 index 00000000000..4906f304a5c --- /dev/null +++ b/crates/core/src/host/module_host/operations.rs @@ -0,0 +1,126 @@ +//! Physical operation ownership, independent of an HTTP/WebSocket waiter's lifetime. +use super::NoSuchModule; +use parking_lot::Mutex; +use std::sync::Arc; +use tokio::sync::{Notify, OwnedSemaphorePermit}; + +#[derive(Default)] +pub(super) struct ModuleOperations { + state: Mutex, + drained: Notify, +} + +#[derive(Default)] +struct State { + closed: bool, + active: usize, +} + +impl ModuleOperations { + pub(super) fn begin(self: &Arc) -> Result { + let mut state = self.state.lock(); + if state.closed { + return Err(NoSuchModule); + } + state.active = state.active.checked_add(1).ok_or(NoSuchModule)?; + Ok(OperationLease { + _token: Arc::new(ActiveOperation(self.clone())), + pool_slot: None, + }) + } + + #[cfg(test)] + pub(super) fn active(&self) -> usize { + self.state.lock().active + } + + #[cfg(test)] + pub(super) fn is_closed(&self) -> bool { + self.state.lock().closed + } + + pub(super) fn close(&self) { + self.state.lock().closed = true; + } + + pub(super) async fn drained(&self) { + loop { + let notified = self.drained.notified(); + tokio::pin!(notified); + // Register before reading the counter, including on a multi-threaded runtime. + notified.as_mut().enable(); + if self.state.lock().active == 0 { + return; + } + notified.await; + } + } +} + +/// Clones describe one admitted operation, not additional admissions. A clone +/// moves into the physical job/request before it is enqueued. Cancellation of +/// the caller therefore cannot release admission or a pooled instance's slot. +#[derive(Clone)] +pub(in crate::host) struct OperationLease { + _token: Arc, + pool_slot: Option>, +} + +impl OperationLease { + pub(super) fn with_pool_slot(mut self, slot: Option>) -> Self { + self.pool_slot = slot; + self + } +} + +struct ActiveOperation(Arc); + +impl Drop for ActiveOperation { + fn drop(&mut self) { + let mut state = self.0.state.lock(); + state.active -= 1; + if state.active == 0 { + self.0.drained.notify_waiters(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn close_counts_previously_admitted_work_even_before_enqueue() { + let operations = Arc::new(ModuleOperations::default()); + let operation = operations.begin().unwrap(); + operations.close(); + assert!(operations.begin().is_err()); + let waiter = tokio::spawn({ + let operations = operations.clone(); + async move { operations.drained().await } + }); + tokio::task::yield_now().await; + assert!(!waiter.is_finished()); + drop(operation); + waiter.await.unwrap(); + operations.drained().await; + } + + #[tokio::test] + async fn cancelled_waiter_does_not_release_physical_job_or_pool_slot() { + let operations = Arc::new(ModuleOperations::default()); + let slots = Arc::new(tokio::sync::Semaphore::new(1)); + let caller = operations + .begin() + .unwrap() + .with_pool_slot(Some(Arc::new(slots.clone().acquire_owned().await.unwrap()))); + let physical_job = caller.clone(); + drop(caller); + operations.close(); + assert_eq!(slots.available_permits(), 0); + assert_eq!(operations.state.lock().active, 1); + drop(physical_job); + operations.drained().await; + assert_eq!(slots.available_permits(), 1); + } +} diff --git a/crates/core/src/host/scheduler.rs b/crates/core/src/host/scheduler.rs index f001a19ab53..9e60d770bb2 100644 --- a/crates/core/src/host/scheduler.rs +++ b/crates/core/src/host/scheduler.rs @@ -318,6 +318,8 @@ impl ScheduledFunctionParams { pub(crate) enum CallScheduledFunctionError { #[error(transparent)] NoSuchModule(#[from] NoSuchModule), + #[error("module instance startup failed: {0}")] + InstanceStartup(anyhow::Error), } #[cfg(target_pointer_width = "64")] @@ -395,6 +397,17 @@ impl SchedulerActor { // If the module already exited, leave the `ScheduledFunction` in // the database for when the module restarts. Err(CallScheduledFunctionError::NoSuchModule(_)) => {} + Err(CallScheduledFunctionError::InstanceStartup(error)) => { + // No transaction or procedure ran. Keep the schedule available + // and retry with a delay, including under shared deadline pressure. + log::warn!("scheduled procedure instance startup failed: {error:#}"); + if startup_retry_is_needed(module_host.info().relational_db(), &item) { + let key = self.queue.insert(item, Duration::from_secs(1)); + if let Some(id) = id { + self.key_map.insert(id, key); + } + } + } Ok(CallScheduledFunctionResult { reschedule: None }) => { // nothing to do } @@ -418,6 +431,26 @@ impl SchedulerActor { } } +/// A cancelled table-backed schedule must not retry forever when every new +/// procedure instance fails before reaching the normal schedule-row lookup. +fn startup_retry_is_needed(db: &RelationalDB, item: &QueueItem) -> bool { + let QueueItem::Id { id, .. } = item else { + return true; + }; + let exists = db.with_read_only(Workload::Internal, |tx| { + db.iter_by_col_eq(tx, id.table_id, id.id_column, &id.schedule_id.into()) + .map(|mut rows| rows.next().is_some()) + }); + match exists { + Ok(exists) => exists, + Err(error) => { + // A failed read is not proof that the user cancelled the schedule. + log::warn!("could not check scheduled procedure startup retry: {error:#}"); + true + } + } +} + #[derive(Debug)] pub(crate) struct CallScheduledFunctionResult { reschedule: Option, @@ -827,3 +860,61 @@ fn read_schedule_at(row: &RowRef<'_>, at_column: ColId) -> anyhow::Result QueueItem { + QueueItem::Id { + id: ScheduledFunctionId { + table_id, + schedule_id, + id_column: 0.into(), + at_column: 0.into(), + }, + function_name: "task".into(), + at: Timestamp::now(), + } + } + + #[test] + fn execution_deadline_startup_retry_stops_after_schedule_deletion() -> anyhow::Result<()> { + let db = TestDB::in_memory()?; + let table = db.create_table_for_test("pending", &[("id", AlgebraicType::U64)], &[0.into()])?; + with_auto_commit(&db, |tx| { + insert(&db, tx, table, &(7u64,))?; + insert(&db, tx, table, &(8u64,))?; + Ok::<_, anyhow::Error>(()) + })?; + + let cancelled = item(table, 7); + assert!(startup_retry_is_needed(&db, &cancelled)); + with_auto_commit(&db, |tx| { + assert_eq!(db.delete_by_rel(tx, table, [product!(7u64)]), 1); + Ok::<_, anyhow::Error>(()) + })?; + + // An unrelated remaining schedule cannot keep the cancelled ID alive. + assert!(!startup_retry_is_needed(&db, &cancelled)); + assert!(startup_retry_is_needed(&db, &item(table, 8))); + Ok(()) + } + + #[test] + fn execution_deadline_startup_retry_preserves_read_errors_and_volatile_calls() -> anyhow::Result<()> { + let db = TestDB::in_memory()?; + // The point read fails rather than positively observing a missing row. + assert!(startup_retry_is_needed(&db, &item(u32::MAX.into(), 7))); + assert!(startup_retry_is_needed( + &db, + &QueueItem::VolatileNonatomicImmediate { + function_name: "task".into(), + args: FunctionArgs::Nullary, + }, + )); + Ok(()) + } +} diff --git a/crates/core/src/host/v8/execution_deadline.rs b/crates/core/src/host/v8/execution_deadline.rs new file mode 100644 index 00000000000..ac6b1f3d1c9 --- /dev/null +++ b/crates/core/src/host/v8/execution_deadline.rs @@ -0,0 +1,272 @@ +//! Direct cross-thread termination, without executing a V8 interrupt callback. +//! +//! Pinned v8 145's IsolateHandle is Send+Sync and protects termination with its +//! annex mutex. No scope, raw isolate pointer, Rc or Cell leaves the isolate +//! thread. One persistent timer serves bounded active registrations. Finishing +//! or dropping a guard waits out any in-flight termination before the isolate +//! may reset termination or begin another invocation. + +use std::{ + collections::BTreeMap, + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Condvar, Mutex, OnceLock, + }, + time::{Duration, Instant}, +}; + +#[derive(Debug, thiserror::Error)] +#[error("JavaScript execution exceeded its wall-clock limit")] +pub(super) struct ExecutionTimedOut; + +const MAX_ACTIVE_DEADLINES: usize = 16_384; +static ACTIVE_DEADLINES: AtomicUsize = AtomicUsize::new(0); +type Key = (Instant, u64); +static SERVICE: OnceLock, String>> = OnceLock::new(); + +struct Capacity; +impl Capacity { + fn acquire() -> io::Result { + ACTIVE_DEADLINES + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { + (count < MAX_ACTIVE_DEADLINES).then_some(count + 1) + }) + .map_err(|_| io::Error::other("JavaScript execution deadline capacity exhausted"))?; + Ok(Self) + } +} +impl Drop for Capacity { + fn drop(&mut self) { + ACTIVE_DEADLINES.fetch_sub(1, Ordering::Relaxed); + } +} + +struct Registration { + expires: Instant, + state: Mutex, +} +struct State { + finished: bool, + expired: bool, + handle: v8::IsolateHandle, +} +#[derive(Default)] +struct Queue { + sequence: u64, + entries: BTreeMap>, +} +struct DeadlineService { + queue: Mutex, + wake: Condvar, +} + +impl DeadlineService { + fn start() -> io::Result> { + let service = Arc::new(Self { + queue: Mutex::new(Queue::default()), + wake: Condvar::new(), + }); + std::thread::Builder::new().name("v8-deadlines".into()).spawn({ + let service = service.clone(); + move || service.run() + })?; + Ok(service) + } + + fn run(&self) { + loop { + let registration = { + let mut queue = self.queue.lock().unwrap_or_else(|error| error.into_inner()); + loop { + let Some((&(expires, _), _)) = queue.entries.first_key_value() else { + queue = self.wake.wait(queue).unwrap_or_else(|error| error.into_inner()); + continue; + }; + let remaining = expires.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break queue.entries.pop_first().unwrap().1; + } + queue = self + .wake + .wait_timeout(queue, remaining) + .unwrap_or_else(|error| error.into_inner()) + .0; + } + }; + // Never hold the queue lock while acquiring a registration lock. + // Cancellation may hold this lock while removing its queued entry. + let mut state = registration.state.lock().unwrap_or_else(|error| error.into_inner()); + if !state.finished { + state.expired = true; + // finish() must acquire this same lock before resetting V8. + state.handle.terminate_execution(); + } + } + } +} + +pub(super) struct ExecutionDeadline { + service: Arc, + registration: Arc, + key: Key, + finished: bool, + // Includes expired calls still returning from a native syscall. The timer + // popping its queue entry never releases this active-call reservation. + _capacity: Capacity, +} + +impl ExecutionDeadline { + pub fn start(handle: v8::IsolateHandle, timeout: Duration) -> io::Result { + let capacity = Capacity::acquire()?; + let service = SERVICE + .get_or_init(|| DeadlineService::start().map_err(|error| error.to_string())) + .as_ref() + .map_err(|error| io::Error::other(error.clone()))? + .clone(); + let expires = Instant::now() + timeout; + let registration = Arc::new(Registration { + expires, + state: Mutex::new(State { + finished: false, + expired: false, + handle, + }), + }); + let mut queue = service.queue.lock().unwrap_or_else(|error| error.into_inner()); + queue.sequence = queue + .sequence + .checked_add(1) + .ok_or_else(|| io::Error::other("JavaScript execution deadline sequence exhausted"))?; + let key = (expires, queue.sequence); + queue.entries.insert(key, registration.clone()); + drop(queue); + service.wake.notify_one(); + Ok(Self { + service, + registration, + key, + finished: false, + _capacity: capacity, + }) + } + + pub fn finish(mut self) -> bool { + let expired = self.cancel(); + self.finished = true; + expired + } + + fn cancel(&self) -> bool { + let mut state = self + .registration + .state + .lock() + .unwrap_or_else(|error| error.into_inner()); + if !state.finished { + // A timer delayed by OS scheduling must still make a late return + // roll back, even if it has not issued its termination request yet. + state.expired |= Instant::now() >= self.registration.expires; + state.finished = true; + } + let expired = state.expired; + // The timer releases the queue lock before taking this lock, so this + // order cannot invert its locks. Remove completed calls immediately, + // rather than accumulating their registrations for 120 seconds. + self.service + .queue + .lock() + .unwrap_or_else(|error| error.into_inner()) + .entries + .remove(&self.key); + drop(state); + self.service.wake.notify_one(); + expired + } +} + +impl Drop for ExecutionDeadline { + fn drop(&mut self) { + if !self.finished { + self.cancel(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::host::v8::to_value::test::with_scope; + + fn run(scope: &mut v8::PinScope<'_, '_>, source: &str) -> bool { + let source = v8::String::new(scope, source).unwrap(); + v8::Script::compile(scope, source, None).unwrap().run(scope).is_some() + } + + #[test] + fn deadline_terminates_and_isolate_can_run_again() { + with_scope(|scope| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_millis(40)).unwrap(); + assert!(!run(scope, "for (;;) {}")); + assert!(deadline.finish()); + scope.cancel_terminate_execution(); + assert!(run(scope, "1 + 1")); + }); + } + + #[test] + fn finished_and_dropped_registrations_cannot_terminate_later_execution() { + with_scope(|scope| { + for drop_guard in [false, true] { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_millis(40)).unwrap(); + let service = deadline.service.clone(); + let key = deadline.key; + if drop_guard { + drop(deadline); + } else { + assert!(!deadline.finish()); + } + assert!(!service.queue.lock().unwrap().entries.contains_key(&key)); + assert!(run( + scope, + "{ const end = Date.now() + 80; while (Date.now() < end) {} }" + )); + } + for _ in 0..1_000 { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_secs(1)).unwrap(); + let service = deadline.service.clone(); + let key = deadline.key; + assert!(run(scope, "1 + 1")); + assert!(!deadline.finish()); + assert!(!service.queue.lock().unwrap().entries.contains_key(&key)); + } + }); + } + + #[test] + fn late_return_is_expired_even_before_timer_observes_it() { + with_scope(|scope| { + // A service without a timer deterministically models OS delay. + let expires = Instant::now(); + let deadline = ExecutionDeadline { + service: Arc::new(DeadlineService { + queue: Mutex::new(Queue::default()), + wake: Condvar::new(), + }), + registration: Arc::new(Registration { + expires, + state: Mutex::new(State { + finished: false, + expired: false, + handle: scope.thread_safe_handle(), + }), + }), + key: (expires, 1), + finished: false, + _capacity: Capacity::acquire().unwrap(), + }; + assert!(deadline.finish()); + assert!(run(scope, "1 + 1")); + }); + } +} diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 433fe5e2af8..7f8238c81f2 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -58,6 +58,7 @@ use self::error::{ catch_exception, exception_already_thrown, log_traceback, ErrorOrException, ExcResult, ExceptionThrown, PinTryCatch, Throwable, }; +use self::execution_deadline::{ExecutionDeadline, ExecutionTimedOut}; use self::ser::serialize_to_js; use self::string::{str_from_ident, IntoJsString}; use self::syscall::{ @@ -74,6 +75,7 @@ use crate::client::{ClientActorId, MeteredUnboundedReceiver, MeteredUnboundedSen use crate::config::{V8Config, V8HeapPolicyConfig}; use crate::host::host_controller::CallProcedureReturn; use crate::host::instance_env::{ChunkPool, InstanceEnv, TxSlot}; +use crate::host::module_host::OperationLease; use crate::host::module_host::{ call_identity_connected, init_database, ClientConnectedError, HttpHandlerCallError, OneOffQueryRequest, SqlCommand, SqlCommandResult, ViewCommand, ViewCommandMetric, ViewCommandResult, @@ -86,6 +88,7 @@ use crate::host::wasm_common::module_host_actor::{ ReducerExecuteResult, ReducerOp, ViewExecuteResult, ViewOp, WasmInstance, }; use crate::host::wasm_common::{RowIters, TimingSpanSet}; +use crate::host::ProcedureCallError; use crate::host::{ModuleHost, ReducerCallError, ReducerCallResult, Scheduler}; use crate::messages::control_db::HostType; use crate::module_host_context::ModuleCreationContext; @@ -110,8 +113,8 @@ use std::cell::Cell; use std::num::NonZeroUsize; use std::os::raw::c_void; use std::panic::{self, AssertUnwindSafe}; -use std::sync::{Arc, LazyLock}; -use std::time::Instant; +use std::sync::{Arc, LazyLock, OnceLock}; +use std::time::{Duration, Instant}; use tokio::sync::{mpsc, oneshot}; use v8::script_compiler::{compile_module, Source}; use v8::{ @@ -123,6 +126,7 @@ mod budget; mod builtins; mod de; mod error; +mod execution_deadline; mod from_value; mod ser; mod string; @@ -154,6 +158,7 @@ impl V8Runtime { program_bytes: &[u8], core: AllocatedJobCore, ) -> anyhow::Result { + self.config.validate_execution_timeout()?; V8_RUNTIME_GLOBAL .make_actor(mcc, program_bytes, core, self.config) .await @@ -256,6 +261,7 @@ impl V8RuntimeInner { load_balance_guard.clone(), core_pinner.clone(), heap_policy, + config.execution_timeout, metrics.clone(), ) .await?; @@ -266,6 +272,7 @@ impl V8RuntimeInner { core_pinner, procedure_instance_pool_size: config.procedure_instance_pool_size, heap_policy: config.heap_policy, + execution_timeout: config.execution_timeout, metrics, }; @@ -281,6 +288,7 @@ pub struct JsModule { core_pinner: CorePinner, procedure_instance_pool_size: NonZeroUsize, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, } @@ -305,7 +313,10 @@ impl JsModule { self.procedure_instance_pool_size } - async fn create_procedure_instance(&self) -> JsProcedureInstance { + async fn create_procedure_instance( + &self, + operation: Option, + ) -> anyhow::Result { let program = self.program.clone(); let common = self.common.clone(); let load_balance_guard = self.load_balance_guard.clone(); @@ -320,15 +331,23 @@ impl JsModule { load_balance_guard, core_pinner, heap_policy, + self.execution_timeout, metrics, + operation, ) - .await - .expect("`spawn_procedure_instance_worker` should succeed when passed `ModuleCommon`"); - instance + .await?; + Ok(instance) + } + + pub(in crate::host) async fn create_instance_for_operation( + &self, + operation: Option, + ) -> anyhow::Result { + self.create_procedure_instance(operation).await } - pub async fn create_instance(&self) -> JsProcedureInstance { - self.create_procedure_instance().await + pub async fn create_instance(&self) -> anyhow::Result { + self.create_procedure_instance(None).await } } @@ -461,7 +480,8 @@ impl JsInstanceEnv { /// and friends. #[derive(Clone)] pub struct JsMainInstance { - tx: MeteredUnboundedSender, + tx: MeteredUnboundedSender>, + operation: Option, } /// A procedure instance for a [`JsModule`]. @@ -469,16 +489,48 @@ pub struct JsMainInstance { /// Procedure instances are checked out exclusively from the procedure pool and /// only execute procedure-style requests. pub struct JsProcedureInstance { - tx: mpsc::Sender, + tx: mpsc::Sender>, + operation: Option, + startup_failure: ProcedureStartupStatus, +} + +// Set only when a replacement generation fails before receiving another +// request. Publishing before closing the receiver proves queued calls did not +// execute. Unexplained worker exits and actual Rust panics remain fatal. +type ProcedureStartupStatus = Arc>; + +#[derive(Clone, Debug, thiserror::Error)] +#[error("procedure isolate startup failed: {0}")] +pub(in crate::host) struct JsProcedureStartupError(Arc); + +struct PhysicalRequest { + request: R, + operation: Option, } impl JsMainInstance { + pub(in crate::host) fn with_operation(mut self, operation: OperationLease) -> Self { + self.operation = Some(operation); + self + } + async fn request(&self, request: R) -> R::Response { - send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| request.into_worker_request(reply_tx)).await + send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| PhysicalRequest { + request: request.into_worker_request(reply_tx), + operation: self.operation.clone(), + }) + .await } async fn send_detached_request(&self, ctx: &'static str, request: JsMainWorkerRequest) { - if self.tx.send(request).is_err() { + if self + .tx + .send(PhysicalRequest { + request, + operation: self.operation.clone(), + }) + .is_err() + { panic!("JS worker exited before accepting `{ctx}`"); } } @@ -488,11 +540,13 @@ impl JsMainInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { self.request(UpdateDatabaseRequest { program, old_module_info, policy, + deployment, }) .await } @@ -548,8 +602,12 @@ impl JsMainInstance { self.request(DisconnectClientRequest { client_id }).await } - pub async fn init_database(&self, program: Program) -> anyhow::Result> { - self.request(InitDatabaseRequest { program }).await + pub async fn init_database( + &self, + program: Program, + deployment: Option, + ) -> anyhow::Result> { + self.request(InitDatabaseRequest { program, deployment }).await } pub async fn call_view(&self, cmd: ViewCommand) -> ViewCommandResult { @@ -633,6 +691,7 @@ js_main_request! { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, } => "update_database", anyhow::Result, UpdateDatabase } @@ -675,6 +734,7 @@ js_main_request! { js_main_request! { InitDatabaseRequest { program: Program, + deployment: Option, } => "init_database", anyhow::Result>, InitDatabase } @@ -691,16 +751,24 @@ js_main_request! { } impl JsProcedureInstance { + pub(in crate::host) fn set_operation(&mut self, operation: Option) { + self.operation = operation; + } + pub(in crate::host) fn is_closed(&self) -> bool { - self.tx.is_closed() + self.startup_failure.get().is_some() || self.tx.is_closed() } async fn send_request( &self, ctx: &'static str, request: impl FnOnce(JsReplyTx) -> JsProcedureWorkerRequest, - ) -> T { - send_js_request(ctx, &self.tx, request).await + ) -> Result { + send_js_request(ctx, &self.tx, &self.startup_failure, |reply_tx| PhysicalRequest { + request: request(reply_tx), + operation: self.operation.clone(), + }) + .await } pub async fn call_procedure(&self, params: CallProcedureParams) -> CallProcedureReturn { @@ -709,6 +777,10 @@ impl JsProcedureInstance { params, }) .await + .unwrap_or_else(|error| CallProcedureReturn { + result: Err(ProcedureCallError::InternalError(error.to_string())), + tx_offset: None, + }) } pub async fn call_http_handler( @@ -719,25 +791,38 @@ impl JsProcedureInstance { JsProcedureWorkerRequest::CallHttpHandler { reply_tx, params } }) .await + .map_err(|error| HttpHandlerCallError::InternalError(error.to_string()))? } - pub(in crate::host) async fn enqueue_procedure(&self, params: CallProcedureParams) -> JsProcedureCall { + pub(in crate::host) async fn enqueue_procedure( + &self, + params: CallProcedureParams, + ) -> Result { let (reply_tx, reply_rx) = oneshot::channel(); if self .tx - .send(JsProcedureWorkerRequest::CallProcedure { reply_tx, params }) + .send(PhysicalRequest { + request: JsProcedureWorkerRequest::CallProcedure { reply_tx, params }, + operation: self.operation.clone(), + }) .await .is_err() { + if let Some(error) = self.startup_failure.get() { + return Err(error.clone()); + } panic!("JS worker exited before accepting `call_procedure`"); } - JsProcedureCall { reply_rx } + Ok(JsProcedureCall { + reply_rx, + startup_failure: self.startup_failure.clone(), + }) } pub(in crate::host) async fn call_scheduled_procedure( &self, params: ScheduledFunctionParams, - ) -> CallScheduledFunctionResult { + ) -> Result { self.send_request("scheduled_procedure", |reply_tx| { JsProcedureWorkerRequest::ScheduledProcedure { reply_tx, params } }) @@ -748,26 +833,33 @@ impl JsProcedureInstance { async fn send_js_request( ctx: &'static str, tx: &mpsc::Sender, + startup_failure: &ProcedureStartupStatus, request: impl FnOnce(JsReplyTx) -> Req, -) -> T +) -> Result where Req: Send + 'static, { let (reply_tx, reply_rx) = oneshot::channel(); if tx.send(request(reply_tx)).await.is_err() { + if let Some(error) = startup_failure.get() { + return Err(error.clone()); + } panic!("JS worker exited before accepting `{ctx}`"); } match reply_rx.await { - Ok(Ok(value)) => value, + Ok(Ok(value)) => Ok(value), Ok(Err(panic)) => panic::resume_unwind(panic), - Err(_) => panic!("JS worker exited before replying to `{ctx}`"), + Err(_) => match startup_failure.get() { + Some(error) => Err(error.clone()), + None => panic!("JS worker exited before replying to `{ctx}`"), + }, } } -async fn send_js_unbounded_request( +async fn send_js_unbounded_request( ctx: &'static str, - tx: &MeteredUnboundedSender, - request: impl FnOnce(JsReplyTx) -> JsMainWorkerRequest, + tx: &MeteredUnboundedSender, + request: impl FnOnce(JsReplyTx) -> Req, ) -> T { let (reply_tx, reply_rx) = oneshot::channel(); if tx.send(request(reply_tx)).is_err() { @@ -787,11 +879,13 @@ pub(in crate::host) type JsFatalHook = Arc; pub(in crate::host) struct JsProcedureCall { reply_rx: oneshot::Receiver>, + startup_failure: ProcedureStartupStatus, } pub(in crate::host) enum JsProcedureCallCompletion { Completed(CallProcedureReturn), Panicked, + StartupFailed(JsProcedureStartupError), WorkerExited, } @@ -800,7 +894,10 @@ impl JsProcedureCall { match self.reply_rx.await { Ok(Ok(ret)) => JsProcedureCallCompletion::Completed(ret), Ok(Err(_panic)) => JsProcedureCallCompletion::Panicked, - Err(_) => JsProcedureCallCompletion::WorkerExited, + Err(_) => match self.startup_failure.get() { + Some(error) => JsProcedureCallCompletion::StartupFailed(error.clone()), + None => JsProcedureCallCompletion::WorkerExited, + }, } } } @@ -817,6 +914,7 @@ enum JsMainWorkerRequest { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, }, /// See [`JsMainInstance::call_reducer`]. CallReducer { @@ -877,6 +975,7 @@ enum JsMainWorkerRequest { InitDatabase { reply_tx: JsReplyTx>>, program: Program, + deployment: Option, }, } @@ -899,7 +998,7 @@ enum JsProcedureWorkerRequest { }, } -static_assert_size!(CallReducerParams, 192); +static_assert_size!(CallReducerParams, 208); fn send_worker_reply(ctx: &str, reply_tx: JsReplyTx, value: T) { if reply_tx.send(Ok(value)).is_err() { @@ -1186,7 +1285,9 @@ fn startup_instance_worker<'scope>( scope: &mut PinScope<'scope, '_>, program: Arc, module_or_mcc: Either, + execution_timeout: Duration, ) -> anyhow::Result<(HookFunctions<'scope>, ModuleCommon)> { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), execution_timeout)?; let hook_functions = catch_exception(scope, |scope| { // Start-up the user's module. let exports_obj = eval_user_module(scope, &program)?; @@ -1195,13 +1296,19 @@ fn startup_instance_worker<'scope>( let hooks = get_hooks(scope, exports_obj)?.ok_or_else(|| anyhow::anyhow!("must export schema as default export"))?; Ok(hooks) - })?; + }); + let expired = deadline.finish(); + scope.cancel_terminate_execution(); + if expired { + return Err(ExecutionTimedOut.into()); + } + let hook_functions = hook_functions?; // If we don't have a module, make one. let module_common = match module_or_mcc { Either::Left(module_common) => module_common, Either::Right(mcc) => { - let def = extract_description(scope, &hook_functions, &mcc.replica_ctx)?; + let def = extract_description(scope, &hook_functions, &mcc.replica_ctx, execution_timeout)?; // Validate and create a common module from the raw definition. build_common_module_from_raw(mcc, def)? @@ -1264,6 +1371,7 @@ async fn spawn_main_instance_worker( load_balance_guard: Arc, core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, ) -> anyhow::Result<(ModuleCommon, JsMainInstance)> { spawn_instance_worker::( @@ -1272,18 +1380,23 @@ async fn spawn_main_instance_worker( load_balance_guard, core_pinner, heap_policy, + execution_timeout, metrics, + None, ) .await } +#[allow(clippy::too_many_arguments)] async fn spawn_procedure_instance_worker( program: Arc, module_or_mcc: Either, load_balance_guard: Arc, core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, + operation: Option, ) -> anyhow::Result<(ModuleCommon, JsProcedureInstance)> { spawn_instance_worker::( program, @@ -1291,7 +1404,9 @@ async fn spawn_procedure_instance_worker( load_balance_guard, core_pinner, heap_policy, + execution_timeout, metrics, + operation, ) .await } @@ -1310,9 +1425,9 @@ trait JsWorkerSpec { fn channel(database_identity: &Identity) -> (Self::Sender, Self::Receiver); - fn make_instance(tx: Self::Sender) -> Self::Instance; + fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance; - fn blocking_recv(rx: &mut Self::Receiver) -> Option; + fn blocking_recv(rx: &mut Self::Receiver) -> Option>; fn handle_request( request: Self::Request, @@ -1326,8 +1441,8 @@ trait JsWorkerSpec { impl JsWorkerSpec for MainJsWorker { type Request = JsMainWorkerRequest; type Instance = JsMainInstance; - type Sender = MeteredUnboundedSender; - type Receiver = MeteredUnboundedReceiver; + type Sender = MeteredUnboundedSender>; + type Receiver = MeteredUnboundedReceiver>; const KIND: JsWorkerKind = JsWorkerKind::Main; @@ -1342,11 +1457,11 @@ impl JsWorkerSpec for MainJsWorker { ) } - fn make_instance(tx: Self::Sender) -> Self::Instance { - JsMainInstance { tx } + fn make_instance(tx: Self::Sender, _startup_failure: ProcedureStartupStatus) -> Self::Instance { + JsMainInstance { tx, operation: None } } - fn blocking_recv(rx: &mut Self::Receiver) -> Option { + fn blocking_recv(rx: &mut Self::Receiver) -> Option> { rx.blocking_recv() } @@ -1364,8 +1479,8 @@ impl JsWorkerSpec for MainJsWorker { impl JsWorkerSpec for ProcedureJsWorker { type Request = JsProcedureWorkerRequest; type Instance = JsProcedureInstance; - type Sender = mpsc::Sender; - type Receiver = mpsc::Receiver; + type Sender = mpsc::Sender>; + type Receiver = mpsc::Receiver>; const KIND: JsWorkerKind = JsWorkerKind::Procedure; @@ -1373,11 +1488,15 @@ impl JsWorkerSpec for ProcedureJsWorker { mpsc::channel(JS_PROCEDURE_INSTANCE_QUEUE_CAPACITY) } - fn make_instance(tx: Self::Sender) -> Self::Instance { - JsProcedureInstance { tx } + fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance { + JsProcedureInstance { + tx, + startup_failure, + operation: None, + } } - fn blocking_recv(rx: &mut Self::Receiver) -> Option { + fn blocking_recv(rx: &mut Self::Receiver) -> Option> { rx.blocking_recv() } @@ -1407,8 +1526,9 @@ fn handle_main_worker_request( program, old_module_info, policy, + deployment, } => handle_worker_request("update_database", reply_tx, || { - let res = instance_common.update_database(program, old_module_info, policy, inst); + let res = instance_common.update_database(program, old_module_info, policy, deployment, inst); (res, false) }), JsMainWorkerRequest::CallReducer { reply_tx, params } => { @@ -1496,14 +1616,22 @@ fn handle_main_worker_request( (res, trapped) }) } - JsMainWorkerRequest::InitDatabase { reply_tx, program } => { - handle_worker_request("init_database", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); - let (res, trapped): (Result, anyhow::Error>, bool) = - init_database(replica_ctx, &info.module_def, program, call_reducer); - (res, trapped) - }) - } + JsMainWorkerRequest::InitDatabase { + reply_tx, + program, + deployment, + } => handle_worker_request("init_database", reply_tx, || { + let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let (res, trapped): (Result, anyhow::Error>, bool) = init_database( + replica_ctx, + &info.module_def, + info.module_hash, + program, + deployment, + call_reducer, + ); + (res, trapped) + }), } } @@ -1569,21 +1697,24 @@ fn spawn_v8_worker_thread(worker_kind: JsWorkerKind, database_identity: Identity /// Spawns an instance worker for `program` and returns on success the /// corresponding instance handle that talks to the worker. /// -/// When [`ModuleCommon`] is passed, it's assumed that this program has already -/// been validated. In that case, `Ok(_)` should be returned. +/// When [`ModuleCommon`] is passed, the program has already been validated. +/// Starting another isolate can still fail, including its execution deadline. /// /// Otherwise, when [`ModuleCreationContext`] is passed, this is the first time /// both the module and instance are created. /// /// `load_balance_guard` and `core_pinner` should both be from the same /// [`AllocatedJobCore`], and are used to manage the core pinning of this thread. +#[allow(clippy::too_many_arguments)] async fn spawn_instance_worker( program: Arc, module_or_mcc: Either, load_balance_guard: Arc, mut core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, instance_metrics: InstanceManagerMetrics, + operation: Option, ) -> anyhow::Result<(ModuleCommon, W::Instance)> where W: JsWorkerSpec + 'static, @@ -1596,6 +1727,8 @@ where Either::Right(mcc) => mcc.replica_ctx.database_identity, }; let (request_tx, mut request_rx) = W::channel(&database_identity); + let startup_failure = ProcedureStartupStatus::default(); + let worker_startup_failure = startup_failure.clone(); let rt = tokio::runtime::Handle::current(); @@ -1608,6 +1741,7 @@ where let mut startup_result_tx = Some(result_tx); let mut module_common_for_recreate = None::; + let mut physical_operation = operation; 'worker: loop { let replacing_instance = module_common_for_recreate.is_some(); let generation_start_time = replacing_instance.then(Instant::now); @@ -1640,7 +1774,7 @@ where .expect("our builtin code shouldn't error"); // Setup the JS module, find call_reducer, and maybe build the module. - startup_instance_worker(scope, program.clone(), generation_module_or_mcc) + startup_instance_worker(scope, program.clone(), generation_module_or_mcc, execution_timeout) })); let (hooks, module_common) = match startup_result { @@ -1663,6 +1797,7 @@ where log::error!("startup result receiver disconnected"); } } else { + let _ = worker_startup_failure.set(JsProcedureStartupError(format!("{err:#}").into())); log::error!("failed to restart JS worker: {err:#}"); } return; @@ -1707,6 +1842,7 @@ where .v8_heap_limit_hit .with_label_values(&info.database_identity), initial_heap_limit: heap_policy.heap_limit_bytes, + execution_timeout, }; let _initial_heap_stats = sample_heap_stats(inst.scope, &mut heap_metrics); @@ -1714,9 +1850,11 @@ where // // The loop is terminated when the last worker instance handle is dropped. // This will cause channels, scopes, and the isolate to be cleaned up. + physical_operation.take(); let mut requests_since_heap_check = 0u64; let mut last_heap_check_at = Instant::now(); - while let Some(request) = W::blocking_recv(&mut request_rx) { + while let Some(PhysicalRequest { request, operation }) = W::blocking_recv(&mut request_rx) { + physical_operation = operation; core_pinner.pin_if_changed(); let mut outcome = @@ -1747,7 +1885,9 @@ where } match outcome { - WorkerRequestOutcome::Continue => {} + WorkerRequestOutcome::Continue => { + physical_operation.take(); + } WorkerRequestOutcome::RecreateInstance => { instance_metrics.track_instance_removed(); continue 'worker; @@ -1763,7 +1903,7 @@ where // Get the module, if any, and get any setup errors from the worker. let res: Result = result_rx.await.expect("should have a sender"); res.map(|opt_mc| { - let inst = W::make_instance(request_tx); + let inst = W::make_instance(request_tx, startup_failure); (opt_mc, inst) }) } @@ -1853,11 +1993,12 @@ struct V8Instance<'a, 'scope, 'isolate> { /// Metric for the number of times the v8 heap limit has been hit. heap_limit_hit_metric: &'a IntCounter, initial_heap_limit: usize, + execution_timeout: Duration, } impl WasmInstance for V8Instance<'_, '_, '_> { fn extract_descriptions(&mut self) -> Result { - extract_description(self.scope, self.hooks, self.replica_ctx) + extract_description(self.scope, self.hooks, self.replica_ctx, self.execution_timeout) } fn replica_ctx(&self) -> &Arc { @@ -1978,12 +2119,14 @@ where // are released when the reducer/view/procedure returns. v8::scope!(let scope, scope); - // TODO(v8): Start the budget timeout and long-running logger. let env = env_on_isolate_unwrap(scope); // Start the timer. // We'd like this tightly around `call`. env.start_funcall(op.name().clone(), op.timestamp(), op.call_type()); + env.instance_env.set_call_auth_flags(op.call_auth_flags()); + env.instance_env + .set_hosted_auth(op.hosted_auth().map(|proof| std::sync::Arc::new(proof.clone()))); // Wrap the call in `TryCatch`. // @@ -1992,7 +2135,12 @@ where // opened by the caller before entering `common_call`. v8::tc_scope!(let scope, scope); - let call_result = call(scope, inst.hooks, op).map_err(|mut e| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), inst.execution_timeout); + let mut call_result = match &deadline { + Ok(_) => call(scope, inst.hooks, op), + Err(error) => Err(anyhow::anyhow!("cannot start JavaScript execution deadline: {error}").into()), + } + .map_err(|mut e| { if let ErrorOrException::Exception(_) = e { // If we're terminating execution, don't try to check `instanceof`. if scope.can_continue() @@ -2012,10 +2160,7 @@ where // We can continue. ExecutionError::Recoverable(e.unwrap_or_else(Into::into)) } else if scope.has_terminated() { - // We can continue if we do `Isolate::cancel_terminate_execution`. - // Must be called *after* we check `has_terminated()`, or else it will - // cause it to return `false`. - scope.cancel_terminate_execution(); + // Reset only after synchronizing with this invocation's timer. let e = e.unwrap_or_else(|unknown| termination_error.unwrap_or_else(|| unknown.into())); ExecutionError::Recoverable(e) } else { @@ -2024,6 +2169,14 @@ where } }); + // The timer also covers exception inspection, which can execute user + // getters or Symbol.hasInstance. Synchronizing with it before resetting prevents a + // delayed timer from terminating the next invocation of this isolate. + if deadline.is_ok_and(ExecutionDeadline::finish) { + // Even a call that returned at the exact boundary must roll back. + call_result = Err(ExecutionError::Recoverable(ExecutionTimedOut.into())); + } + // Ensure there's no lingering termination request. termination_flag.clear(); scope.cancel_terminate_execution(); @@ -2064,14 +2217,22 @@ fn extract_description<'scope>( scope: &mut PinScope<'scope, '_>, hooks: &HookFunctions<'_>, replica_ctx: &ReplicaContext, + execution_timeout: Duration, ) -> Result { run_describer( |a, b, c| log_traceback(replica_ctx, a, b, c), || { - Ok(catch_exception(scope, |scope| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), execution_timeout)?; + let result = catch_exception(scope, |scope| { let def = call_describe_module(scope, hooks)?; Ok(def) - })?) + }); + let expired = deadline.finish(); + scope.cancel_terminate_execution(); + if expired { + return Err(ExecutionTimedOut.into()); + } + Ok(result?) }, ) } @@ -2113,6 +2274,8 @@ mod test { name: &ReducerName::for_test("foobar"), caller_identity: &Identity::ONE, caller_connection_id: &ConnectionId::ZERO, + call_auth_flags: 0, + hosted_auth: None, timestamp: Timestamp::from_micros_since_unix_epoch(24), args: &ArgsTuple::nullary(), }; diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index 3d93e0b2679..38c130fa84d 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -44,6 +44,8 @@ pub fn call_call_procedure( name: _, caller_identity: sender, caller_connection_id: connection_id, + call_auth_flags: _, + hosted_auth: _, timestamp, arg_bytes: procedure_args, } = op; @@ -449,13 +451,19 @@ pub fn console_log<'scope>( let mut buf = scratch_buf::<128>(); let msg = msg.to_rust_cow_lossy(scope, &mut buf); - let frame: Local<'_, v8::StackFrame> = v8::StackTrace::current_stack_trace(scope, 2) - .ok_or_else(exception_already_thrown)? - .get_frame(scope, 1) - .ok_or_else(exception_already_thrown)?; + let trace = v8::StackTrace::current_stack_trace(scope, 2).ok_or_else(exception_already_thrown)?; + // The normal bindings add a logging wrapper, but modules may call this + // syscall directly, including from top-level code. V8's GetFrame does not + // check the index in release builds, so never request an absent frame. + let frame = match trace.get_frame_count() { + 0 => None, + 1 => trace.get_frame(scope, 0), + _ => trace.get_frame(scope, 1), + }; + let line_number = frame.map(|frame| frame.get_line_number() as u32); let mut buf = scratch_buf::<32>(); let filename = frame - .get_script_name(scope) + .and_then(|frame| frame.get_script_name(scope)) .map(|s| s.to_rust_cow_lossy(scope, &mut buf)); let level = (level as u8).into(); @@ -469,7 +477,7 @@ pub fn console_log<'scope>( tracing::warn!( "{}:{} {msg}", filename.as_deref().unwrap_or("unknown"), - frame.get_line_number() + line_number.unwrap_or_default() ); })?; @@ -479,7 +487,7 @@ pub fn console_log<'scope>( ts: InstanceEnv::now_for_logging(), target: None, filename: filename.as_deref(), - line_number: Some(frame.get_line_number() as u32), + line_number, function, message: &msg, }; diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index a09e7cbba0c..bb942a47f2d 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -62,6 +62,8 @@ fn resolve_sys_module_inner<'scope>( (1, 3) => Ok(v1::sys_v1_3(scope)), (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), + (2, 2) => Ok(v2::sys_v2_2(scope)), + (2, 3) => Ok(v2::sys_v2_3(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v1.rs b/crates/core/src/host/v8/syscall/v1.rs index f3aea9f6c52..ab852eda401 100644 --- a/crates/core/src/host/v8/syscall/v1.rs +++ b/crates/core/src/host/v8/syscall/v1.rs @@ -495,6 +495,8 @@ pub(super) fn call_call_reducer( name: _, caller_identity: sender, caller_connection_id: conn_id, + call_auth_flags: _, + hosted_auth: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index f49d2260549..4f174855182 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -169,6 +169,37 @@ pub(super) fn sys_v2_1<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } +/// Invocation authentication is independent of transaction or connection state. +pub(super) fn sys_v2_2<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!( + scope, + "spacetime:sys@2.2", + (with_sys_result, AbiCall::GetCallAuthFlags, get_call_auth_flags), + ) +} + +pub(super) fn sys_v2_3<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!(scope, "spacetime:sys@2.3", (with_sys_result, AbiCall::EnvGet, env_get),) +} + +fn env_get<'s>( + scope: &mut PinScope<'s, '_>, + args: FunctionCallbackArguments<'s>, +) -> SysCallResult> { + let key: String = deserialize_js(scope, args.get(0))?; + match get_env(scope)?.instance_env.env_get(&key)? { + Some(value) => Ok(value + .into_string(scope) + .map_err(|_| RangeError("environment value could not be represented").throw(scope))? + .into()), + None => Ok(v8::null(scope).into()), + } +} + +fn get_call_auth_flags(scope: &mut PinScope<'_, '_>, _args: FunctionCallbackArguments<'_>) -> SysCallResult { + Ok(get_env(scope)?.instance_env.get_call_auth_flags()) +} + /// Registers a function in `module` /// where the function has `name` and does `body`. fn register_module_fun( @@ -449,6 +480,8 @@ pub(super) fn call_call_reducer<'scope>( name: _, caller_identity: sender, caller_connection_id: conn_id, + call_auth_flags: _, + hosted_auth: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 1e7fd18b5e1..26569d072af 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -362,6 +362,8 @@ pub fn err_to_errno(err: NodesError) -> Result<(NonZeroU16, Option), Nod NodesError::DecodeRow(_) => errno::BSATN_DECODE_ERROR, NodesError::DecodeValue(_) => errno::BSATN_DECODE_ERROR, NodesError::TableNotFound => errno::NO_SUCH_TABLE, + NodesError::InvalidEnvironmentKey => errno::HOST_CALL_FAILURE, + NodesError::EnvironmentSourceLimit => errno::NO_SPACE, NodesError::IndexNotFound => errno::NO_SUCH_INDEX, NodesError::IndexNotUnique => errno::INDEX_NOT_UNIQUE, NodesError::IndexRowNotFound => errno::NO_SUCH_ROW, @@ -442,6 +444,8 @@ macro_rules! abi_funcs { "spacetime_10.4"::datastore_delete_by_index_scan_point_bsatn, "spacetime_10.5"::datastore_clear, + "spacetime_10.6"::get_call_auth_flags, + "spacetime_10.7"::env_get, } $link_async! { diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 2fb5eab0492..0b1569177d4 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -1,7 +1,10 @@ use super::instrumentation::CallTimes; use super::*; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::check_hosted_admission; use crate::client::ClientActorId; use crate::database_logger; +use crate::db::deployment::{self, CommitAdmission, DeploymentCommit}; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; use crate::host::host_controller::CallProcedureReturn; @@ -333,7 +336,7 @@ pub enum InitializationError { #[error(transparent)] Validation(#[from] ValidationError), #[error(transparent)] - ModuleValidation(#[from] spacetimedb_schema::error::ValidationErrors), + ModuleValidation(#[from] Box), #[error("setup function returned an error: {0}")] Setup(Box), #[error("wasm trap while calling {func:?}")] @@ -475,9 +478,10 @@ impl WasmModuleInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { self.common - .update_database(program, old_module_info, policy, &mut self.instance) + .update_database(program, old_module_info, policy, deployment, &mut self.instance) } pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { @@ -531,11 +535,23 @@ impl WasmModuleInstance { res } - pub fn init_database(&mut self, program: Program) -> anyhow::Result> { - let module_def = &self.common.info.clone().module_def; + pub fn init_database( + &mut self, + program: Program, + deployment: Option, + ) -> anyhow::Result> { + let info = self.common.info.clone(); + let module_def = &info.module_def; let replica_ctx = &self.instance.replica_ctx().clone(); let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params); - let (res, trapped) = init_database(replica_ctx, module_def, program, call_reducer); + let (res, trapped) = init_database( + replica_ctx, + module_def, + info.module_hash, + program, + deployment, + call_reducer, + ); self.trapped = trapped; res } @@ -630,12 +646,82 @@ impl InstanceCommon { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, inst: &mut I, ) -> Result { let replica_ctx = inst.replica_ctx().clone(); let system_logger = replica_ctx.logger.system_logger(); let stdb = &replica_ctx.relational_db(); + let timestamp = Timestamp::now(); + let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, (admission, unchanged)) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<_> { + ensure!( + self.info.module_hash == program.hash, + "program does not match the instantiated module" + ); + let admission = if let Some(request) = &deployment { + deployment::validate_deployment_program(request, &program, &self.info.module_def)?; + deployment::check_deployment_commit(tx, request, timestamp, &Default::default())? + } else { + deployment::require_unmanaged_publication(tx)?; + CommitAdmission::Ready + }; + if matches!(admission, CommitAdmission::AlreadyCommitted(_)) { + return Ok((admission, false)); + } + deployment::validate_active_hosted_grants(tx, &self.info.module_def)?; + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use spacetimedb_datastore::system_tables::{read_hash_from_col, StModuleFields, ST_MODULE_ID}; + let row = tx.iter(ST_MODULE_ID)?.next().context("database is not initialized")?; + let stored_hash = read_hash_from_col(row, StModuleFields::ProgramHash)?; + ensure!( + stored_hash == old_module_info.module_hash, + "module changed before publication admission" + ); + Ok((admission, stored_hash == program.hash)) + })?; + if let CommitAdmission::AlreadyCommitted(result) = admission { + let (offset, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + let (sender, tx_offset) = tokio::sync::oneshot::channel(); + let _ = sender.send(offset); + return Ok(UpdateDatabaseResult::DeploymentAlreadyCommitted { + result, + tx_offset, + durable_offset: stdb.durable_tx_offset(), + }); + } + if unchanged { + let Some(request) = &deployment else { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + return Ok(UpdateDatabaseResult::NoUpdateNeeded); + }; + let (tx, _) = stdb.with_auto_rollback(tx, |tx| { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default()) + })?; + let event = ModuleEvent { + timestamp, + caller_identity: request.publisher, + caller_connection_id: None, + function_call: ModuleFunctionCall::update(), + status: EventStatus::Committed(DatabaseUpdate::default()), + reducer_return_value: None, + energy_quanta_used: FunctionBudget::ZERO.into(), + host_execution_duration: Duration::ZERO, + request_id: None, + timer: None, + }; + let durable_offset = stdb.durable_tx_offset(); + let CommitAndBroadcastEventSuccess { tx_offset, .. } = + commit_and_broadcast_event(&self.info.subscriptions, None, event, tx); + return Ok(UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + }); + } + let plan: MigratePlan = match policy.try_migrate( self.info.database_identity, old_module_info.module_hash, @@ -645,17 +731,24 @@ impl InstanceCommon { ) { Ok(plan) => plan, Err(e) => { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); return match e { MigrationPolicyError::AutoMigrateFailure(e) => Ok(UpdateDatabaseResult::AutoMigrateError(e.into())), _ => Ok(UpdateDatabaseResult::ErrorExecutingMigration(e.into())), - } + }; } }; let program_hash = program.hash; let host_type = HostType::from(program.kind); - let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| stdb.update_program(tx, program))?; + let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + stdb.update_program(tx, program)?; + if let Some(request) = &deployment { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default())?; + } + Ok(()) + })?; system_logger.info(&format!("Updated program to {program_hash}")); let auth_ctx = AuthCtx::for_current(replica_ctx.database.owner_identity); @@ -748,7 +841,9 @@ impl InstanceCommon { tx: MutTxId, inst: &mut I, ) -> Result<(ViewCallResult, bool), anyhow::Error> { - let view_calls = collect_subscribed_view_calls(&tx, &self.info.module_def, self.info.owner_identity)?; + let (tx, view_calls) = inst.replica_ctx().relational_db().with_auto_rollback(tx, |tx| { + collect_subscribed_view_calls(tx, &self.info.module_def, self.info.owner_identity) + })?; Ok(self.execute_view_calls(tx, view_calls, inst)) } @@ -761,11 +856,29 @@ impl InstanceCommon { timestamp, caller_identity, caller_connection_id, + call_auth_flags, + hosted_auth, timer, procedure_id, args, } = params; + let admission = inst + .replica_ctx() + .relational_db() + .with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, inst.replica_ctx().relational_db(), hosted_auth.as_deref()) + }); + if let Err(err) = admission { + return ( + CallProcedureReturn { + result: Err(ProcedureCallError::InternalError(err.to_string())), + tx_offset: None, + }, + false, + ); + } + // We've already validated by this point that the procedure exists, // so it's fine to use the panicking `procedure_by_id`. let procedure_def = self.info.module_def.procedure_by_id(procedure_id); @@ -780,6 +893,8 @@ impl InstanceCommon { name: procedure_name.clone(), caller_identity, caller_connection_id, + call_auth_flags, + hosted_auth, timestamp, arg_bytes: args.get_bsatn().clone(), }; @@ -957,6 +1072,8 @@ impl InstanceCommon { timestamp, caller_identity, caller_connection_id, + call_auth_flags, + hosted_auth, client, request_id, reducer_id, @@ -980,12 +1097,41 @@ impl InstanceCommon { name: reducer_name, caller_identity: &caller_identity, caller_connection_id: &caller_connection_id, + call_auth_flags, + hosted_auth, timestamp, args: &args, }; let workload = Workload::Reducer(ReducerContext::from(op.clone())); let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable, workload)); + if let Err(err) = check_hosted_admission(&tx, stdb, op.hosted_auth.as_deref()) { + let event = ModuleEvent { + timestamp, + caller_identity, + caller_connection_id: caller_connection_id_opt, + function_call: ModuleFunctionCall { + reducer: Some(reducer_name.clone()), + reducer_id, + args, + }, + status: EventStatus::FailedInternal(err.to_string()), + reducer_return_value: None, + energy_quanta_used: crate::energy::EnergyQuanta::ZERO, + host_execution_duration: Default::default(), + request_id, + timer, + }; + let event = commit_and_broadcast_event(&info.subscriptions, client, event, tx).event; + return ( + ReducerCallResult { + outcome: ReducerOutcome::from(&event.status), + energy_used: crate::energy::EnergyQuanta::ZERO, + execution_duration: Default::default(), + }, + false, + ); + } let mut tx_slot = inst.tx_slot(); let vm_metrics = self.vm_metrics.get_for_reducer_id(reducer_id); @@ -1772,6 +1918,12 @@ pub trait InstanceOp { fn name(&self) -> &Identifier; fn timestamp(&self) -> Timestamp; fn call_type(&self) -> FuncCallType; + fn call_auth_flags(&self) -> u32 { + 0 + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + None + } } /// Describes a view call in a cheaply shareable way. @@ -1842,12 +1994,20 @@ pub struct ReducerOp<'a> { pub name: &'a ReducerName, pub caller_identity: &'a Identity, pub caller_connection_id: &'a ConnectionId, + pub call_auth_flags: u32, + pub hosted_auth: Option>, pub timestamp: Timestamp, /// The arguments passed to the reducer. pub args: &'a ArgsTuple, } impl InstanceOp for ReducerOp<'_> { + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + self.hosted_auth.as_deref() + } fn name(&self) -> &Identifier { self.name.as_identifier() } @@ -1866,6 +2026,8 @@ impl From> for execution_context::ReducerContext { name, caller_identity, caller_connection_id, + call_auth_flags: _, + hosted_auth: _, timestamp, args, }: ReducerOp<'_>, @@ -1887,11 +2049,19 @@ pub struct ProcedureOp { pub name: Identifier, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + pub call_auth_flags: u32, + pub hosted_auth: Option>, pub timestamp: Timestamp, pub arg_bytes: Bytes, } impl InstanceOp for ProcedureOp { + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + self.hosted_auth.as_deref() + } fn name(&self) -> &Identifier { &self.name } diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 23113f1ab6c..13f427b3af2 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -28,6 +28,10 @@ use std::sync::Arc; use std::time::Instant; use wasmtime::{AsContext, Caller, StoreContextMut}; +/// Env reads may retain at most 2 MiB of value bytes outside the Wasm heap. +/// Other outstanding byte sources count against this interface's handle limit. +const MAX_OUTSTANDING_ENV_SOURCES: usize = 256; + /// A stream of bytes which the WASM module can read from /// using [`WasmInstanceEnv::bytes_source_read`]. /// @@ -188,7 +192,14 @@ impl WasmInstanceEnv { // This allows the module to avoid allocating and make a system call in those cases. if bytes.is_empty() { Ok(BytesSourceId::INVALID) - } else if bytes.len() > u32::MAX as usize { + } else { + self.create_present_bytes_source(bytes) + } + } + + /// Allocate a valid source even for an empty value when zero means absence. + fn create_present_bytes_source(&mut self, bytes: bytes::Bytes) -> RtResult { + if bytes.len() > u32::MAX as usize { // There's no inherent reason we need to error here, // other than that it makes it impossible to report the length in `bytes_source_remaining_length` // and that all of our usage of `BytesSource`s as of writing (pgoldman 2025-09-26) @@ -1544,6 +1555,51 @@ impl WasmInstanceEnv { }) } + /// Read an environment value as a nullable BytesSource. Zero means missing; + /// a present empty string always receives a nonzero, consumable source. + pub fn env_get( + caller: Caller<'_, Self>, + key: WasmPtr, + key_len: u32, + target_ptr: WasmPtr, + ) -> RtResult { + Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { + if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { + return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); + } + let (mem, env) = Self::mem_env(caller); + let key = mem.deref_str(key, key_len)?; + match env.instance_env.env_get(key)? { + None => Ok(0), + Some(value) => { + // These buffers live on the host heap until consumed or the + // invocation ends. Bound retained reads from hand-written Wasm. + if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { + return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); + } + Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) + } + } + }) + } + + /// Returns host-verified invocation flags. Bit 0 is internal authority. + /// This does not read tables and is available outside transactions. + pub fn get_call_auth_flags(caller: Caller<'_, Self>) -> u32 { + caller.data().instance_env.get_call_auth_flags() + } + + pub(crate) fn set_hosted_auth( + &mut self, + auth: Option>, + ) { + self.instance_env.set_hosted_auth(auth); + } + + pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { + self.instance_env.set_call_auth_flags(flags); + } + /// Writes the identity of the module into `out = out_ptr[..32]`. /// /// # Traps diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index c89131fa08b..65d1b6b5bb9 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -55,7 +55,7 @@ impl WasmtimeModule { WasmtimeModule { module } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 5); + pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 7); pub(super) fn link_imports(linker: &mut Linker) -> anyhow::Result<()> { link_imports(linker, AsyncImportMode::SyncStub) @@ -635,6 +635,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { store .data_mut() .start_funcall(reducer_name, args_bytes, op.timestamp, op.call_type()); + store.data_mut().set_call_auth_flags(op.call_auth_flags); + store.data_mut().set_hosted_auth(op.hosted_auth.clone()); let call_result = call_sync_typed_func( &self.call_reducer, @@ -758,6 +760,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { store .data_mut() .start_funcall(op.name.clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); + store.data_mut().set_call_auth_flags(op.call_auth_flags); + store.data_mut().set_hosted_auth(op.hosted_auth.clone()); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { diff --git a/crates/core/src/sql/ast.rs b/crates/core/src/sql/ast.rs index 892430ba1ea..4759c89e91d 100644 --- a/crates/core/src/sql/ast.rs +++ b/crates/core/src/sql/ast.rs @@ -34,6 +34,9 @@ impl SchemaView for SchemaViewer<'_, T> { } fn schema_for_table(&self, table_id: TableId) -> Option> { + if spacetimedb_datastore::system_tables::is_host_only_read_table(table_id) { + return None; + } self.tx .get_schema(table_id) .filter(|schema| self.auth.has_read_access(schema.table_access)) diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 2ecf80ad11a..d7b97a479cd 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1,3 +1,4 @@ +use crate::auth::invocation::{check_hosted_admission, SqlCallAuth}; use std::sync::Arc; use std::time::Duration; @@ -11,13 +12,14 @@ use crate::host::module_host::{ ViewOutcome, WasmInstance, }; use crate::host::{ArgsTuple, ModuleHost}; -use crate::subscription::module_subscription_actor::{commit_and_broadcast_event, ModuleSubscriptions}; +use crate::subscription::module_subscription_actor::ModuleSubscriptions; use crate::subscription::module_subscription_manager::TransactionOffset; use crate::subscription::tx::DeltaTx; use anyhow::anyhow; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_expr::statement::Statement; +#[cfg(test)] use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_lib::Timestamp; @@ -52,11 +54,15 @@ pub struct SqlResult { pub async fn run( db: Arc, sql_text: String, - auth: AuthCtx, + auth: impl Into, subs: Option, module: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result { + let auth = auth.into(); + if auth.hosted.is_some() && module.is_none() { + return Err(anyhow!("hosted SQL requires a module with hosted authentication capability").into()); + } match module { Some(module) => module.call_sql(db, sql_text, auth, subs, head).await, None => run_inner::(None, db, sql_text, auth, subs, head).map(|x| x.0), @@ -70,7 +76,7 @@ pub(crate) fn run_with_instance( instance: &mut RefInstance, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result<(SqlResult, bool), DBError> { @@ -81,14 +87,38 @@ fn run_inner( instance: Option<&mut RefInstance>, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result<(SqlResult, bool), DBError> { // We parse the sql statement in a mutable transaction. // If it turns out to be a query, we downgrade the tx. let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { - compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth) + check_hosted_admission(tx, &db, auth.hosted.as_deref())?; + let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; + // Check mutation authority while the automatic rollback guard owns + // the transaction, including rejected administrative statements. + if matches!(&stmt, Statement::DML(_) | Statement::Environment(_)) && !auth.has_write_access() { + return Err(anyhow!( + "Caller {} is not authorized to run SQL mutations", + auth.caller() + )); + } + if let Statement::DML(dml) = &stmt + && dml.table_id() == spacetimedb_datastore::system_tables::ST_ENV_ID + { + return Err(anyhow!( + "Use SET env.KEY or DELETE env.KEY to modify database environment variables" + )); + } + if let Statement::DML(dml) = &stmt + && spacetimedb_datastore::system_tables::is_host_managed_deployment_table(dml.table_id()) + { + return Err(anyhow!( + "Deployment and container authorization metadata may only be changed by the host" + )); + } + Ok(stmt) })?; let mut metrics = ExecutionMetrics::default(); @@ -101,7 +131,7 @@ fn run_inner( None => (tx, false), }; - let (tx_data, tx_metrics_mut, tx) = db.commit_tx_downgrade(tx, Workload::Sql); + let (tx_data, tx_metrics_mut, tx) = db.commit_tx_downgrade(tx, Workload::Sql)?; let (tx_offset_send, tx_offset) = oneshot::channel(); // Release the tx on drop, so that we record metrics @@ -141,14 +171,21 @@ fn run_inner( trapped, )) } - Statement::DML(stmt) => { - // An extra layer of auth is required for DML - if !auth.has_write_access() { - return Err(anyhow!("Caller {} is not authorized to run SQL DML statements", auth.caller()).into()); - } - + stmt @ (Statement::DML(_) | Statement::Environment(_)) => { // Evaluate the mutation - let (mut tx, _) = db.with_auto_rollback(tx, |tx| execute_dml_stmt(&auth, stmt, tx, &mut metrics))?; + let (mut tx, _) = db.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + match stmt { + Statement::DML(stmt) => execute_dml_stmt(&auth, stmt, tx, &mut metrics)?, + Statement::Environment(environment) => match environment.value { + Some(value) => crate::db::environment::set(&db, tx, &environment.key, &value)?, + None => { + crate::db::environment::delete(&db, tx, &environment.key)?; + } + }, + Statement::Select(_) => unreachable!(), + } + Ok(()) + })?; // Update transaction metrics tx.metrics.merge(metrics); @@ -209,7 +246,10 @@ fn run_inner( request_id: None, timer: None, }; - let res = commit_and_broadcast_event(&subs.unwrap(), None, event, tx); + let res = subs + .unwrap() + .commit_and_broadcast_event(None, event, tx)? + .map_err(|_| anyhow!("SQL transaction write conflict"))?; Ok(( SqlResult { tx_offset: res.tx_offset, @@ -243,6 +283,76 @@ pub(crate) mod tests { use spacetimedb_schema::schema::{ColumnSchema, TableSchema}; use spacetimedb_schema::table_name::TableName; + #[test] + fn environment_sql_enforces_permissions_escaping_limits_and_rollback() { + use spacetimedb_lib::identity::SqlPermission; + let db = TestDB::in_memory().unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let owner = AuthCtx::for_current(Identity::ZERO); + let viewer = AuthCtx::with_permissions( + Identity::ONE, + Arc::new(|permission| matches!(permission, SqlPermission::Read(_))), + ); + let outsider = AuthCtx::new(Identity::ZERO, Identity::ONE); + let execute = |statement: &str, auth: AuthCtx| { + runtime.block_on(run(db.clone(), statement.to_string(), auth, None, None, &mut vec![])) + }; + let value = "it's \\quoted;\nUTF-8 é\0tail"; + execute( + &format!( + "/* prefix */ SET env.Mixed_Key = '{}'; -- suffix", + value.replace('\'', "''") + ), + owner.clone(), + ) + .unwrap(); + execute("SET env.EMPTY TO ''", owner.clone()).unwrap(); + let rows = execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", viewer.clone()) + .unwrap() + .rows; + assert_eq!(rows, vec![product![value]]); + assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); + for auth in [viewer, outsider] { + assert!(execute("SET env.Mixed_Key = 'forbidden'", auth.clone()).is_err()); + assert!(execute("DELETE env.Mixed_Key", auth).is_err()); + } + for statement in [ + "SET env.EMPTY = 5".to_string(), + "SET env.\"BAD-KEY\" = 'value'".to_string(), + format!("SET env.EMPTY = '{}'", "x".repeat(8193)), + "SET env.EMPTY = 'changed'; DELETE env.Mixed_Key".to_string(), + "DELETE env.Mixed_Key WHERE true".to_string(), + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')".to_string(), + "UPDATE st_env SET value = 'bypass'".to_string(), + "DELETE FROM st_env".to_string(), + ] { + assert!( + execute(&statement, owner.clone()).is_err(), + "unexpectedly accepted {statement}" + ); + } + // Every rejected path released its transaction and preserved old data. + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner.clone()) + .unwrap() + .rows, + vec![product![""]] + ); + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", owner.clone()) + .unwrap() + .rows, + vec![product![value]] + ); + execute("SET env.EMPTY = 'updated'", owner.clone()).unwrap(); + execute("DELETE env.EMPTY", owner.clone()).unwrap(); + execute("DELETE env.EMPTY", owner.clone()).unwrap(); + assert!(execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner) + .unwrap() + .rows + .is_empty()); + } + /// Short-cut for simplify test execution pub(crate) fn run_for_testing(db: &Arc, sql_text: &str) -> Result, DBError> { let (subs, runtime) = ModuleSubscriptions::for_test_new_runtime(db.clone()); @@ -387,6 +497,25 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn owner_sql_cannot_rewrite_container_authority_or_deployment() -> ResultTest<()> { + let db = TestDB::in_memory()?; + for table in [ + "st_deployment", + "st_publish_fence", + "st_deployment_operation", + "st_container_fence", + "st_connection_auth", + ] { + // This uses the owner test context. Permission to mutate application + // tables does not grant permission to replace host operational state. + let error = run_for_testing(&db, &format!("DELETE FROM {table}")).unwrap_err(); + assert!(error.to_string().contains("may only be changed by the host"), "{error}"); + assert!(run_for_testing(&db, &format!("SELECT * FROM {table}"))?.is_empty()); + } + Ok(()) + } + #[test] fn test_limit() -> ResultTest<()> { let (db, _) = create_data(5)?; diff --git a/crates/core/src/subscription/module_subscription_actor.rs b/crates/core/src/subscription/module_subscription_actor.rs index d748fdd09ab..7bfb60f960a 100644 --- a/crates/core/src/subscription/module_subscription_actor.rs +++ b/crates/core/src/subscription/module_subscription_actor.rs @@ -7,6 +7,7 @@ use super::module_subscription_manager::{ use super::query::compile_query_with_hashes; use super::tx::DeltaTx; use super::{collect_table_update, TableUpdateType}; +use crate::auth::invocation::check_hosted_admission; use crate::client::messages::{ ProcedureResultMessage, SerializableMessage, SubscriptionData, SubscriptionError, SubscriptionMessage, SubscriptionResult, SubscriptionRows, SubscriptionUpdateMessage, TransactionUpdateMessage, @@ -32,6 +33,7 @@ use spacetimedb_data_structures::map::{HashCollectionExt as _, HashSet}; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; use spacetimedb_datastore::locking_tx_datastore::datastore::TxMetrics; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, TxId}; use spacetimedb_datastore::traits::{IsolationLevel, TxData}; use spacetimedb_durability::TxOffset; @@ -66,6 +68,7 @@ pub struct ModuleSubscriptions { stats: Arc, metrics: Arc, module_def_version: Arc, + hosted_connections: Arc>>>, } #[derive(Debug, Clone)] @@ -305,6 +308,7 @@ impl ModuleSubscriptions { Self { relational_db, subscriptions, + hosted_connections: Arc::new(RwLock::new(Vec::new())), broadcast_queue, stats, metrics, @@ -315,6 +319,46 @@ impl ModuleSubscriptions { } } + /// Register every hosted socket, even if it has no subscriptions. Taking the + /// database transaction before the registry lock serializes with fencing. + pub(crate) fn register_hosted_connection(&self, sender: &Arc) -> anyhow::Result<()> { + self.relational_db.with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, &self.relational_db, sender.auth.hosted.as_ref())?; + let mut connections = self.hosted_connections.write(); + connections.retain(|connection| connection.strong_count() != 0); + connections.push(Arc::downgrade(sender)); + Ok(()) + }) + } + + pub(crate) fn unregister_hosted_connection(&self, sender: &std::sync::Weak) { + self.hosted_connections + .write() + .retain(|connection| connection.strong_count() != 0 && !connection.ptr_eq(sender)); + } + + #[cfg(test)] + pub(crate) fn hosted_connection_count(&self) -> usize { + self.hosted_connections.read().len() + } + + /// Call with the transaction containing the new target fence. After its + /// durable commit, await every returned actor's completion before acking the + /// fence barrier. This also cancels socket batches already dequeued for I/O. + pub fn cancel_invalid_hosted_connections(&self, tx: &S) -> Vec { + let mut cancelled = Vec::new(); + self.hosted_connections.write().retain(|connection| { + let Some(connection) = connection.upgrade() else { + return false; + }; + if check_hosted_admission(tx, &self.relational_db, connection.auth.hosted.as_ref()).is_err() { + cancelled.push(connection.cancel_hosted_connection()); + } + true + }); + cancelled + } + pub fn set_module_def_version(&self, version: RawModuleDefVersion) { self.module_def_version .store(Self::encode_module_def_version(version), Ordering::Release); @@ -334,7 +378,8 @@ impl ModuleSubscriptions { fn decode_module_def_version(version: u8) -> RawModuleDefVersion { match version { 1 => RawModuleDefVersion::V10, - _ => RawModuleDefVersion::V9OrEarlier, + 0 => RawModuleDefVersion::V9OrEarlier, + _ => unreachable!("invalid stored module definition version"), } } @@ -635,6 +680,7 @@ impl ModuleSubscriptions { let hash_with_param = QueryHash::from_string(&sql, auth.caller(), true); let (mut_tx, _) = self.begin_mut_tx(Workload::Subscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let existing_query = { let guard = self.subscriptions.read(); @@ -737,6 +783,8 @@ impl ModuleSubscriptions { ) }; + let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let mut subscriptions = self.subscriptions.write(); let queries = return_on_err!( @@ -753,7 +801,8 @@ impl ModuleSubscriptions { return Ok(None); }; - let (mut tx, tx_offset) = self.unsubscribe_views(query, auth.caller())?; + let mut_tx = ScopeGuard::::into_inner(mut_tx); + let (mut tx, tx_offset) = self.unsubscribe_views_and_downgrade_tx(mut_tx, query, auth.caller())?; let (table_rows, metrics) = return_on_err_with_sql!( self.evaluate_initial_subscription(sender.clone(), query.clone(), &tx, &auth, TableUpdateType::Unsubscribe), @@ -816,6 +865,7 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -933,6 +983,7 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -1013,7 +1064,7 @@ impl ModuleSubscriptions { #[allow(clippy::type_complexity)] fn compile_queries( &self, - sender: Identity, + sender: &ClientConnectionSender, auth: AuthCtx, queries: &[Box], num_queries: usize, @@ -1029,13 +1080,14 @@ impl ModuleSubscriptions { subscribe_to_all_tables = true; continue; } - let hash = QueryHash::from_string(sql, sender, false); - let hash_with_param = QueryHash::from_string(sql, sender, true); + let hash = QueryHash::from_string(sql, sender.id.identity, false); + let hash_with_param = QueryHash::from_string(sql, sender.id.identity, true); query_hashes.push((sql, hash, hash_with_param)); } // We always get the db lock before the subscription lock to avoid deadlocks. let (mut_tx, _tx_offset) = self.begin_mut_tx(Workload::Subscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let compile_timer = metrics.compilation_time.start_timer(); @@ -1276,13 +1328,7 @@ impl ModuleSubscriptions { subscription_metrics.num_queries_subscribed.inc_by(num_queries as _); let (queries, auth, mut_tx, _compile_timer) = return_on_err!( - self.compile_queries( - sender.id.identity, - auth, - &request.query_strings, - num_queries, - subscription_metrics - ), + self.compile_queries(&sender, auth, &request.query_strings, num_queries, subscription_metrics), send_err_msg, (None, false) ); @@ -1367,13 +1413,7 @@ impl ModuleSubscriptions { subscription_metrics.num_queries_subscribed.inc_by(num_queries as _); let (queries, auth, mut_tx, compile_timer) = return_on_err!( - self.compile_queries( - sender.id.identity, - auth, - &request.query_strings, - num_queries, - subscription_metrics - ), + self.compile_queries(&sender, auth, &request.query_strings, num_queries, subscription_metrics), send_err_msg, (None, false) ); @@ -1522,7 +1562,7 @@ impl ModuleSubscriptions { subscription_metrics.num_queries_subscribed.inc_by(num_queries as _); let (queries, auth, mut_tx, compile_timer) = self.compile_queries( - sender.id.identity, + &sender, auth, &subscription.query_strings, num_queries, @@ -1669,7 +1709,7 @@ impl ModuleSubscriptions { // We'll later ensure tx is released/cleaned up once out of scope. let (read_tx, tx_data, tx_metrics_mut) = match &mut event.status { EventStatus::Committed(db_update) => { - let (tx_data, tx_metrics, read_tx) = stdb.commit_tx_downgrade(tx, Workload::Update); + let (tx_data, tx_metrics, read_tx) = stdb.commit_tx_downgrade(tx, Workload::Update)?; *db_update = DatabaseUpdate::from_writes(&tx_data); (read_tx, tx_data, tx_metrics) } @@ -1777,7 +1817,7 @@ impl ModuleSubscriptions { sender: Identity, ) -> Result<(TxGuard, TransactionOffset), DBError> { Self::_unsubscribe_views(&mut tx, view_collector, sender)?; - let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Unsubscribe); + let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Unsubscribe)?; let opts = GuardTxOptions::from_mut(tx_data, tx_metrics_mut); Ok(self.guard_tx(tx, opts)) } @@ -1812,7 +1852,7 @@ impl ModuleSubscriptions { (tx, trapped) = ModuleHost::materialize_views(tx, instance, view_collector, sender, Workload::Subscribe)?; }; - let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Subscribe); + let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Subscribe)?; let opts = GuardTxOptions::from_mut(tx_data, tx_metrics_mut); let (a, b) = self.guard_tx(tx, opts); diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index 03c3392942f..98710d5fc38 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -1776,7 +1776,13 @@ pub struct BroadcastQueue(SenderWithGauge); #[derive(thiserror::Error, Debug)] #[error(transparent)] -pub struct BroadcastError(#[from] mpsc::error::SendError); +pub struct BroadcastError(Box>); + +impl From> for BroadcastError { + fn from(error: mpsc::error::SendError) -> Self { + Self(Box::new(error)) + } +} impl BroadcastQueue { fn send(&self, message: SendWorkerMessage) -> Result<(), BroadcastError> { @@ -1980,7 +1986,7 @@ impl SendWorker { } // Send all the other updates. - let hide_reducer_info_for_non_callers = matches!(module_def_version, RawModuleDefVersion::V10); + let hide_reducer_info_for_non_callers = !matches!(module_def_version, RawModuleDefVersion::V9OrEarlier); for (id, update) in client_id_updates.drain() { let database_update = SubscriptionUpdateMessage::from_event_and_update(&event, update); diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index c479be085d9..b31deca8c68 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -309,6 +309,9 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); + for schema in crate::system_tables::deployment_system_schemas() { + self.create_table(schema.table_id, schema.into()); + } // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index e9d67103b16..f1f9a89fe67 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1006,6 +1006,10 @@ pub(crate) mod tests { ST_VIEW_ARG_NAME, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_NAME, ST_VIEW_ID, ST_VIEW_NAME, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_NAME, ST_VIEW_SUB_ID, ST_VIEW_SUB_NAME, }; + use crate::system_tables::{ + ST_CONNECTION_AUTH_ID, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, + ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, ST_PUBLISH_FENCE_ID, + }; use crate::traits::{IsolationLevel, MutTx}; use crate::Result; use core::{fmt, mem}; @@ -1472,6 +1476,13 @@ pub(crate) mod tests { TableRow { id: ST_TABLE_ACCESSOR_ID.into(), name: ST_TABLE_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, + TableRow { id: ST_ENV_ID.into(), name: "st_env", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_DEPLOYMENT_ID.into(), name: "st_deployment", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_PUBLISH_FENCE_ID.into(), name: "st_publish_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_DEPLOYMENT_OPERATION_ID.into(), name: "st_deployment_operation", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONTAINER_FENCE_ID.into(), name: "st_container_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONNECTION_AUTH_ID.into(), name: "st_connection_auth", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONTAINER_ENVIRONMENT_ID.into(), name: "st_container_environment", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] @@ -1569,6 +1580,30 @@ pub(crate) mod tests { ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 0, name: "table_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 1, name: "col_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 2, name: "accessor_name", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 0, name: "key", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 1, name: "value", ty: AlgebraicType::String }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 0, name: "key", ty: AlgebraicType::U8 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 1, name: "revision", ty: AlgebraicType::U256 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 2, name: "last_operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 3, name: "payload", ty: AlgebraicType::bytes() }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 0, name: "key", ty: AlgebraicType::U8 }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 1, name: "publication_epoch", ty: AlgebraicType::U64 }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 2, name: "operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 0, name: "operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 1, name: "previous_revision", ty: AlgebraicType::option(AlgebraicType::U256) }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 2, name: "committed_revision", ty: AlgebraicType::U256 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 3, name: "commit_result", ty: AlgebraicType::bytes() }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 4, name: "expires_at", ty: AlgebraicType::I64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 0, name: "source_identity", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 1, name: "generation", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 2, name: "target_grant_revision", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 3, name: "target_set_hash", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 4, name: "allowed", ty: AlgebraicType::Bool }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 0, name: "connection_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 1, name: "sender_identity", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 2, name: "call_auth_flags", ty: AlgebraicType::U32 }, + ColRow { table: ST_CONTAINER_ENVIRONMENT_ID.into(), pos: 0, name: "generation", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_ENVIRONMENT_ID.into(), pos: 1, name: "payload", ty: AlgebraicType::bytes() }, ])); #[rustfmt::skip] assert_eq!(query.scan_st_indexes()?, map_array([ @@ -1601,6 +1636,13 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col(0), name: "st_env_key_idx_btree", }, + IndexRow { id: 31, table: ST_DEPLOYMENT_ID.into(), col: col(0), name: "st_deployment_key_idx_btree", }, + IndexRow { id: 32, table: ST_PUBLISH_FENCE_ID.into(), col: col(0), name: "st_publish_fence_key_idx_btree", }, + IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col(0), name: "st_deployment_operation_operation_id_idx_btree", }, + IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col(0), name: "st_container_fence_source_identity_idx_btree", }, + IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col(0), name: "st_connection_auth_connection_id_idx_btree", }, + IndexRow { id: 36, table: ST_CONTAINER_ENVIRONMENT_ID.into(), col: col(0), name: "st_container_environment_generation_idx_btree", }, ])); let start = ST_RESERVED_SEQUENCE_RANGE as i128 + 1; #[rustfmt::skip] @@ -1646,6 +1688,13 @@ pub(crate) mod tests { ConstraintRow { constraint_id: 23, table_id: ST_INDEX_ACCESSOR_ID.into(), unique_columns: col(1), constraint_name: "st_index_accessor_accessor_name_key", }, ConstraintRow { constraint_id: 24, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 1], constraint_name: "st_column_accessor_table_name_col_name_key", }, ConstraintRow { constraint_id: 25, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 2], constraint_name: "st_column_accessor_table_name_accessor_name_key", }, + ConstraintRow { constraint_id: 26, table_id: ST_ENV_ID.into(), unique_columns: col(0), constraint_name: "st_env_key_key", }, + ConstraintRow { constraint_id: 27, table_id: ST_DEPLOYMENT_ID.into(), unique_columns: col(0), constraint_name: "st_deployment_key_key", }, + ConstraintRow { constraint_id: 28, table_id: ST_PUBLISH_FENCE_ID.into(), unique_columns: col(0), constraint_name: "st_publish_fence_key_key", }, + ConstraintRow { constraint_id: 29, table_id: ST_DEPLOYMENT_OPERATION_ID.into(), unique_columns: col(0), constraint_name: "st_deployment_operation_operation_id_key", }, + ConstraintRow { constraint_id: 30, table_id: ST_CONTAINER_FENCE_ID.into(), unique_columns: col(0), constraint_name: "st_container_fence_source_identity_key", }, + ConstraintRow { constraint_id: 31, table_id: ST_CONNECTION_AUTH_ID.into(), unique_columns: col(0), constraint_name: "st_connection_auth_connection_id_key", }, + ConstraintRow { constraint_id: 32, table_id: ST_CONTAINER_ENVIRONMENT_ID.into(), unique_columns: col(0), constraint_name: "st_container_environment_generation_key", }, ])); // Verify we get back the tables correctly with the proper ids... @@ -2079,6 +2128,13 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col(0), name: "st_env_key_idx_btree", }, + IndexRow { id: 31, table: ST_DEPLOYMENT_ID.into(), col: col(0), name: "st_deployment_key_idx_btree", }, + IndexRow { id: 32, table: ST_PUBLISH_FENCE_ID.into(), col: col(0), name: "st_publish_fence_key_idx_btree", }, + IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col(0), name: "st_deployment_operation_operation_id_idx_btree", }, + IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col(0), name: "st_container_fence_source_identity_idx_btree", }, + IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col(0), name: "st_connection_auth_connection_id_idx_btree", }, + IndexRow { id: 36, table: ST_CONTAINER_ENVIRONMENT_ID.into(), col: col(0), name: "st_container_environment_generation_idx_btree", }, IndexRow { id: seq_start, table: FIRST_NON_SYSTEM_ID, col: col(0), name: "Foo_id_idx_btree", }, IndexRow { id: seq_start + 1, table: FIRST_NON_SYSTEM_ID, col: col(1), name: "Foo_name_idx_btree", }, IndexRow { id: seq_start + 2, table: FIRST_NON_SYSTEM_ID, col: col(2), name: "Foo_age_idx_btree", }, diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 9008bc6da4e..b946d3667ac 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -11,9 +11,10 @@ use super::{ use crate::{ error::ViewError, system_tables::{ - system_tables, ConnectionIdViaU128, IdentityViaU256, StConnectionCredentialsFields, StConnectionCredentialsRow, - StViewColumnFields, StViewFields, StViewParamFields, StViewParamRow, StViewSubFields, StViewSubRow, - ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, ST_VIEW_ID, ST_VIEW_PARAM_ID, ST_VIEW_SUB_ID, + system_tables, ConnectionIdViaU128, IdentityViaU256, StConnectionAuthFields, StConnectionCredentialsFields, + StConnectionCredentialsRow, StViewColumnFields, StViewFields, StViewParamFields, StViewParamRow, + StViewSubFields, StViewSubRow, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, + ST_VIEW_ID, ST_VIEW_PARAM_ID, ST_VIEW_SUB_ID, }, }; use crate::{ @@ -2676,7 +2677,13 @@ impl MutTxId { ); } } - self.delete_st_client_credentials(database_identity, connection_id) + self.delete_st_client_credentials(database_identity, connection_id)?; + self.delete_col_eq( + ST_CONNECTION_AUTH_ID, + StConnectionAuthFields::ConnectionId.col_id(), + &ConnectionIdViaU128::from(connection_id).into(), + )?; + Ok(()) } /// Look up a client row by identity and connection ID in the `st_clients` system table. diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index e75cc76b365..546e4464de9 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -175,7 +175,7 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result false, + ST_CONNECTION_CREDENTIALS_ID | ST_CONNECTION_AUTH_ID | ST_CONTAINER_ENVIRONMENT_ID => false, // We don't define any system views, so none of the view-related tables can be system meta-descriptors. ST_VIEW_ID | ST_VIEW_PARAM_ID | ST_VIEW_COLUMN_ID | ST_VIEW_SUB_ID | ST_VIEW_ARG_ID => false, ST_EVENT_TABLE_ID => { @@ -205,7 +205,9 @@ pub enum SystemTable { st_table_accessor, } -pub fn system_tables() -> [TableSchema; 20] { +pub fn system_tables() -> [TableSchema; 27] { + let [env, deployment, publish_fence, deployment_operation, container_fence, connection_auth, container_environment] = + deployment_system_schemas(); [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -228,6 +230,13 @@ pub fn system_tables() -> [TableSchema; 20] { st_table_accessor_schema(), st_index_accessor_schema(), st_column_accessor_schema(), + env, + deployment, + publish_fence, + deployment_operation, + container_fence, + connection_auth, + container_environment, ] } @@ -311,6 +320,9 @@ macro_rules! st_fields_enum { } } +mod deployment; +pub use deployment::*; + // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StTableFields { "table_id", TableId = 0, @@ -668,6 +680,8 @@ fn system_module_def() -> ModuleDef { .with_unique_constraint(st_column_accessor_table_alias_cols) .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); + deployment::register_tables(&mut builder); + let result = builder .finish() .try_into() @@ -693,6 +707,7 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); + deployment::validate_tables(&result); result } @@ -741,6 +756,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_key", ConstraintId(23)); m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); + m.extend(deployment::CONSTRAINTS); m }; } @@ -779,6 +795,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_idx_btree", IndexId(27)); m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); + m.extend(deployment::INDEXES); m }; } @@ -968,7 +985,7 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_TABLE_ACCESSOR_ID => Some(st_table_accessor_schema()), ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), - _ => None, + table => deployment::system_schema(table), } } diff --git a/crates/datastore/src/system_tables/deployment.rs b/crates/datastore/src/system_tables/deployment.rs new file mode 100644 index 00000000000..03a43a30ece --- /dev/null +++ b/crates/datastore/src/system_tables/deployment.rs @@ -0,0 +1,274 @@ +//! Stable system schemas for environment and container deployment state. +//! +//! The deployment payload and operation result have versioned binary encodings, +//! so adding a protocol version does not change the system table row layout. +//! Publication and target fences are operational metadata: application restore +//! must preserve/reconcile their current authority before admitting execution. + +use super::*; + +pub const ST_ENV_ID: TableId = TableId(21); +pub const ST_DEPLOYMENT_ID: TableId = TableId(22); +pub const ST_PUBLISH_FENCE_ID: TableId = TableId(23); +pub const ST_DEPLOYMENT_OPERATION_ID: TableId = TableId(24); +pub const ST_CONTAINER_FENCE_ID: TableId = TableId(25); +pub const ST_CONNECTION_AUTH_ID: TableId = TableId(26); +pub const ST_CONTAINER_ENVIRONMENT_ID: TableId = TableId(27); + +pub const ST_ENV_NAME: &str = "st_env"; +pub const ST_DEPLOYMENT_NAME: &str = "st_deployment"; +pub const ST_PUBLISH_FENCE_NAME: &str = "st_publish_fence"; +pub const ST_DEPLOYMENT_OPERATION_NAME: &str = "st_deployment_operation"; +pub const ST_CONTAINER_FENCE_NAME: &str = "st_container_fence"; +pub const ST_CONNECTION_AUTH_NAME: &str = "st_connection_auth"; +pub const ST_CONTAINER_ENVIRONMENT_NAME: &str = "st_container_environment"; + +st_fields_enum!(enum StEnvFields { + "key", Key = 0, + "value", Value = 1, +}); +st_fields_enum!(enum StDeploymentFields { + "key", Key = 0, + "revision", Revision = 1, + "last_operation_id", LastOperationId = 2, + "payload", Payload = 3, +}); +st_fields_enum!(enum StPublishFenceFields { + "key", Key = 0, + "publication_epoch", PublicationEpoch = 1, + "operation_id", OperationId = 2, +}); +st_fields_enum!(enum StDeploymentOperationFields { + "operation_id", OperationId = 0, + "previous_revision", PreviousRevision = 1, + "committed_revision", CommittedRevision = 2, + "commit_result", CommitResult = 3, + "expires_at", ExpiresAt = 4, +}); +st_fields_enum!(enum StContainerFenceFields { + "source_identity", SourceIdentity = 0, + "generation", Generation = 1, + "target_grant_revision", TargetGrantRevision = 2, + "target_set_hash", TargetSetHash = 3, + "allowed", Allowed = 4, +}); +st_fields_enum!(enum StConnectionAuthFields { + "connection_id", ConnectionId = 0, + "sender_identity", SenderIdentity = 1, + "call_auth_flags", CallAuthFlags = 2, +}); + +st_fields_enum!(enum StContainerEnvironmentFields { + "generation", Generation = 0, + "payload", Payload = 1, +}); + +/// Secret-bearing host state. Never expose its retained history through SQL or module syscalls. +#[derive(Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StContainerEnvironmentRow { + pub generation: u64, + pub payload: Box<[u8]>, +} + +impl std::fmt::Debug for StContainerEnvironmentRow { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StContainerEnvironmentRow") + .field("generation", &self.generation) + .field("payload", &"[redacted]") + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StEnvRow { + pub key: String, + pub value: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDeploymentRow { + pub key: u8, + pub revision: Hash, + pub last_operation_id: u128, + pub payload: Box<[u8]>, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StPublishFenceRow { + pub key: u8, + pub publication_epoch: u64, + pub operation_id: u128, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDeploymentOperationRow { + pub operation_id: u128, + pub previous_revision: Option, + pub committed_revision: Hash, + pub commit_result: Box<[u8]>, + pub expires_at: TimestampViaI64, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StContainerFenceRow { + pub source_identity: IdentityViaU256, + pub generation: u64, + pub target_grant_revision: u64, + pub target_set_hash: Hash, + pub allowed: bool, +} + +/// Captured host authentication for lifecycle cleanup, including crash recovery. +/// Only hosted connections need a row; absent rows retain ordinary flags zero. +/// JWT claims and sender equality never reconstruct these flags. +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StConnectionAuthRow { + pub connection_id: ConnectionIdViaU128, + pub sender_identity: IdentityViaU256, + pub call_auth_flags: u32, +} + +macro_rules! row_conversions { + ($($row:ty),+ $(,)?) => {$ ( + impl TryFrom> for $row { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } + } + impl From<$row> for ProductValue { + fn from(row: $row) -> Self { to_product_value(&row) } + } + )+}; +} + +row_conversions!( + StEnvRow, + StDeploymentRow, + StPublishFenceRow, + StDeploymentOperationRow, + StContainerFenceRow, + StConnectionAuthRow, + StContainerEnvironmentRow +); + +pub(super) fn register_tables(builder: &mut RawModuleDefV9Builder) { + fn register(builder: &mut RawModuleDefV9Builder, name: &'static str) { + let ty = builder.add_type::(); + builder + .build_table(name, *ty.as_ref().expect("system row must be a product")) + .with_type(TableType::System) + .with_access(v9::TableAccess::Private) + .with_primary_key(ColId(0)) + .with_unique_constraint(ColId(0)) + .with_index_no_accessor_name(btree(ColId(0))); + } + register::(builder, ST_ENV_NAME); + register::(builder, ST_DEPLOYMENT_NAME); + register::(builder, ST_PUBLISH_FENCE_NAME); + register::(builder, ST_DEPLOYMENT_OPERATION_NAME); + register::(builder, ST_CONTAINER_FENCE_NAME); + register::(builder, ST_CONNECTION_AUTH_NAME); + register::(builder, ST_CONTAINER_ENVIRONMENT_NAME); +} + +pub(super) fn validate_tables(def: &ModuleDef) { + validate_system_table::(def, ST_ENV_NAME); + validate_system_table::(def, ST_DEPLOYMENT_NAME); + validate_system_table::(def, ST_PUBLISH_FENCE_NAME); + validate_system_table::(def, ST_DEPLOYMENT_OPERATION_NAME); + validate_system_table::(def, ST_CONTAINER_FENCE_NAME); + validate_system_table::(def, ST_CONNECTION_AUTH_NAME); + validate_system_table::(def, ST_CONTAINER_ENVIRONMENT_NAME); +} + +pub(crate) fn deployment_system_schemas() -> [TableSchema; 7] { + [ + st_schema(ST_ENV_NAME, ST_ENV_ID), + st_schema(ST_DEPLOYMENT_NAME, ST_DEPLOYMENT_ID), + st_schema(ST_PUBLISH_FENCE_NAME, ST_PUBLISH_FENCE_ID), + st_schema(ST_DEPLOYMENT_OPERATION_NAME, ST_DEPLOYMENT_OPERATION_ID), + st_schema(ST_CONTAINER_FENCE_NAME, ST_CONTAINER_FENCE_ID), + st_schema(ST_CONNECTION_AUTH_NAME, ST_CONNECTION_AUTH_ID), + st_schema(ST_CONTAINER_ENVIRONMENT_NAME, ST_CONTAINER_ENVIRONMENT_ID), + ] +} + +pub(super) fn system_schema(table: TableId) -> Option { + let name = match table { + ST_ENV_ID => ST_ENV_NAME, + ST_DEPLOYMENT_ID => ST_DEPLOYMENT_NAME, + ST_PUBLISH_FENCE_ID => ST_PUBLISH_FENCE_NAME, + ST_DEPLOYMENT_OPERATION_ID => ST_DEPLOYMENT_OPERATION_NAME, + ST_CONTAINER_FENCE_ID => ST_CONTAINER_FENCE_NAME, + ST_CONNECTION_AUTH_ID => ST_CONNECTION_AUTH_NAME, + ST_CONTAINER_ENVIRONMENT_ID => ST_CONTAINER_ENVIRONMENT_NAME, + _ => return None, + }; + Some(st_schema(name, table)) +} + +/// These tables are read through dedicated host operations by module code. +/// In particular, resolving a numeric table or index ID must not bypass this. +pub fn is_module_restricted_table(table: TableId) -> bool { + matches!( + table, + ST_ENV_ID + | ST_DEPLOYMENT_ID + | ST_PUBLISH_FENCE_ID + | ST_DEPLOYMENT_OPERATION_ID + | ST_CONTAINER_FENCE_ID + | ST_CONNECTION_AUTH_ID + | ST_CONTAINER_ENVIRONMENT_ID + ) +} + +/// Snapshot history is exclusively available through authenticated host operations. +pub fn is_host_only_read_table(table: TableId) -> bool { + table == ST_CONTAINER_ENVIRONMENT_ID +} + +pub fn is_module_restricted_index(index: IndexId) -> bool { + INDEXES.iter().any(|(_, restricted)| *restricted == index) +} + +/// Environment management has its own validated SQL path. Deployment and +/// authorization metadata may only be changed by authenticated host operations. +pub fn is_host_managed_deployment_table(table: TableId) -> bool { + matches!( + table, + ST_DEPLOYMENT_ID + | ST_PUBLISH_FENCE_ID + | ST_DEPLOYMENT_OPERATION_ID + | ST_CONTAINER_FENCE_ID + | ST_CONNECTION_AUTH_ID + | ST_CONTAINER_ENVIRONMENT_ID + ) +} + +pub(super) const CONSTRAINTS: [(&str, ConstraintId); 7] = [ + ("st_env_key_key", ConstraintId(26)), + ("st_deployment_key_key", ConstraintId(27)), + ("st_publish_fence_key_key", ConstraintId(28)), + ("st_deployment_operation_operation_id_key", ConstraintId(29)), + ("st_container_fence_source_identity_key", ConstraintId(30)), + ("st_connection_auth_connection_id_key", ConstraintId(31)), + ("st_container_environment_generation_key", ConstraintId(32)), +]; + +pub(super) const INDEXES: [(&str, IndexId); 7] = [ + ("st_env_key_idx_btree", IndexId(30)), + ("st_deployment_key_idx_btree", IndexId(31)), + ("st_publish_fence_key_idx_btree", IndexId(32)), + ("st_deployment_operation_operation_id_idx_btree", IndexId(33)), + ("st_container_fence_source_identity_idx_btree", IndexId(34)), + ("st_connection_auth_connection_id_idx_btree", IndexId(35)), + ("st_container_environment_generation_idx_btree", IndexId(36)), +]; diff --git a/crates/expr/src/errors.rs b/crates/expr/src/errors.rs index 489882ce8c8..9148ed64b7b 100644 --- a/crates/expr/src/errors.rs +++ b/crates/expr/src/errors.rs @@ -132,6 +132,10 @@ pub struct DmlOnView { #[derive(Error, Debug)] pub enum TypingError { + #[error(transparent)] + Environment(#[from] spacetimedb_lib::environment::EnvironmentValidationError), + #[error("environment values must be SQL string literals")] + EnvironmentValueType, #[error(transparent)] Unsupported(#[from] Unsupported), #[error(transparent)] diff --git a/crates/expr/src/statement.rs b/crates/expr/src/statement.rs index b7422dd031c..797f2c8fc7c 100644 --- a/crates/expr/src/statement.rs +++ b/crates/expr/src/statement.rs @@ -31,6 +31,12 @@ use super::{ pub enum Statement { Select(ProjectList), DML(DML), + Environment(EnvironmentWrite), +} + +pub struct EnvironmentWrite { + pub key: Box, + pub value: Option>, } pub enum DML { @@ -458,6 +464,21 @@ pub fn parse_and_type_sql(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Ty SqlAst::Update(update) => Ok(Statement::DML(DML::Update(type_update(update, tx)?))), SqlAst::Set(set) => Ok(Statement::DML(DML::Insert(type_and_rewrite_set(set, tx)?))), SqlAst::Show(show) => Ok(Statement::Select(type_and_rewrite_show(show, tx)?)), + SqlAst::Environment(environment) => { + // Resolve through the normal private-table visibility check. + tx.schema("st_env").ok_or_else(|| Unresolved::table("st_env"))?; + let key = &*environment.key.0; + spacetimedb_lib::environment::validate_key(key)?; + let value = match environment.value { + None => None, + Some(SqlLiteral::Str(value)) => { + spacetimedb_lib::environment::validate_value(&value)?; + Some(value) + } + Some(_) => return Err(TypingError::EnvironmentValueType), + }; + Ok(Statement::Environment(EnvironmentWrite { key: key.into(), value })) + } } } diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs new file mode 100644 index 00000000000..02d9c581824 --- /dev/null +++ b/crates/lib/src/container.rs @@ -0,0 +1,478 @@ +//! Shared, versioned container deployment data. +//! +//! A decoded declaration is untrusted. Publication must normalize and validate +//! the complete effective deployment before hashing or admitting it. Runtime +//! capabilities and database authorization are additional admission checks. + +use crate::{bsatn, hash_bytes, Hash, SpacetimeType}; +use std::{collections::BTreeSet, fmt, str::FromStr}; + +#[cfg(feature = "serde")] +pub mod endpoints; +#[cfg(feature = "serde")] +pub mod operations; + +/// Version of the normalized deployment encoding, independent of module ABI. +pub const CONTAINER_SPEC_VERSION: u32 = 1; +pub const MAX_ARGV_ENTRIES: usize = 256; +pub const MAX_ENV_KEYS: usize = 256; +pub const MAX_PORTS: usize = 16; +pub const MAX_EXEC_STRING_BYTES: usize = 32 * 1024; +/// Includes NUL terminators, 64-bit pointer arrays, and reserved startup space. +pub const MAX_EXEC_BYTES: usize = 128 * 1024; +pub const EXEC_RESERVED_BYTES: usize = 4096; +pub const DEFAULT_STOP_GRACE_MS: u32 = 30_000; +pub const MAX_STOP_GRACE_MS: u32 = 120_000; + +/// The digest of an OCI object. This is never a SpacetimeDB Keccak-256 program key. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, SpacetimeType)] +#[sats(crate = crate)] +pub enum OciDigest { + Sha256(Hash), +} + +impl OciDigest { + pub const fn sha256(bytes: [u8; 32]) -> Self { + Self::Sha256(Hash::from_byte_array(bytes)) + } + + pub fn as_bytes(&self) -> &[u8; 32] { + match self { + Self::Sha256(hash) => &hash.data, + } + } +} + +impl fmt::Display for OciDigest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Sha256(bytes) => write!(f, "sha256:{}", bytes.to_hex()), + } + } +} + +impl FromStr for OciDigest { + type Err = ContainerValidationError; + + fn from_str(value: &str) -> Result { + let hex = value + .strip_prefix("sha256:") + .ok_or_else(|| invalid("image_manifest", "only sha256 OCI digests are supported"))?; + if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(invalid( + "image_manifest", + "expected 64 lowercase hexadecimal digest characters", + )); + } + let mut bytes = [0; 32]; + hex::decode_to_slice(hex, &mut bytes).map_err(|_| invalid("image_manifest", "invalid SHA-256 digest"))?; + Ok(Self::sha256(bytes)) + } +} + +#[cfg(feature = "serde")] +impl serde::Serialize for OciDigest { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +#[cfg(feature = "serde")] +impl<'de> serde::Deserialize<'de> for OciDigest { + fn deserialize>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ImagePlatform { + pub os: String, + pub architecture: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum ContainerMode { + Service, + Job, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum RestartPolicy { + Never, + OnFailure, + Always, +} + +impl RestartPolicy { + /// Only process termination drives this policy. Readiness is independent. + pub fn restarts_after(self, exit_code: Option) -> bool { + match self { + Self::Never => false, + Self::OnFailure => exit_code != Some(0), + Self::Always => true, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerResources { + pub cpu_millicores: u64, + pub memory_bytes: u64, + pub scratch_bytes: u64, + /// Linux tasks, including threads and commands started through exec. + pub pids_max: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum PortProtocol { + Http, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum PortExposure { + Public, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ReadinessProbe { + Tcp(TcpProbe), + Http(HttpProbe), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct TcpProbe { + pub timeout_ms: u32, + pub interval_ms: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct HttpProbe { + pub path: String, + pub timeout_ms: u32, + pub interval_ms: u32, +} + +impl Default for ReadinessProbe { + fn default() -> Self { + Self::Tcp(TcpProbe { + timeout_ms: 1000, + interval_ms: 5000, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerPort { + pub name: String, + pub port: u16, + pub protocol: PortProtocol, + /// Required in the public input, with no implicit exposure default. + pub exposure: PortExposure, + #[cfg_attr(feature = "serde", serde(default))] + pub readiness_probe: ReadinessProbe, +} + +/// A placeholder declaration is never admitted until the mount protocol ships. +/// Keeping explicit declarations lets Stage 1 return an unsupported-feature error. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerMount { + pub database: String, + pub source: String, + pub target: String, + pub read_only: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerSpec { + pub image_manifest: OciDigest, + pub image_platform: ImagePlatform, + /// Effective complete argv after applying OCI Entrypoint/Cmd or override. + pub argv: Vec, + pub user: String, + pub working_directory: String, + pub mode: ContainerMode, + pub restart: RestartPolicy, + pub env_keys: Vec, + pub resources: ContainerResources, + pub ports: Vec, + pub mounts: Vec, + pub stop_grace_ms: u32, +} + +/// Explicit component removal is distinct from omission or an empty replacement. +#[derive(Clone, Debug, Default, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +#[expect( + clippy::large_enum_variant, + reason = "the normalized publish request owns its single container spec" +)] +pub enum ContainerAction { + #[default] + Keep, + Set(ContainerSpec), + Remove, +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid container {field}: {reason}")] +pub struct ContainerValidationError { + pub field: &'static str, + pub reason: &'static str, +} + +fn invalid(field: &'static str, reason: &'static str) -> ContainerValidationError { + ContainerValidationError { field, reason } +} + +/// Limits are operational configuration; normalized specs are checked again at +/// admission against the selected node's enforceable capacities and capabilities. +#[derive(Clone, Copy, Debug)] +pub struct ContainerSpecLimits { + pub resources: ContainerResources, +} + +impl Default for ContainerSpecLimits { + fn default() -> Self { + Self { + resources: ContainerResources { + cpu_millicores: 64_000, + memory_bytes: 128 * 1024 * 1024 * 1024, + scratch_bytes: 1024 * 1024 * 1024 * 1024, + pids_max: 4096, + }, + } + } +} + +impl ContainerSpec { + /// Sort semantically unordered declarations so equivalent specs hash alike. + /// Duplicates are rejected, never silently deduplicated. + pub fn normalize(mut self, limits: &ContainerSpecLimits) -> Result { + self.validate(limits)?; + self.env_keys.sort_unstable(); + self.ports.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + Ok(self) + } + + pub fn validate(&self, limits: &ContainerSpecLimits) -> Result<(), ContainerValidationError> { + if self.image_platform.os != "linux" || !matches!(self.image_platform.architecture.as_str(), "amd64" | "arm64") + { + return Err(invalid("image_platform", "expected linux/amd64 or linux/arm64")); + } + if !self.mounts.is_empty() { + return Err(invalid("mounts", "SpacetimeFS mounts are not supported by Stage 1")); + } + if self.mode == ContainerMode::Job && self.restart == RestartPolicy::Always { + return Err(invalid("restart", "jobs cannot use the always restart policy")); + } + if self.argv.is_empty() || self.argv.len() > MAX_ARGV_ENTRIES || self.argv[0].is_empty() { + return Err(invalid( + "argv", + "a nonempty command with at most 256 arguments is required", + )); + } + validate_exec_size(&self.argv, &[])?; + if self.user.len() > 255 || self.user.bytes().any(|b| b == 0 || b.is_ascii_control()) { + return Err(invalid( + "user", + "user must fit 255 bytes and contain no control characters", + )); + } + if !self.working_directory.starts_with('/') + || self.working_directory.len() > 4096 + || self.working_directory.contains('\0') + { + return Err(invalid( + "working_directory", + "expected an absolute Linux path of at most 4096 bytes", + )); + } + if self.stop_grace_ms > MAX_STOP_GRACE_MS { + return Err(invalid("stop_grace_ms", "stop grace exceeds the supported deadline")); + } + let r = self.resources; + let max = limits.resources; + if r.cpu_millicores == 0 + || r.cpu_millicores > max.cpu_millicores + || r.memory_bytes == 0 + || r.memory_bytes > max.memory_bytes + || r.scratch_bytes == 0 + || r.scratch_bytes > max.scratch_bytes + || r.pids_max == 0 + || r.pids_max > max.pids_max + { + return Err(invalid( + "resources", + "resource reservations must be positive and within server limits", + )); + } + if self.env_keys.len() > MAX_ENV_KEYS { + return Err(invalid("env_keys", "too many environment keys")); + } + let mut keys = BTreeSet::new(); + for key in &self.env_keys { + validate_env_key(key)?; + if !keys.insert(key) { + return Err(invalid("env_keys", "duplicate environment key")); + } + } + if self.ports.len() > MAX_PORTS { + return Err(invalid("ports", "too many declared ports")); + } + let (mut names, mut numbers) = (BTreeSet::new(), BTreeSet::new()); + for port in &self.ports { + let bytes = port.name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 32 + || !bytes[0].is_ascii_lowercase() + || !bytes + .iter() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || *b == b'-') + { + return Err(invalid("ports.name", "expected [a-z][a-z0-9-]{0,31}")); + } + if port.port == 0 || !names.insert(&port.name) || !numbers.insert(port.port) { + return Err(invalid("ports", "port names and nonzero port numbers must be unique")); + } + let (timeout, interval) = match &port.readiness_probe { + ReadinessProbe::Tcp(TcpProbe { + timeout_ms, + interval_ms, + }) => (*timeout_ms, *interval_ms), + ReadinessProbe::Http(HttpProbe { + path, + timeout_ms, + interval_ms, + }) => { + if !path.starts_with('/') + || path.starts_with("//") + || path.len() > 2048 + || path + .bytes() + .any(|b| b.is_ascii_control() || b == b' ' || b == b'\\' || b == b'#') + { + return Err(invalid( + "readiness_probe.path", + "expected a bounded origin-relative HTTP path", + )); + } + (*timeout_ms, *interval_ms) + } + }; + if timeout == 0 || timeout > 30_000 || interval == 0 || interval > 300_000 || timeout > interval { + return Err(invalid("readiness_probe", "invalid probe timeout or interval")); + } + } + Ok(()) + } + + /// Domain-separated, versioned BSATN encoding, after normalization. + /// This hashes the container spec only; the full deployment also includes + /// the module selection and uses its own revision domain. + pub fn canonical_hash(&self, limits: &ContainerSpecLimits) -> Result { + let normalized = self.clone().normalize(limits)?; + let encoded = bsatn::to_vec(&(CONTAINER_SPEC_VERSION, normalized)) + .expect("encoding an in-memory container specification cannot fail"); + let mut bytes = b"spacetimedb/container-spec\0".to_vec(); + bytes.extend(encoded); + Ok(hash_bytes(&bytes)) + } +} + +pub fn validate_env_key(key: &str) -> Result<(), ContainerValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > 256 + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(invalid("env_keys", "invalid POSIX environment variable name")); + } + if key.starts_with("SPACETIMEDB_") { + return Err(invalid( + "env_keys", + "SPACETIMEDB_ variables are reserved for the platform", + )); + } + Ok(()) +} + +/// Call after merging image environment, the database snapshot, and platform +/// variables. Error messages deliberately contain neither argv nor env values. +pub fn validate_exec_size(argv: &[String], env: &[String]) -> Result<(), ContainerValidationError> { + let count = argv + .len() + .checked_add(env.len()) + .and_then(|n| n.checked_add(2)) + .ok_or_else(|| invalid("exec", "too many arguments or environment entries"))?; + let mut total = count + .checked_mul(8) + .and_then(|n| n.checked_add(EXEC_RESERVED_BYTES)) + .ok_or_else(|| invalid("exec", "argument and environment size overflow"))?; + for value in argv.iter().chain(env) { + if value.contains('\0') || value.len() >= MAX_EXEC_STRING_BYTES { + return Err(invalid( + "exec", + "argument or environment entry contains NUL or exceeds the per-entry limit", + )); + } + total = total + .checked_add(value.len() + 1) + .ok_or_else(|| invalid("exec", "argument and environment size overflow"))?; + } + if total > MAX_EXEC_BYTES { + return Err(invalid( + "exec", + "combined argument and environment size exceeds the startup limit", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/container/endpoints.rs b/crates/lib/src/container/endpoints.rs new file mode 100644 index 00000000000..d5ab7964b11 --- /dev/null +++ b/crates/lib/src/container/endpoints.rs @@ -0,0 +1,57 @@ +//! Public address discovery. These addresses carry no administrative or runtime +//! authority and do not promise that an application is currently ready. + +use super::PortProtocol; +use crate::Identity; +use serde::{Deserialize, Serialize}; + +pub const ENDPOINTS_PENDING: &str = "endpoints_pending"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerEndpoints { + pub database_identity: Identity, + pub endpoints: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerEndpoint { + pub name: String, + pub protocol: PortProtocol, + pub url: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovery_wire_contains_only_explicit_public_addresses() { + let response = ContainerEndpoints { + database_identity: Identity::ZERO, + endpoints: vec![ContainerEndpoint { + name: "http".into(), + protocol: PortProtocol::Http, + url: "https://aaaqeayeaudaocajbifqydiob4.container.example.net/".into(), + }], + }; + let encoded = serde_json::to_value(&response).unwrap(); + assert_eq!(encoded.as_object().unwrap().len(), 2); + assert_eq!( + encoded["endpoints"][0], + serde_json::json!({ + "name": "http", + "protocol": "http", + "url": "https://aaaqeayeaudaocajbifqydiob4.container.example.net/" + }) + ); + assert_eq!( + serde_json::from_value::(encoded.clone()).unwrap(), + response + ); + let mut changed = encoded; + changed["endpoints"][0]["upstream"] = serde_json::json!("10.0.0.1:8080"); + assert!(serde_json::from_value::(changed).is_err()); + } +} diff --git a/crates/lib/src/container/operations.rs b/crates/lib/src/container/operations.rs new file mode 100644 index 00000000000..f6b499f1db8 --- /dev/null +++ b/crates/lib/src/container/operations.rs @@ -0,0 +1,199 @@ +//! Control-plane container inspection and idempotent lifecycle messages. +use crate::{container::endpoints::ContainerEndpoint, deployment::uuid_json, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContainerAction { + Start, + Stop, + Restart, +} +impl ContainerAction { + pub const fn path(self) -> &'static str { + match self { + Self::Start => "start", + Self::Stop => "stop", + Self::Restart => "restart", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerOperationRequest { + #[serde(with = "uuid_json")] + pub request_id: Uuid, +} + +/// Confirms admission only. It does not claim physical stop or runtime readiness. +/// An exact retry returns the generation originally recorded for this request. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerOperationReceipt { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub request_id: Uuid, + pub action: ContainerAction, + #[serde(with = "decimal_u64")] + pub generation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DesiredState { + Stopped, + Running, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ObservedState { + Pending, + Starting, + Running, + Ready, + Draining, + Stopped, + Completed, + Failed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Condition { + None, + Fencing, + NoLeader, + NodeUnavailable, + Capacity, + BalanceUnavailable, + BalanceExhausted, + TargetUnavailable, + PullFailed, + LaunchFailed, + ReadinessFailed, + ExitFailure, + OutOfMemory, + NodePressure, + LeaseExpired, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CurrentInstance { + #[serde(with = "decimal_u64")] + pub generation: u64, + pub state: ObservedState, + pub observed_revision: Option, + /// Application snapshot identity only. No values or hashes of values. + pub applied_env_generation: Option, + pub exit_code: Option, + pub oom_killed: bool, + pub condition: Condition, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperationalState { + pub desired_revision: Hash, + pub desired_state: DesiredState, + #[serde(with = "decimal_u64")] + pub generation: u64, + pub condition: Condition, + pub restart_pending: bool, + pub restart_attempt: u32, + /// Unix milliseconds, represented exactly for browser clients. + #[serde(with = "decimal_u64")] + pub restart_not_before_ms: u64, + /// Reports only this operational generation. A historical attempt is never + /// presented as the currently authorized replacement. + pub current_instance: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum EndpointStatus { + Available { endpoints: Vec }, + Pending, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerStatus { + pub database_identity: Identity, + pub published: bool, + pub operational: Option, + /// Address discovery is independent of readiness and remains available + /// while stopped. Pending never means an empty declaration. + pub endpoints: EndpointStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContainerErrorCode { + InvalidRequest, + AccessDenied, + NotFound, + Conflict, + Unavailable, + OutcomeUnknown, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerApiError { + pub error: ContainerErrorCode, +} + +mod decimal_u64 { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(number: &u64, serializer: S) -> Result { + serializer.collect_str(number) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + let value: u64 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal u64")); + } + Ok(value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn receipt_preserves_full_generation_and_rejects_lossy_or_extra_fields() { + let receipt = ContainerOperationReceipt { + database_identity: Identity::ONE, + request_id: Uuid::from_u128(0x01950000000070008000000000000001), + action: ContainerAction::Restart, + generation: u64::MAX, + }; + let mut value = serde_json::to_value(&receipt).unwrap(); + assert_eq!(value["generation"], u64::MAX.to_string()); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + receipt + ); + for generation in [ + serde_json::json!(1), + serde_json::json!("01"), + serde_json::json!("18446744073709551616"), + ] { + value["generation"] = generation; + assert!(serde_json::from_value::(value.clone()).is_err()); + } + value = serde_json::to_value(receipt).unwrap(); + value["credential"] = serde_json::json!("unexpected"); + assert!(serde_json::from_value::(value).is_err()); + } + #[test] + fn pending_discovery_is_distinct_from_empty() { + assert_ne!( + serde_json::to_value(EndpointStatus::Pending).unwrap(), + serde_json::to_value(EndpointStatus::Available { endpoints: vec![] }).unwrap() + ); + } +} diff --git a/crates/lib/src/container/tests.rs b/crates/lib/src/container/tests.rs new file mode 100644 index 00000000000..dd7c208f43c --- /dev/null +++ b/crates/lib/src/container/tests.rs @@ -0,0 +1,246 @@ +use super::*; + +fn spec() -> ContainerSpec { + ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec!["API_KEY".into(), "DATABASE_URL".into()], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![ContainerPort { + name: "http".into(), + port: 8080, + protocol: PortProtocol::Http, + exposure: PortExposure::Public, + readiness_probe: ReadinessProbe::default(), + }], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } +} + +#[test] +fn oci_digest_is_strict_and_cannot_be_used_for_path_traversal() { + let digest = format!("sha256:{}", "ab".repeat(32)); + assert_eq!(digest.parse::().unwrap().to_string(), digest); + for value in [ + "sha256:../../host", + "sha256:abc", + "sha512:abc", + "blake3:abc", + "SHA256:abc", + "sha256:", + ] { + assert!(value.parse::().is_err(), "{value}"); + } + assert!(format!("sha256:{}", "AB".repeat(32)).parse::().is_err()); + assert!(format!("sha256:{}\n", "ab".repeat(32)).parse::().is_err()); +} + +#[test] +fn canonical_hash_ignores_declaration_order_but_preserves_launch_semantics() { + let limits = ContainerSpecLimits::default(); + let mut original = spec(); + original.ports.push(ContainerPort { + name: "admin".into(), + port: 8081, + ..original.ports[0].clone() + }); + let hash = original.canonical_hash(&limits).unwrap(); + let mut reordered = original.clone(); + reordered.env_keys.reverse(); + reordered.ports.reverse(); + assert_eq!(reordered.canonical_hash(&limits).unwrap(), hash); + let mut changed = reordered; + changed.argv.push("--read-only".into()); + assert_ne!(changed.canonical_hash(&limits).unwrap(), hash); + assert_ne!( + ContainerSpec { + resources: ContainerResources { + pids_max: 63, + ..original.resources + }, + ..original.clone() + } + .canonical_hash(&limits) + .unwrap(), + hash + ); + let bytes = bsatn::to_vec(&original).unwrap(); + assert_eq!(bsatn::from_slice::(&bytes).unwrap(), original); +} + +#[test] +fn duplicate_declarations_are_not_silently_deduplicated() { + let limits = ContainerSpecLimits::default(); + let mut value = spec(); + value.env_keys.push(value.env_keys[0].clone()); + assert_eq!(value.normalize(&limits).unwrap_err().field, "env_keys"); + let mut value = spec(); + value.ports.push(ContainerPort { + name: "another".into(), + ..value.ports[0].clone() + }); + assert_eq!(value.normalize(&limits).unwrap_err().field, "ports"); + let mut value = spec(); + value.ports.push(ContainerPort { + port: 9090, + ..value.ports[0].clone() + }); + assert_eq!(value.normalize(&limits).unwrap_err().field, "ports"); +} + +#[test] +fn readiness_cannot_change_authority_or_inject_a_request() { + for path in [ + "https://internal/", + "//internal/", + "/\\internal/", + "/health\r\nHost: internal", + "/health#fragment", + ] { + let mut value = spec(); + value.ports[0].readiness_probe = ReadinessProbe::Http(HttpProbe { + path: path.into(), + timeout_ms: 1000, + interval_ms: 5000, + }); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "readiness_probe.path" + ); + } + let mut value = spec(); + value.ports[0].readiness_probe = ReadinessProbe::Http(HttpProbe { + path: "/health?full=1".into(), + timeout_ms: 1000, + interval_ms: 5000, + }); + value.validate(&ContainerSpecLimits::default()).unwrap(); +} + +#[test] +fn stage_one_rejects_mounts_and_unsupported_platforms() { + let mut value = spec(); + value.mounts.push(ContainerMount { + database: "self".into(), + source: "/".into(), + target: "/spacetime".into(), + read_only: false, + }); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "mounts" + ); + let mut value = spec(); + value.image_platform.os = "windows".into(); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "image_platform" + ); +} + +#[test] +fn startup_size_counts_environment_and_pointers_without_leaking_values() { + let secret = "THIS_VALUE_MUST_NOT_APPEAR_IN_ERRORS"; + let env = vec![format!("KEY={secret}\0")]; + let error = validate_exec_size(&spec().argv, &env).unwrap_err().to_string(); + assert!(!error.contains(secret)); + assert!(!error.contains("KEY=")); + let bounded_strings = vec!["x".repeat(MAX_EXEC_STRING_BYTES - 1); 5]; + assert!(validate_exec_size(&spec().argv, &bounded_strings).is_err()); + let many_empty_entries = vec![String::new(); MAX_EXEC_BYTES / 8]; + assert!(validate_exec_size(&[], &many_empty_entries).is_err()); + assert!(validate_exec_size(&spec().argv, &["KEY=value".into()]).is_ok()); +} + +#[test] +fn tenant_environment_cannot_override_platform_discovery_or_credentials() { + for key in [ + "SPACETIMEDB_DATABASE_IDENTITY", + "SPACETIMEDB_SERVER_URI", + "SPACETIMEDB_CREDENTIAL_BROKER", + "SPACETIMEDB_TOKEN", + "A=B", + "0BAD", + "BAD\0KEY", + ] { + assert!(validate_env_key(key).is_err(), "{key:?}"); + } + for key in ["API_KEY", "_CUSTOM", "path", "A1"] { + validate_env_key(key).unwrap(); + } +} + +#[test] +fn zero_or_overflowing_resource_requests_are_not_admitted() { + let limits = ContainerSpecLimits::default(); + for cpu in [0, u64::MAX, limits.resources.cpu_millicores + 1] { + let mut value = spec(); + value.resources.cpu_millicores = cpu; + assert_eq!(value.validate(&limits).unwrap_err().field, "resources"); + } + let mut value = spec(); + value.resources.scratch_bytes = 0; + assert!(value.validate(&limits).is_err()); +} + +#[test] +fn successful_jobs_and_on_failure_services_remain_terminal() { + assert!(!RestartPolicy::Never.restarts_after(None)); + assert!(!RestartPolicy::Never.restarts_after(Some(1))); + assert!(!RestartPolicy::OnFailure.restarts_after(Some(0))); + assert!(RestartPolicy::OnFailure.restarts_after(None)); + assert!(RestartPolicy::OnFailure.restarts_after(Some(1))); + let mut job = spec(); + job.mode = ContainerMode::Job; + job.restart = RestartPolicy::Always; + assert_eq!( + job.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "restart" + ); +} + +#[cfg(feature = "serde")] +#[test] +fn json_requires_explicit_port_exposure_and_rejects_privileged_fields() { + let original = spec(); + let json = serde_json::to_value(&original).unwrap(); + assert_eq!(serde_json::from_value::(json.clone()).unwrap(), original); + let mut missing = json.clone(); + missing["ports"][0].as_object_mut().unwrap().remove("exposure"); + assert!(serde_json::from_value::(missing).is_err()); + let mut injected = json; + injected + .as_object_mut() + .unwrap() + .insert("privileged".into(), true.into()); + assert!(serde_json::from_value::(injected).is_err()); + let keep: ContainerAction = serde_json::from_str(r#"{"action":"keep"}"#).unwrap(); + assert_eq!(keep, ContainerAction::Keep); + let remove: ContainerAction = serde_json::from_str(r#"{"action":"remove"}"#).unwrap(); + assert_eq!(remove, ContainerAction::Remove); + assert!(serde_json::from_str::(r#"{"action":"remove","value":{}}"#).is_err()); +} + +#[test] +fn concrete_container_actions_preserve_wire_tags() { + let container = spec(); + assert_eq!(bsatn::to_vec(&ContainerAction::Keep).unwrap(), [0]); + assert_eq!(bsatn::to_vec(&ContainerAction::Remove).unwrap(), [2]); + let mut expected = vec![1]; + expected.extend(bsatn::to_vec(&container).unwrap()); + assert_eq!(bsatn::to_vec(&ContainerAction::Set(container)).unwrap(), expected); +} diff --git a/crates/lib/src/container_environment.rs b/crates/lib/src/container_environment.rs new file mode 100644 index 00000000000..f2ff648a45e --- /dev/null +++ b/crates/lib/src/container_environment.rs @@ -0,0 +1,40 @@ +//! Immutable application environment identity, independent of expiring launch credentials. +//! +//! These values describe a request. They do not authenticate a client or grant +//! access to environment values. Only the trusted host/control protocol may +//! capture, read, or close a snapshot. + +use crate::{Hash, Identity, SpacetimeType, Uuid}; + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct EnvironmentSnapshotScope { + pub cluster: String, + pub database_id: u64, + pub database_identity: Identity, + pub node_id: u64, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub node_incarnation: Uuid, + pub generation: u64, + pub deployment_revision: Hash, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub start_request: Uuid, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub env_generation: Uuid, + /// The complete sorted, unique key list of the committed container declaration. + pub env_keys: Vec, +} + +/// Stable identity of a committed capture. No secret values or value hashes. +/// A durability barrier belongs to each proof, not to this immutable receipt. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct EnvironmentSnapshotReceipt { + pub scope: EnvironmentSnapshotScope, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub capture_receipt: Uuid, +} diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 47a4281e86a..17f3e377c74 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -95,6 +95,10 @@ pub enum RawModuleDefV10Section { /// HTTP route definitions. HttpRoutes(Vec), + + /// Module bindings capabilities, independent of function visibility. + /// Older hosts reject this section instead of silently ignoring its requirements. + Capabilities(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -317,6 +321,9 @@ pub struct RawReducerDefV10 { } /// The visibility of a function (reducer or procedure). +/// +/// New variants MUST be appended to preserve existing BSATN tags. Older hosts +/// reject unknown tags, so new restrictions cannot be silently discarded. #[derive(Debug, Copy, Clone, SpacetimeType)] #[sats(crate = crate)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] @@ -326,11 +333,31 @@ pub enum FunctionVisibility { /// Still callable by the module owner, collaborators, /// and internal module code. /// - /// Enabled for lifecycle reducers and scheduled functions by default. + /// The default for scheduled functions. Older lifecycle definitions also use + /// this tag; lifecycle assignments always enforce host-event-only invocation. Private, - /// Callable from client code. + /// Callable from client code, with the historical contextual defaults. + /// Scheduled functions become Private; lifecycle reducers remain host event handlers. ClientCallable, + + /// Callable only by a host-verified internal invocation. + Internal, + + /// Explicitly callable from client code, including when scheduled. + /// This separate tag preserves the meaning of existing ClientCallable definitions. + ExplicitClientCallable, +} + +impl FunctionVisibility { + /// Encode a source declaration without changing historical contextual defaults. + pub fn from_declaration(declared: Option, default: Self) -> Self { + match declared { + Some(Self::ClientCallable | Self::ExplicitClientCallable) => Self::ExplicitClientCallable, + Some(visibility) => visibility, + None => default, + } + } } /// A schedule definition. @@ -1027,10 +1054,20 @@ impl RawModuleDefV10Builder { /// This is because `SpacetimeType` is not implemented for `ReducerContext`, /// so it can never act like an ordinary argument.) pub fn add_reducer(&mut self, source_name: impl Into, params: ProductType) { + self.add_reducer_with_visibility(source_name, params, None); + } + + /// Add a reducer with an optional explicit visibility declaration. + pub fn add_reducer_with_visibility( + &mut self, + source_name: impl Into, + params: ProductType, + visibility: Option, + ) { self.reducers_mut().push(RawReducerDefV10 { source_name: source_name.into(), params, - visibility: FunctionVisibility::ClientCallable, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::ClientCallable), ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }); @@ -1051,12 +1088,23 @@ impl RawModuleDefV10Builder { source_name: impl Into, params: ProductType, return_type: AlgebraicType, + ) { + self.add_procedure_with_visibility(source_name, params, return_type, None); + } + + /// Add a procedure with an optional explicit visibility declaration. + pub fn add_procedure_with_visibility( + &mut self, + source_name: impl Into, + params: ProductType, + return_type: AlgebraicType, + visibility: Option, ) { self.procedures_mut().push(RawProcedureDefV10 { source_name: source_name.into(), params, return_type, - visibility: FunctionVisibility::ClientCallable, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::ClientCallable), }) } @@ -1088,6 +1136,19 @@ impl RawModuleDefV10Builder { lifecycle_spec: Lifecycle, function_name: impl Into, params: ProductType, + ) { + self.add_lifecycle_reducer_with_visibility(lifecycle_spec, function_name, params, None); + } + + /// Add a lifecycle reducer with an optional visibility declaration. + /// Source bindings must reject explicit Private or public lifecycle annotations. + /// The raw Private tag remains accepted for compatibility with existing modules. + pub fn add_lifecycle_reducer_with_visibility( + &mut self, + lifecycle_spec: Lifecycle, + function_name: impl Into, + params: ProductType, + visibility: Option, ) { let function_name = function_name.into(); self.lifecycle_reducers_mut().push(RawLifeCycleReducerDefV10 { @@ -1098,7 +1159,7 @@ impl RawModuleDefV10Builder { self.reducers_mut().push(RawReducerDefV10 { source_name: function_name, params, - visibility: FunctionVisibility::Private, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::Private), ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }); @@ -1124,6 +1185,22 @@ impl RawModuleDefV10Builder { }); } + /// Declare a module bindings capability. + pub fn add_capability(&mut self, capability: impl Into) { + if let Some(RawModuleDefV10Section::Capabilities(names)) = self + .module + .sections + .iter_mut() + .find(|section| matches!(section, RawModuleDefV10Section::Capabilities(_))) + { + names.push(capability.into()); + } else { + self.module + .sections + .push(RawModuleDefV10Section::Capabilities(vec![capability.into()])); + } + } + /// Add a row-level security policy to the module. /// /// The `sql` expression should be a valid SQL expression that will be used to filter rows. @@ -1406,3 +1483,141 @@ impl RawTableDefBuilderV10<'_> { .map(|i| ColId(i as u16)) } } + +#[cfg(test)] +mod compatibility_tests { + use super::*; + use crate::{bsatn, RawModuleDef}; + + // Frozen pre-extension wire types. Do not replace the visibility, function, + // or section definitions below with their current counterparts. + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacyVisibility { + Private, + ClientCallable, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyReducer { + source_name: RawIdentifier, + params: ProductType, + visibility: LegacyVisibility, + ok_return_type: AlgebraicType, + err_return_type: AlgebraicType, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyProcedure { + source_name: RawIdentifier, + params: ProductType, + return_type: AlgebraicType, + visibility: LegacyVisibility, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacySection { + Typespace(Typespace), + Types(Vec), + Tables(Vec), + Reducers(Vec), + Procedures(Vec), + Views(Vec), + Schedules(Vec), + LifeCycleReducers(Vec), + RowLevelSecurity(Vec), + CaseConversionPolicy(CaseConversionPolicy), + ExplicitNames(ExplicitNames), + HttpHandlers(Vec), + HttpRoutes(Vec), + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyV10 { + sections: Vec, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacyModule { + V8BackCompat(crate::RawModuleDefV8), + V9(super::super::v9::RawModuleDefV9), + V10(LegacyV10), + } + + #[test] + fn existing_v10_wire_tags_and_function_products_are_unchanged() { + for (visibility, expected) in [ + (FunctionVisibility::Private, 0), + (FunctionVisibility::ClientCallable, 1), + (FunctionVisibility::Internal, 2), + (FunctionVisibility::ExplicitClientCallable, 3), + ] { + assert_eq!(bsatn::to_vec(&visibility).unwrap(), [expected]); + } + let legacy = LegacyModule::V10(LegacyV10 { + sections: vec![ + LegacySection::Reducers(vec![LegacyReducer { + source_name: "run".into(), + params: ProductType::unit(), + visibility: LegacyVisibility::ClientCallable, + ok_return_type: reducer_default_ok_return_type(), + err_return_type: reducer_default_err_return_type(), + }]), + LegacySection::Procedures(vec![LegacyProcedure { + source_name: "read".into(), + params: ProductType::unit(), + return_type: AlgebraicType::U64, + visibility: LegacyVisibility::Private, + }]), + ], + }); + let bytes = bsatn::to_vec(&legacy).unwrap(); + assert_eq!(bytes[0], 2); + let current: RawModuleDef = bsatn::from_slice(&bytes).unwrap(); + assert_eq!(bsatn::to_vec(¤t).unwrap(), bytes); + let frozen: LegacyModule = bsatn::from_slice(&bsatn::to_vec(¤t).unwrap()).unwrap(); + assert_eq!(bsatn::to_vec(&frozen).unwrap(), bytes); + + assert_eq!( + bsatn::to_vec(&RawModuleDefV10Section::HttpRoutes(vec![])).unwrap(), + [12, 0, 0, 0, 0] + ); + assert_eq!( + bsatn::to_vec(&RawModuleDefV10Section::Capabilities(vec![])).unwrap(), + [13, 0, 0, 0, 0] + ); + } + + #[test] + fn older_hosts_reject_new_visibility_and_capabilities() { + for visibility in [FunctionVisibility::Internal, FunctionVisibility::ExplicitClientCallable] { + for procedure in [false, true] { + let mut builder = RawModuleDefV10Builder::new(); + if procedure { + builder.add_procedure_with_visibility( + "run", + ProductType::unit(), + AlgebraicType::unit(), + Some(visibility), + ); + } else { + builder.add_reducer_with_visibility("run", ProductType::unit(), Some(visibility)); + } + let bytes = bsatn::to_vec(&RawModuleDef::V10(builder.finish())).unwrap(); + assert_eq!(bytes[0], 2); + assert!(bsatn::from_slice::(&bytes).is_err()); + assert!(bsatn::from_slice::(&bytes).is_ok()); + } + } + let mut builder = RawModuleDefV10Builder::new(); + builder.add_capability("hosted_auth_v1"); + let bytes = bsatn::to_vec(&RawModuleDef::V10(builder.finish())).unwrap(); + assert!(bsatn::from_slice::(&bytes).is_err()); + assert!(bsatn::from_slice::(&bytes).is_ok()); + } +} diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs new file mode 100644 index 00000000000..c1fc5e211da --- /dev/null +++ b/crates/lib/src/deployment.rs @@ -0,0 +1,256 @@ +//! Normalized deployment protocol shared by publication coordinators and hosts. +//! +//! Module artifacts and OCI objects are uploaded separately. A deployment is +//! immutable configuration, never a place for runtime credentials or env values. + +use crate::container::{ContainerAction, ContainerSpec, ContainerSpecLimits, ContainerValidationError}; +use crate::{bsatn, hash_bytes, Hash, SpacetimeType, Uuid}; + +#[cfg(feature = "serde")] +pub mod api; +pub mod manifest; + +pub const PUBLISH_PROTOCOL_VERSION: u32 = 1; +pub const SYSTEM_EMPTY_MODULE_VERSION: u32 = 1; +/// Immutable built-in program, also used by clients for authorized migration +/// preflight. Replacing these bytes requires a new system-module version. +pub const SYSTEM_EMPTY_MODULE_V1_BYTES: &[u8] = include_bytes!("deployment/system_empty_v1.wasm"); +/// Immutable Keccak-256 program identity of the version-1 bundled empty Wasm +/// module. Control can verify initial program bytes without linking the host. +pub const SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH: Hash = Hash::from_byte_array([ + 0x08, 0x36, 0x50, 0x97, 0xd1, 0xef, 0x20, 0x26, 0x53, 0x61, 0x5f, 0x02, 0xcc, 0x46, 0xbe, 0x16, 0x08, 0x80, 0x44, + 0xc6, 0xb9, 0xcd, 0x96, 0x60, 0xe3, 0xa0, 0xf3, 0x36, 0xe4, 0x79, 0x18, 0xf0, +]); +/// SHA-256 descriptor of the same immutable bundled version-1 bytes. Clients +/// can name SystemEmpty in a prepared manifest without linking the host. +pub const SYSTEM_EMPTY_MODULE_V1_ARTIFACT: manifest::ModuleArtifact = manifest::ModuleArtifact { + digest: crate::container::OciDigest::sha256([ + 0x90, 0x37, 0x89, 0x67, 0xf4, 0xf5, 0xdb, 0x99, 0x37, 0x30, 0xe3, 0x67, 0x11, 0x67, 0x1a, 0x94, 0xdc, 0x5e, + 0xc6, 0x36, 0x75, 0xf6, 0x83, 0x60, 0xa2, 0x1d, 0xd1, 0x52, 0x4b, 0xcd, 0x7a, 0xa3, + ]), + size_bytes: 250, +}; +pub const MAX_DEPLOYMENT_BYTES: usize = 256 * 1024; +pub const PUBLISH_RETRY_WINDOW_MS: u64 = 7 * 24 * 60 * 60 * 1000; +pub const MAX_OPERATION_CLOCK_SKEW_MS: u64 = 5 * 60 * 1000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum UserModuleKind { + Wasm, + Js, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct UserModule { + pub kind: UserModuleKind, + /// The existing module program hash, not an OCI object digest. + pub program_hash: Hash, +} + +/// Explicit module replacement/removal, with its own exported schema name. +#[derive(Clone, Debug, Default, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ModuleAction { + #[default] + Keep, + Set(UserModule), + Remove, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "kind", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ModuleComponent { + SystemEmpty(u32), + User(UserModule), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct DeploymentSpecV1 { + pub module: ModuleComponent, + pub container: Option, +} + +/// Persist the discriminant along with the payload. Unknown encodings fail to +/// decode; a restore must never fall back to an empty or older deployment. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde( + tag = "version", + content = "deployment", + rename_all = "snake_case", + deny_unknown_fields + ) +)] +pub enum DeploymentSpec { + V1(DeploymentSpecV1), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct PublishEnvelope { + pub version: u32, + #[cfg_attr(feature = "serde", serde(with = "uuid_json"))] + pub operation_id: Uuid, + /// Exact compare-and-set precondition. None means no deployment record has + /// been installed yet, rather than permission to overwrite any revision. + pub expected_revision: Option, + #[cfg_attr(feature = "serde", serde(default))] + pub module_action: ModuleAction, + #[cfg_attr(feature = "serde", serde(default))] + pub container_action: ContainerAction, +} + +#[derive(Debug, thiserror::Error)] +pub enum DeploymentValidationError { + #[error("unsupported publish protocol version")] + UnsupportedVersion, + #[error("unsupported platform empty-module version")] + UnsupportedEmptyModule, + #[error("operation_id must be a version 7 UUID")] + InvalidOperationId, + #[error("publication operation has expired")] + ExpiredOperation, + #[error("publication operation timestamp is too far in the future")] + FutureOperation, + #[error("deployment exceeds the protocol size limit")] + TooLarge, + #[error("deployment encoding is invalid or unsupported")] + InvalidEncoding, + #[error(transparent)] + Container(#[from] ContainerValidationError), +} + +impl DeploymentSpec { + pub fn current(&self) -> &DeploymentSpecV1 { + match self { + Self::V1(spec) => spec, + } + } + + pub fn normalize(self, limits: &ContainerSpecLimits) -> Result { + let Self::V1(mut spec) = self; + if let ModuleComponent::SystemEmpty(version) = spec.module + && version != SYSTEM_EMPTY_MODULE_VERSION + { + return Err(DeploymentValidationError::UnsupportedEmptyModule); + } + spec.container = spec.container.map(|spec| spec.normalize(limits)).transpose()?; + let spec = Self::V1(spec); + spec.encode()?; + Ok(spec) + } + + pub fn encode(&self) -> Result, DeploymentValidationError> { + let bytes = bsatn::to_vec(self).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + Ok(bytes.into()) + } + + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + bsatn::from_slice(bytes).map_err(|_| DeploymentValidationError::InvalidEncoding) + } + + /// Call on the result of normalize. Unlike a request fingerprint, this is + /// independent of operation ID, publisher, and mutable execution status. + pub fn revision(&self) -> Result { + let mut bytes = b"spacetimedb/deployment\0".to_vec(); + bytes.extend_from_slice(&self.encode()?); + Ok(hash_bytes(bytes)) + } +} + +impl PublishEnvelope { + pub fn resolve( + &self, + previous: Option<&DeploymentSpec>, + limits: &ContainerSpecLimits, + ) -> Result { + if self.version != PUBLISH_PROTOCOL_VERSION { + return Err(DeploymentValidationError::UnsupportedVersion); + } + use spacetimedb_sats::uuid::Version; + if !matches!(self.operation_id.get_version(), Some(Version::V7)) { + return Err(DeploymentValidationError::InvalidOperationId); + } + let prior = previous.map(DeploymentSpec::current); + let module = match &self.module_action { + ModuleAction::Keep => prior + .map(|p| p.module.clone()) + .unwrap_or(ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION)), + ModuleAction::Set(module) => ModuleComponent::User(module.clone()), + ModuleAction::Remove => ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + }; + let container = match &self.container_action { + ContainerAction::Keep => prior.and_then(|p| p.container.clone()), + ContainerAction::Set(container) => Some(container.clone()), + ContainerAction::Remove => None, + }; + DeploymentSpec::V1(DeploymentSpecV1 { module, container }).normalize(limits) + } + + pub fn requires_container_permission(&self) -> bool { + matches!(self.container_action, ContainerAction::Set(_)) + } +} + +/// UUIDv7 embeds its creation millisecond. This makes an expired retry +/// distinguishable from a new request even after its ledger row is collected. +/// A fresh UUID with a changed timestamp is a different operation. +pub fn operation_expiry_ms(id: Uuid, now_ms: u64) -> Result { + if id.get_version() != Some(spacetimedb_sats::uuid::Version::V7) { + return Err(DeploymentValidationError::InvalidOperationId); + } + let created_ms = (id.as_u128() >> 80) as u64; + if created_ms > now_ms.saturating_add(MAX_OPERATION_CLOCK_SKEW_MS) { + return Err(DeploymentValidationError::FutureOperation); + } + let expires_ms = created_ms + PUBLISH_RETRY_WINDOW_MS; + if now_ms >= expires_ms { + return Err(DeploymentValidationError::ExpiredOperation); + } + Ok(expires_ms) +} + +#[cfg(feature = "serde")] +pub mod uuid_json { + use super::Uuid; + pub fn serialize(id: &Uuid, serializer: S) -> Result { + serializer.collect_str(id) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + Uuid::parse_str(&value).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/deployment/api.rs b/crates/lib/src/deployment/api.rs new file mode 100644 index 00000000000..c1adcebef6e --- /dev/null +++ b/crates/lib/src/deployment/api.rs @@ -0,0 +1,117 @@ +//! HTTP publication messages shared by the CLI, dashboard and Cloud. Artifacts +//! are uploaded separately; no message carries credentials or environment values. + +use super::{manifest::PreparedDeploymentManifest, uuid_json, DeploymentSpec, PUBLISH_PROTOCOL_VERSION}; +use crate::{container::OciDigest, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactReference { + pub digest: OciDigest, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishRequest { + pub manifest: PreparedDeploymentManifest, + /// Must match the server-generated reservation when creating a database. + pub creation: Option, + /// Original uploaded OCI index or executable manifest. Required for Set. + /// Keep uses the prior retained executable manifest; Remove has no image. + pub image_source: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CreationOptions { + pub parent: Option, + pub organization: Option, + pub num_replicas: Option, + #[serde(default = "default_anti_affinity")] + pub enforce_anti_affinity: bool, +} +fn default_anti_affinity() -> bool { + true +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReserveDatabaseRequest { + pub version: u32, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub options: CreationOptions, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DatabaseReservation { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub expires_at: String, + pub staging_open: bool, + pub artifact_endpoint: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PublicationPhase { + Prepared, + Quiescing, + Committed, + Activating, + Complete, + AbortedBeforeCommit, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicationStatus { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub phase: PublicationPhase, + pub expected_revision: Option, + pub proposed_revision: Hash, + pub error: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeploymentStatus { + pub database_identity: Identity, + /// None means this database has not yet used managed publication. + pub revision: Option, + pub deployment: DeploymentSpec, + /// Lets Keep select exactly the currently installed program bytes. + pub module_artifact: ArtifactReference, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishPermission { + pub identity: Identity, + pub can_publish: bool, + /// Decimal string preserves all u64 revision values in browser clients. + pub source_revision: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicationCapabilities { + pub version: u32, + pub enabled: bool, + pub artifact_endpoint: Option, +} +impl PublicationCapabilities { + pub fn disabled() -> Self { + Self { + version: PUBLISH_PROTOCOL_VERSION, + enabled: false, + artifact_endpoint: None, + } + } +} diff --git a/crates/lib/src/deployment/manifest.rs b/crates/lib/src/deployment/manifest.rs new file mode 100644 index 00000000000..0727643b0c1 --- /dev/null +++ b/crates/lib/src/deployment/manifest.rs @@ -0,0 +1,172 @@ +//! Immutable publication recovery inputs. The manifest's SHA-256 artifact +//! digest binds migration intent as well as the effective deployment. Its +//! digest differs from the deployment revision, which excludes migration policy. + +use super::{DeploymentSpec, DeploymentValidationError, PublishEnvelope, MAX_DEPLOYMENT_BYTES}; +use crate::container::{ContainerSpecLimits, OciDigest}; +use crate::{bsatn, Hash, SpacetimeType}; + +pub const MAX_MODULE_ARTIFACT_BYTES: u64 = 32 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ModuleArtifact { + /// SHA-256 of the complete stored bytes, distinct from the module's Keccak hash. + pub digest: OciDigest, + pub size_bytes: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "policy", content = "token", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum PreparedMigrationPolicy { + Compatible, + /// The existing migration token binds database Identity and old/new module + /// hashes. Recovery must retain the originally acknowledged policy. + BreakClients(Hash), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct PreparedDeploymentManifestV1 { + pub envelope: PublishEnvelope, + pub deployment: DeploymentSpec, + /// Always retained, including for the bundled empty module. Genesis and + /// later recovery must select exactly the admitted program bytes. + pub module_artifact: ModuleArtifact, + pub migration_policy: PreparedMigrationPolicy, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde( + tag = "version", + content = "manifest", + rename_all = "snake_case", + deny_unknown_fields + ) +)] +pub enum PreparedDeploymentManifest { + V1(PreparedDeploymentManifestV1), +} + +impl PreparedDeploymentManifest { + pub fn current(&self) -> &PreparedDeploymentManifestV1 { + match self { + Self::V1(manifest) => manifest, + } + } + + /// Validate before retaining the artifact. This checks the encoding and + /// metadata; the artifact service verifies SHA-256/length and the host + /// validates the selected program, capabilities and actual migration. + pub fn validate(&self, limits: &ContainerSpecLimits) -> Result<(), DeploymentValidationError> { + let manifest = self.current(); + if manifest.module_artifact.size_bytes == 0 || manifest.module_artifact.size_bytes > MAX_MODULE_ARTIFACT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + if manifest.deployment.clone().normalize(limits)? != manifest.deployment { + return Err(DeploymentValidationError::InvalidEncoding); + } + // Using the prepared components as the Keep baseline verifies every + // explicit Set/Remove action without inventing a prior deployment. + // The host separately checks the real prior state under its fence. + if manifest.envelope.resolve(Some(&manifest.deployment), limits)? != manifest.deployment { + return Err(DeploymentValidationError::InvalidEncoding); + } + self.encode()?; + Ok(()) + } + + pub fn encode(&self) -> Result, DeploymentValidationError> { + let bytes = bsatn::to_vec(self).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + Ok(bytes.into()) + } + + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + let manifest: Self = bsatn::from_slice(bytes).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + // Reject trailing or noncanonical bytes even if the decoder accepts + // them, so every retained descriptor names one unambiguous manifest. + if manifest.encode()?.as_ref() != bytes { + return Err(DeploymentValidationError::InvalidEncoding); + } + Ok(manifest) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::deployment::{DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; + + fn manifest() -> PreparedDeploymentManifest { + PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope: PublishEnvelope { + version: super::super::PUBLISH_PROTOCOL_VERSION, + operation_id: crate::Uuid::from_u128(0x01991ec4000070008000000000000001), + expected_revision: None, + module_action: super::super::ModuleAction::Keep, + container_action: crate::container::ContainerAction::Keep, + }, + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: None, + }), + module_artifact: ModuleArtifact { + digest: OciDigest::sha256([17; 32]), + size_bytes: 250, + }, + migration_policy: PreparedMigrationPolicy::Compatible, + }) + } + + #[test] + fn retained_manifest_preserves_migration_intent_without_changing_revision() { + let first = manifest(); + let mut acknowledged = first.clone(); + let PreparedDeploymentManifest::V1(value) = &mut acknowledged; + value.migration_policy = PreparedMigrationPolicy::BreakClients(Hash::from_byte_array([32; 32])); + assert_eq!( + first.current().deployment.revision().unwrap(), + acknowledged.current().deployment.revision().unwrap() + ); + let encoded = acknowledged.encode().unwrap(); + assert_ne!(first.encode().unwrap(), encoded); + assert_eq!(PreparedDeploymentManifest::decode(&encoded).unwrap(), acknowledged); + acknowledged.validate(&Default::default()).unwrap(); + } + + #[test] + fn retained_manifest_rejects_unknown_encoding_and_invalid_module_bounds() { + let mut value = manifest(); + let mut trailing = value.encode().unwrap().to_vec(); + trailing.push(0); + assert!(PreparedDeploymentManifest::decode(&trailing).is_err()); + let mut unknown = value.encode().unwrap().to_vec(); + unknown[0] = 1; + assert!(PreparedDeploymentManifest::decode(&unknown).is_err()); + let PreparedDeploymentManifest::V1(inner) = &mut value; + inner.module_artifact.size_bytes = 0; + assert!(value.validate(&Default::default()).is_err()); + let PreparedDeploymentManifest::V1(inner) = &mut value; + inner.module_artifact.size_bytes = MAX_MODULE_ARTIFACT_BYTES + 1; + assert!(value.validate(&Default::default()).is_err()); + } +} diff --git a/crates/lib/src/deployment/system_empty_v1.wasm b/crates/lib/src/deployment/system_empty_v1.wasm new file mode 100644 index 00000000000..811462677fb Binary files /dev/null and b/crates/lib/src/deployment/system_empty_v1.wasm differ diff --git a/crates/lib/src/deployment/tests.rs b/crates/lib/src/deployment/tests.rs new file mode 100644 index 00000000000..e720783d8aa --- /dev/null +++ b/crates/lib/src/deployment/tests.rs @@ -0,0 +1,128 @@ +use super::*; + +fn operation_id() -> Uuid { + Uuid::parse_str("01991ec4-0000-7000-8000-000000000001").unwrap() +} + +fn request(module_action: ModuleAction) -> PublishEnvelope { + PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: operation_id(), + expected_revision: None, + module_action, + container_action: ContainerAction::Keep, + } +} + +#[test] +fn component_actions_preserve_or_explicitly_remove_the_module() { + let limits = ContainerSpecLimits::default(); + let user = UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(b"valid module artifact"), + }; + let set = request(ModuleAction::Set(user.clone())).resolve(None, &limits).unwrap(); + assert_eq!(set.current().module, ModuleComponent::User(user)); + let keep = request(ModuleAction::Keep).resolve(Some(&set), &limits).unwrap(); + assert_eq!(set, keep); + let remove = request(ModuleAction::Remove).resolve(Some(&set), &limits).unwrap(); + assert_eq!( + remove.current().module, + ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION) + ); + assert_ne!(remove.revision().unwrap(), keep.revision().unwrap()); +} + +#[test] +fn concrete_module_actions_preserve_wire_tags_and_export_distinct_names() { + let module = UserModule { + kind: UserModuleKind::Js, + program_hash: hash_bytes(b"module"), + }; + assert_eq!(bsatn::to_vec(&ModuleAction::Keep).unwrap(), [0]); + assert_eq!(bsatn::to_vec(&ModuleAction::Remove).unwrap(), [2]); + let mut expected = vec![1]; + expected.extend(bsatn::to_vec(&module).unwrap()); + assert_eq!(bsatn::to_vec(&ModuleAction::Set(module)).unwrap(), expected); + + use crate::db::raw_def::v10::{RawModuleDefV10Builder, RawModuleDefV10Section}; + let mut builder = RawModuleDefV10Builder::new(); + builder.add_type::(); + let raw = builder.finish(); + let names: Vec<_> = raw + .sections + .iter() + .filter_map(|section| match section { + RawModuleDefV10Section::Types(types) => Some(types), + _ => None, + }) + .flatten() + .map(|ty| &*ty.source_name.source_name) + .collect(); + let distinct: std::collections::BTreeSet<_> = names.iter().copied().collect(); + assert_eq!( + names.len(), + distinct.len(), + "publish envelope exports duplicate type names" + ); + assert!(distinct.contains("ModuleAction")); + assert!(distinct.contains("ContainerAction")); +} + +#[test] +fn unknown_deployments_do_not_fall_back_to_an_empty_module() { + assert!(DeploymentSpec::decode(&[255]).is_err()); + let mut envelope = request(ModuleAction::Keep); + envelope.version += 1; + assert!(matches!( + envelope.resolve(None, &ContainerSpecLimits::default()), + Err(DeploymentValidationError::UnsupportedVersion) + )); + let spec = DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(9000), + container: None, + }); + assert!(matches!( + spec.normalize(&ContainerSpecLimits::default()), + Err(DeploymentValidationError::UnsupportedEmptyModule) + )); +} + +#[test] +fn operation_age_is_enforced_even_when_no_ledger_row_remains() { + let id = operation_id(); + let created_ms = (id.as_u128() >> 80) as u64; + assert_eq!( + operation_expiry_ms(id, created_ms).unwrap(), + created_ms + PUBLISH_RETRY_WINDOW_MS + ); + assert!(matches!( + operation_expiry_ms(id, created_ms + PUBLISH_RETRY_WINDOW_MS), + Err(DeploymentValidationError::ExpiredOperation) + )); + assert!(matches!( + operation_expiry_ms(id, created_ms - MAX_OPERATION_CLOCK_SKEW_MS - 1), + Err(DeploymentValidationError::FutureOperation) + )); + assert!(matches!( + operation_expiry_ms(Uuid::NIL, created_ms), + Err(DeploymentValidationError::InvalidOperationId) + )); +} + +#[cfg(feature = "serde")] +#[test] +fn uuid_json_is_lossless_and_omission_means_keep() { + let envelope = request(ModuleAction::Keep); + let mut json = serde_json::to_value(&envelope).unwrap(); + assert_eq!(json["operation_id"], operation_id().to_string()); + json.as_object_mut().unwrap().remove("module_action"); + json.as_object_mut().unwrap().remove("container_action"); + let parsed: PublishEnvelope = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(parsed, envelope); + assert!(!parsed.requires_container_permission()); + json["publisher_identity"] = "untrusted owner override".into(); + assert!(serde_json::from_value::(json).is_err()); + let binary = bsatn::to_vec(&envelope).unwrap(); + assert_eq!(bsatn::from_slice::(&binary).unwrap(), envelope); +} diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs new file mode 100644 index 00000000000..4963a6f8718 --- /dev/null +++ b/crates/lib/src/environment.rs @@ -0,0 +1,53 @@ +//! Limits shared by the database environment store and its clients. + +pub const MAX_ENV_KEY_BYTES: usize = 256; +pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; +pub const MAX_ENV_VARS: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum EnvironmentValidationError { + #[error("invalid POSIX environment variable name (maximum 256 bytes)")] + InvalidKey, + #[error("environment value exceeds 8192 UTF-8 bytes")] + ValueTooLarge, + #[error("environment store exceeds 256 variables")] + TooManyVariables, +} + +pub fn validate_key(key: &str) -> Result<(), EnvironmentValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > MAX_ENV_KEY_BYTES + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(EnvironmentValidationError::InvalidKey); + } + Ok(()) +} + +/// NUL is representable in the database. Container launch separately rejects it. +pub fn validate_value(value: &str) -> Result<(), EnvironmentValidationError> { + if value.len() > MAX_ENV_VALUE_BYTES { + return Err(EnvironmentValidationError::ValueTooLarge); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_utf8_byte_limits_and_posix_keys_without_container_policy() { + for key in ["", "1FIRST", "A=B", "A\0B", "é", "A-B"] { + assert_eq!(validate_key(key), Err(EnvironmentValidationError::InvalidKey)); + } + assert!(validate_key(&"A".repeat(256)).is_ok()); + assert!(validate_key(&"A".repeat(257)).is_err()); + assert!(validate_key("SPACETIMEDB_USER_DATA").is_ok()); + assert!(validate_value("\0").is_ok()); + assert!(validate_value(&"é".repeat(4096)).is_ok()); + assert!(validate_value(&"é".repeat(4097)).is_err()); + } +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 2e8b9c08336..ae5d93e5e26 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -11,8 +11,12 @@ use std::any::TypeId; use std::collections::{btree_map, BTreeMap}; pub mod connection_id; +pub mod container; +pub mod container_environment; pub mod db; +pub mod deployment; mod direct_index_key; +pub mod environment; pub mod error; mod filterable_value; pub mod http; diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml new file mode 100644 index 00000000000..f32bc83ab59 --- /dev/null +++ b/crates/oci/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "spacetimedb-oci" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license-file = "../../LICENSE.txt" +description = "Bounded OCI image validation for SpacetimeDB container publishing" + +[dependencies] +spacetimedb-lib = { workspace = true, features = ["serde"] } +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2 = "0.10" +flate2.workspace = true +tar.workspace = true +zstd = "0.13" + +[lints] +workspace = true diff --git a/crates/oci/src/layers.rs b/crates/oci/src/layers.rs new file mode 100644 index 00000000000..c7d99cb39de --- /dev/null +++ b/crates/oci/src/layers.rs @@ -0,0 +1,373 @@ +//! Stream-verify layer expansion without extracting anything onto the host. +//! Resource admission uses measured bytes and bounded metadata, never compressed +//! sizes alone. The runtime must also enforce a dedicated finite cache filesystem. + +use crate::{Descriptor, OciDigest}; +use anyhow::{bail, ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + io::{self, Read}, +}; + +#[derive(Clone, Copy, Debug)] +pub struct LayerLimits { + pub max_uncompressed_bytes: u64, + pub max_regular_file_bytes: u64, + pub max_entries: u64, + pub max_metadata_bytes: usize, + pub max_path_bytes: usize, + pub zstd_window_log_max: u32, +} +impl Default for LayerLimits { + fn default() -> Self { + Self { + max_uncompressed_bytes: 128 * 1024 * 1024 * 1024, + max_regular_file_bytes: 64 * 1024 * 1024 * 1024, + max_entries: 1_000_000, + max_metadata_bytes: 64 * 1024, + max_path_bytes: 4096, + zstd_window_log_max: 27, + } + } +} + +#[derive(Clone, Copy, Debug, Serialize)] +pub struct VerifiedLayerSize { + pub uncompressed_tar_bytes: u64, + pub entries: u64, + pub regular_file_bytes: u64, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct VerifiedImageSize { + pub compressed_bytes: u64, + pub uncompressed_tar_bytes: u64, + pub entries: u64, + pub cache_reservation_bytes: u64, +} +impl VerifiedImageSize { + /// Conservative cache accounting, supplemented by the backend's hard + /// filesystem bound. Includes temporary compressed/unpacked copies and + /// per-entry/per-layer filesystem metadata. The trusted artifact service + /// supplies these verified measurements, never a publisher JSON field. + pub fn from_layers(compressed_bytes: u64, layers: &[VerifiedLayerSize]) -> Result { + let mut tar = 0u64; + let mut entries = 0u64; + for layer in layers { + tar = tar + .checked_add(layer.uncompressed_tar_bytes) + .context("expanded image size overflow")?; + entries = entries + .checked_add(layer.entries) + .context("image entry count overflow")?; + } + let reservation = compressed_bytes + .checked_mul(2) + .and_then(|n| tar.checked_mul(3).and_then(|v| n.checked_add(v))) + .and_then(|n| entries.checked_mul(64 * 1024).and_then(|v| n.checked_add(v))) + .and_then(|n| { + (layers.len() as u64) + .checked_mul(16 * 1024 * 1024) + .and_then(|v| n.checked_add(v)) + }) + .context("image cache reservation overflow")?; + Ok(Self { + compressed_bytes, + uncompressed_tar_bytes: tar, + entries, + cache_reservation_bytes: reservation, + }) + } +} + +struct HashBounded { + inner: R, + hash: Sha256, + count: u64, + limit: u64, +} +impl Read for HashBounded { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + if self.count == self.limit { + let mut extra = [0u8; 1]; + if self.inner.read(&mut extra)? == 0 { + return Ok(0); + } + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "layer exceeds configured byte bound", + )); + } + let maximum = buf.len().min((self.limit - self.count).min(usize::MAX as u64) as usize); + let count = self.inner.read(&mut buf[..maximum])?; + self.count += count as u64; + self.hash.update(&buf[..count]); + Ok(count) + } +} + +pub fn verify_layer( + reader: impl Read, + descriptor: &Descriptor, + diff_id: OciDigest, + limits: LayerLimits, +) -> Result { + verify_layer_with_check(reader, descriptor, diff_id, limits, || Ok(())) +} + +/// Check cancellation/deadline between both compressed and expanded reads. +/// The caller retains its worker admission until this synchronous operation ends. +pub fn verify_layer_with_check( + reader: impl Read, + descriptor: &Descriptor, + diff_id: OciDigest, + limits: LayerLimits, + check: impl Fn() -> io::Result<()> + Copy, +) -> Result { + struct Checked { + reader: R, + check: F, + } + impl io::Result<()>> Read for Checked { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + (self.check)()?; + self.reader.read(buf) + } + } + check()?; + ensure!( + descriptor.size > 0 && descriptor.size <= crate::MAX_IMAGE_BYTES, + "invalid compressed layer size" + ); + ensure!( + descriptor.urls.is_empty() && descriptor.data.is_none(), + "external layer sources are unsupported" + ); + let mut compressed = HashBounded { + inner: Checked { reader, check }, + hash: Sha256::new(), + count: 0, + limit: descriptor.size, + }; + let decoder: Box = match descriptor.media_type.as_str() { + "application/vnd.oci.image.layer.v1.tar" | "application/vnd.docker.image.rootfs.diff.tar" => { + Box::new(&mut compressed) + } + "application/vnd.oci.image.layer.v1.tar+gzip" | "application/vnd.docker.image.rootfs.diff.tar.gzip" => { + Box::new(flate2::read::MultiGzDecoder::new(&mut compressed)) + } + "application/vnd.oci.image.layer.v1.tar+zstd" => { + let mut decoder = zstd::stream::read::Decoder::new(&mut compressed)?; + decoder.window_log_max(limits.zstd_window_log_max)?; + Box::new(decoder) + } + _ => bail!("unsupported or foreign layer media type"), + }; + let mut expanded = HashBounded { + inner: Checked { reader: decoder, check }, + hash: Sha256::new(), + count: 0, + limit: limits.max_uncompressed_bytes, + }; + let (entries, regular_file_bytes) = scan_tar(&mut expanded, limits)?; + let uncompressed_tar_bytes = expanded.count; + ensure!( + OciDigest::sha256(expanded.hash.clone().finalize().into()) == diff_id, + "layer uncompressed SHA-256 does not match rootfs diff ID" + ); + drop(expanded); + ensure!( + compressed.count == descriptor.size + && OciDigest::sha256(compressed.hash.finalize().into()) == descriptor.digest, + "compressed layer SHA-256 or length mismatch" + ); + Ok(VerifiedLayerSize { + uncompressed_tar_bytes, + entries, + regular_file_bytes, + }) +} + +fn scan_tar(reader: &mut impl Read, limits: LayerLimits) -> Result<(u64, u64)> { + let mut entries = 0u64; + let mut regular_bytes = 0u64; + let mut pax = BTreeMap::new(); + let mut global = BTreeMap::new(); + let mut long_path = None; + let mut long_link = None; + loop { + let mut block = [0u8; 512]; + let first = reader.read(&mut block[..1])?; + if first == 0 { + break; + } + reader.read_exact(&mut block[1..]).context("truncated TAR header")?; + if block.iter().all(|&b| b == 0) { + continue; + } + entries = entries.checked_add(1).context("TAR entry count overflow")?; + ensure!(entries <= limits.max_entries, "too many layer entries"); + let header = tar::Header::from_byte_slice(&block); + let checksum = block[..148] + .iter() + .chain(&block[156..]) + .map(|&b| u32::from(b)) + .sum::() + + 8 * 32; + ensure!(header.cksum()? == checksum, "invalid TAR header checksum"); + let kind = header.entry_type().as_byte(); + let header_size = header.entry_size()?; + if matches!(kind, b'x' | b'g' | b'L' | b'K') { + ensure!( + header_size <= limits.max_metadata_bytes as u64, + "TAR metadata entry is too large" + ); + let mut bytes = vec![0; header_size as usize]; + reader.read_exact(&mut bytes)?; + skip_padding(reader, header_size)?; + match kind { + b'x' => { + ensure!(pax.is_empty(), "duplicate local PAX metadata"); + pax = parse_pax(&bytes)?; + } + b'g' => { + let update = parse_pax(&bytes)?; + global.extend(update); + ensure!( + global.len() <= 128 + && global.iter().map(|(k, v)| k.len() + v.len()).sum::() + <= limits.max_metadata_bytes, + "global PAX metadata exceeds bound" + ); + } + b'L' => { + ensure!(long_path.is_none(), "duplicate GNU long path"); + long_path = Some(trim_nul(bytes)); + } + b'K' => { + ensure!(long_link.is_none(), "duplicate GNU long link"); + long_link = Some(trim_nul(bytes)); + } + _ => unreachable!(), + } + continue; + } + let property = |name: &str| pax.get(name).or_else(|| global.get(name)); + let size = property("size") + .map(|v| v.parse::()) + .transpose()? + .unwrap_or(header_size); + ensure!(size <= limits.max_regular_file_bytes, "layer file exceeds size bound"); + let raw_path = header.path_bytes(); + let path = property("path") + .map(String::as_bytes) + .or(long_path.as_deref()) + .unwrap_or(&raw_path); + validate_path(path, limits.max_path_bytes, kind == b'5')?; + match kind { + 0 | b'0' | b'7' => { + regular_bytes = regular_bytes.checked_add(size).context("layer file size overflow")?; + } + b'5' => ensure!(size == 0, "directory entry has data"), + b'1' | b'2' => { + ensure!(size == 0, "link entry has data"); + let raw_link = header.link_name_bytes(); + let link = property("linkpath") + .map(String::as_bytes) + .or(long_link.as_deref()) + .or(raw_link.as_deref()) + .context("link target is missing")?; + ensure!( + !link.is_empty() && link.len() <= limits.max_path_bytes && !link.contains(&0), + "invalid link target" + ); + if kind == b'1' { + validate_path(link, limits.max_path_bytes, false)?; + } + // Absolute symbolic links are normal inside a Linux image. + // Extraction remains the confined runtime unpacker's job. + } + _ => bail!("unsupported sparse, special-device, or unknown TAR entry type"), + } + skip_exact(reader, size)?; + skip_padding(reader, size)?; + pax.clear(); + long_path = None; + long_link = None; + } + ensure!( + pax.is_empty() && long_path.is_none() && long_link.is_none(), + "orphaned TAR extension metadata" + ); + Ok((entries, regular_bytes)) +} + +fn trim_nul(mut bytes: Vec) -> Vec { + while bytes.last() == Some(&0) { + bytes.pop(); + } + bytes +} +fn skip_exact(reader: &mut impl Read, mut bytes: u64) -> Result<()> { + let mut buffer = [0u8; 64 * 1024]; + while bytes > 0 { + let count = bytes.min(buffer.len() as u64) as usize; + reader.read_exact(&mut buffer[..count])?; + bytes -= count as u64; + } + Ok(()) +} +fn skip_padding(reader: &mut impl Read, size: u64) -> Result<()> { + skip_exact(reader, (512 - size % 512) % 512) +} +fn validate_path(path: &[u8], max: usize, root_directory: bool) -> Result<()> { + ensure!( + !path.is_empty() && path.len() <= max && !path.contains(&0) && !path.starts_with(b"/"), + "invalid layer entry path" + ); + ensure!( + !path.split(|&b| b == b'/').any(|p| p == b".."), + "layer entry path escapes image root" + ); + ensure!( + root_directory || path.split(|&b| b == b'/').any(|p| !p.is_empty() && p != b"."), + "invalid root file entry" + ); + Ok(()) +} +fn parse_pax(mut bytes: &[u8]) -> Result> { + let mut result = BTreeMap::new(); + while !bytes.is_empty() { + let space = bytes.iter().position(|&b| b == b' ').context("invalid PAX record")?; + ensure!(space > 0 && space <= 10, "invalid PAX length"); + let length = std::str::from_utf8(&bytes[..space])?.parse::()?; + ensure!( + length > space + 2 && length <= bytes.len() && bytes[length - 1] == b'\n', + "invalid PAX record length" + ); + let record = std::str::from_utf8(&bytes[space + 1..length - 1])?; + let (key, value) = record.split_once('=').context("invalid PAX property")?; + ensure!( + key.len() <= 256 + && (matches!( + key, + "path" | "linkpath" | "size" | "mtime" | "atime" | "ctime" | "uid" | "gid" | "uname" | "gname" + ) || key.starts_with("SCHILY.xattr.")), + "unsupported sparse or unknown PAX property" + ); + ensure!( + result.insert(key.to_owned(), value.to_owned()).is_none() && result.len() <= 128, + "duplicate or excessive PAX properties" + ); + bytes = &bytes[length..]; + } + Ok(result) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oci/src/layers/tests.rs b/crates/oci/src/layers/tests.rs new file mode 100644 index 00000000000..044ec7bb43e --- /dev/null +++ b/crates/oci/src/layers/tests.rs @@ -0,0 +1,130 @@ +use super::*; +use std::io::Write; + +fn archive() -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_ustar(); + header.set_path("usr/bin/main").unwrap(); + header.set_size(4); + header.set_mode(0o755); + header.set_cksum(); + builder.append(&header, &b"data"[..]).unwrap(); + builder.into_inner().unwrap() +} +fn descriptor(bytes: &[u8], media: &str) -> Descriptor { + Descriptor { + digest: crate::sha256(bytes), + size: bytes.len() as u64, + media_type: media.into(), + platform: None, + urls: vec![], + data: None, + artifact_type: None, + } +} +fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} +#[test] +fn verifies_plain_gzip_and_zstd_against_actual_expanded_bytes() { + let tar = archive(); + let diff = crate::sha256(&tar); + for (bytes, media) in [ + (tar.clone(), "application/vnd.oci.image.layer.v1.tar"), + (gzip(&tar), "application/vnd.oci.image.layer.v1.tar+gzip"), + ( + zstd::stream::encode_all(&tar[..], 1).unwrap(), + "application/vnd.oci.image.layer.v1.tar+zstd", + ), + ] { + let result = verify_layer(&bytes[..], &descriptor(&bytes, media), diff, LayerLimits::default()).unwrap(); + assert_eq!(result.uncompressed_tar_bytes, tar.len() as u64); + assert_eq!(result.entries, 1); + assert_eq!(result.regular_file_bytes, 4); + assert!( + VerifiedImageSize::from_layers(bytes.len() as u64, &[result]) + .unwrap() + .cache_reservation_bytes + > result.uncompressed_tar_bytes + ); + } +} +#[test] +fn decompression_bomb_is_stopped_before_declared_diff_id_can_be_trusted() { + let expanded = vec![0u8; 1024 * 1024]; + let compressed = gzip(&expanded); + let result = verify_layer( + &compressed[..], + &descriptor(&compressed, "application/vnd.oci.image.layer.v1.tar+gzip"), + crate::sha256(&expanded), + LayerLimits { + max_uncompressed_bytes: 4096, + ..LayerLimits::default() + }, + ); + assert!(result.unwrap_err().to_string().contains("byte bound")); +} +#[test] +fn catches_digest_mismatch_truncation_and_entry_limit() { + let bytes = archive(); + let descriptor = descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar"); + assert!(verify_layer(&bytes[..], &descriptor, crate::sha256(b"wrong"), LayerLimits::default()).is_err()); + assert!(verify_layer( + &bytes[..100], + &descriptor, + crate::sha256(&bytes), + LayerLimits::default() + ) + .is_err()); + assert!(verify_layer( + &bytes[..], + &descriptor, + crate::sha256(&bytes), + LayerLimits { + max_entries: 0, + ..LayerLimits::default() + } + ) + .is_err()); +} +#[test] +fn extension_size_is_bounded_before_allocating_and_sparse_metadata_is_rejected() { + let mut header = tar::Header::new_gnu(); + header.set_path("pax").unwrap(); + header.set_entry_type(tar::EntryType::XHeader); + header.set_size(1 << 30); + header.set_cksum(); + let bytes = header.as_bytes().to_vec(); + assert!(verify_layer( + &bytes[..], + &descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar"), + crate::sha256(&bytes), + LayerLimits::default() + ) + .unwrap_err() + .to_string() + .contains("metadata entry")); + assert!(parse_pax(b"25 GNU.sparse.size=123456\n").is_err()); + assert!(validate_path(b"safe/../../host", 4096, false).is_err()); + assert!(validate_path(b"/absolute", 4096, false).is_err()); +} +#[test] +fn counts_concatenated_gzip_members_and_checks_pax_sizes() { + let tar = archive(); + let mut bytes = gzip(&tar); + bytes.extend(gzip(&tar)); + let mut both = tar.clone(); + both.extend(&tar); + let result = verify_layer( + &bytes[..], + &descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar+gzip"), + crate::sha256(&both), + LayerLimits::default(), + ) + .unwrap(); + assert_eq!(result.entries, 2); + assert_eq!(parse_pax(b"10 size=4\n").unwrap()["size"], "4"); + assert!(parse_pax(b"99 size=4\n").is_err()); +} diff --git a/crates/oci/src/lib.rs b/crates/oci/src/lib.rs new file mode 100644 index 00000000000..aa1bc63f19b --- /dev/null +++ b/crates/oci/src/lib.rs @@ -0,0 +1,377 @@ +//! Validate immutable OCI objects before accepting a container deployment. +//! +//! Registry references and index annotations are discovery inputs. Only verified +//! object bytes and an exact selected platform establish the published image. + +pub mod layers; + +use anyhow::{bail, ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::container::{ImagePlatform, OciDigest, MAX_ARGV_ENTRIES, MAX_ENV_KEYS, MAX_EXEC_STRING_BYTES}; +use std::collections::BTreeMap; + +pub const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; +pub const OCI_INDEX: &str = "application/vnd.oci.image.index.v1+json"; +pub const OCI_CONFIG: &str = "application/vnd.oci.image.config.v1+json"; +pub const DOCKER_MANIFEST: &str = "application/vnd.docker.distribution.manifest.v2+json"; +pub const DOCKER_INDEX: &str = "application/vnd.docker.distribution.manifest.list.v2+json"; +pub const DOCKER_CONFIG: &str = "application/vnd.docker.container.image.v1+json"; +pub const MAX_MANIFEST_BYTES: usize = 4 * 1024 * 1024; +pub const MAX_CONFIG_BYTES: usize = 1024 * 1024; +pub const MAX_LAYERS: usize = 256; +pub const MAX_INDEX_ENTRIES: usize = 256; +pub const MAX_IMAGE_BYTES: u64 = 64 * 1024 * 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Descriptor { + pub media_type: String, + pub digest: OciDigest, + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub urls: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact_type: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Platform { + pub os: String, + pub architecture: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub variant: Option, + #[serde(default, rename = "os.version", skip_serializing_if = "Option::is_none")] + pub os_version: Option, + #[serde(default, rename = "os.features", skip_serializing_if = "Vec::is_empty")] + pub os_features: Vec, +} + +impl Platform { + fn matches(&self, requested: &ImagePlatform) -> bool { + self.os == requested.os + && self.architecture == requested.architecture + && self.os_version.as_deref().is_none_or(str::is_empty) + && self.os_features.is_empty() + && matches!( + (self.architecture.as_str(), self.variant.as_deref()), + (_, None | Some("")) | ("arm64", Some("v8")) + ) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Manifest { + pub schema_version: u32, + pub media_type: String, + pub config: Descriptor, + pub layers: Vec, + #[serde(default)] + artifact_type: Option, + #[serde(default)] + subject: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImageIndex { + pub schema_version: u32, + pub media_type: String, + pub manifests: Vec, +} + +pub fn sha256(bytes: &[u8]) -> OciDigest { + OciDigest::sha256(Sha256::digest(bytes).into()) +} + +pub fn verify_object(descriptor: &Descriptor, bytes: &[u8]) -> Result<()> { + ensure!( + descriptor.size == bytes.len() as u64, + "OCI object length differs from its descriptor" + ); + ensure!( + sha256(bytes) == descriptor.digest, + "OCI object SHA-256 differs from its descriptor" + ); + Ok(()) +} + +fn validate_descriptor(descriptor: &Descriptor, max_bytes: u64) -> Result<()> { + ensure!( + descriptor.size > 0 && descriptor.size <= max_bytes, + "OCI descriptor size exceeds admission bounds" + ); + ensure!( + descriptor.urls.is_empty(), + "external OCI descriptor URLs are not supported" + ); + ensure!(descriptor.data.is_none(), "inline OCI descriptor data is not supported"); + ensure!( + descriptor.artifact_type.is_none(), + "OCI artifacts are not executable images" + ); + Ok(()) +} + +pub fn parse_manifest(bytes: &[u8]) -> Result { + ensure!(bytes.len() <= MAX_MANIFEST_BYTES, "OCI manifest is too large"); + let manifest: Manifest = serde_json::from_slice(bytes).context("invalid OCI image manifest")?; + ensure!(manifest.schema_version == 2, "unsupported OCI manifest schema version"); + ensure!( + matches!(manifest.media_type.as_str(), OCI_MANIFEST | DOCKER_MANIFEST), + "unsupported image manifest media type" + ); + ensure!( + manifest.artifact_type.is_none() && manifest.subject.is_none(), + "OCI artifact manifests are not executable images" + ); + ensure!(manifest.layers.len() <= MAX_LAYERS, "too many OCI image layers"); + validate_descriptor(&manifest.config, MAX_CONFIG_BYTES as u64)?; + ensure!( + matches!(manifest.config.media_type.as_str(), OCI_CONFIG | DOCKER_CONFIG), + "unsupported image config media type" + ); + let mut total = manifest.config.size; + for layer in &manifest.layers { + validate_descriptor(layer, MAX_IMAGE_BYTES)?; + ensure!( + matches!( + layer.media_type.as_str(), + "application/vnd.oci.image.layer.v1.tar" + | "application/vnd.oci.image.layer.v1.tar+gzip" + | "application/vnd.oci.image.layer.v1.tar+zstd" + | "application/vnd.docker.image.rootfs.diff.tar" + | "application/vnd.docker.image.rootfs.diff.tar.gzip" + ), + "unsupported or foreign image layer media type" + ); + total = total.checked_add(layer.size).context("OCI image size overflow")?; + } + ensure!(total <= MAX_IMAGE_BYTES, "OCI image exceeds compressed object quota"); + Ok(manifest) +} + +/// Select exactly one executable image for the requested platform. BuildKit may +/// include attestation descriptors for unknown/unknown; they are never executed. +pub fn select_platform(bytes: &[u8], requested: &ImagePlatform) -> Result { + validate_platform(requested)?; + ensure!(bytes.len() <= MAX_MANIFEST_BYTES, "OCI image index is too large"); + let index: ImageIndex = serde_json::from_slice(bytes).context("invalid OCI image index")?; + ensure!( + index.schema_version == 2 && matches!(index.media_type.as_str(), OCI_INDEX | DOCKER_INDEX), + "unsupported image index format" + ); + ensure!( + index.manifests.len() <= MAX_INDEX_ENTRIES, + "too many image index entries" + ); + let mut selected = None; + for descriptor in index.manifests { + if !descriptor.platform.as_ref().is_some_and(|p| p.matches(requested)) { + continue; + } + validate_descriptor(&descriptor, MAX_MANIFEST_BYTES as u64)?; + ensure!( + matches!(descriptor.media_type.as_str(), OCI_MANIFEST | DOCKER_MANIFEST), + "selected platform is not an image manifest" + ); + ensure!( + selected.replace(descriptor).is_none(), + "OCI index has ambiguous images for the selected platform" + ); + } + selected.context("OCI image does not contain the selected Linux platform") +} + +fn validate_platform(platform: &ImagePlatform) -> Result<()> { + ensure!( + platform.os == "linux" && matches!(platform.architecture.as_str(), "amd64" | "arm64"), + "unsupported container platform" + ); + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ImageConfig { + pub architecture: String, + pub os: String, + #[serde(default)] + pub variant: Option, + #[serde(default)] + pub config: ContainerConfig, + pub rootfs: RootFs, +} + +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct ContainerConfig { + #[serde(default)] + pub entrypoint: Option>, + #[serde(default)] + pub cmd: Option>, + #[serde(default)] + pub user: String, + #[serde(default, rename = "WorkingDir")] + pub working_directory: String, + #[serde(default)] + pub env: Option>, + #[serde(default)] + pub volumes: Option>, +} + +impl std::fmt::Debug for ContainerConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ContainerConfig") + .field("entrypoint", &self.entrypoint) + .field("cmd", &self.cmd) + .field("user", &self.user) + .field("working_directory", &self.working_directory) + .field( + "env_keys", + &self.env.as_ref().map(|env| { + env.iter() + .map(|v| v.split('=').next().unwrap_or("")) + .collect::>() + }), + ) + .field("volumes", &self.volumes.as_ref().map(|v| v.keys().collect::>())) + .finish() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RootFs { + #[serde(rename = "type")] + pub kind: String, + pub diff_ids: Vec, +} + +pub fn parse_config(bytes: &[u8], manifest: &Manifest, platform: &ImagePlatform) -> Result { + validate_platform(platform)?; + ensure!(bytes.len() <= MAX_CONFIG_BYTES, "OCI image config is too large"); + verify_object(&manifest.config, bytes)?; + let image: ImageConfig = serde_json::from_slice(bytes).context("invalid OCI image config")?; + ensure!( + Platform { + os: image.os.clone(), + architecture: image.architecture.clone(), + variant: image.variant.clone(), + os_version: None, + os_features: vec![] + } + .matches(platform), + "image config platform does not match selected platform" + ); + ensure!( + image.rootfs.kind == "layers" && image.rootfs.diff_ids.len() == manifest.layers.len(), + "image rootfs does not match layer descriptors" + ); + ensure!( + image.config.volumes.as_ref().is_none_or(BTreeMap::is_empty), + "image-declared volumes are unsupported in Stage 1" + ); + image.config.environment()?; + for argv in [&image.config.entrypoint, &image.config.cmd].into_iter().flatten() { + ensure!(argv.len() <= MAX_ARGV_ENTRIES, "too many image command arguments"); + for arg in argv { + validate_string(arg)?; + } + } + validate_string(&image.config.user)?; + validate_string(&image.config.working_directory)?; + ensure!( + image.config.working_directory.is_empty() || image.config.working_directory.starts_with('/'), + "image working directory must be absolute" + ); + Ok(image) +} + +fn validate_string(value: &str) -> Result<()> { + ensure!( + value.len() < MAX_EXEC_STRING_BYTES && !value.contains('\0'), + "invalid container startup string" + ); + Ok(()) +} + +impl ContainerConfig { + /// Keep image defaults separately from the normalized spec. Environment + /// values remain in the immutable image, never in hot deployment metadata. + pub fn environment(&self) -> Result> { + let env = self.env.as_deref().unwrap_or_default(); + ensure!(env.len() <= MAX_ENV_KEYS, "too many image environment variables"); + let mut result = BTreeMap::new(); + for value in env { + validate_string(value)?; + let (key, value) = value + .split_once('=') + .context("image environment entry must contain '='")?; + ensure!( + valid_env_key(key) && !key.starts_with("SPACETIMEDB_"), + "invalid or reserved image environment key" + ); + ensure!( + result.insert(key.to_owned(), value.to_owned()).is_none(), + "duplicate image environment key" + ); + } + Ok(result) + } + + pub fn argv(&self, override_command: Option<&[String]>) -> Result> { + let argv = match override_command { + Some(argv) => argv.to_vec(), + None => self + .entrypoint + .iter() + .flatten() + .chain(self.cmd.iter().flatten()) + .cloned() + .collect(), + }; + ensure!( + !argv.is_empty() && !argv[0].is_empty() && argv.len() <= MAX_ARGV_ENTRIES, + "image needs a nonempty main command" + ); + for arg in &argv { + validate_string(arg)?; + } + Ok(argv) + } +} + +pub fn valid_env_key(key: &str) -> bool { + let mut bytes = key.bytes(); + key.len() <= 256 + && bytes.next().is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') + && bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') +} + +/// Descriptor closure used to pin all required objects before desired-state +/// commit. Reject a digest repeated with conflicting lengths or media types. +pub fn object_closure(manifest_descriptor: Descriptor, manifest: &Manifest) -> Result> { + validate_descriptor(&manifest_descriptor, MAX_MANIFEST_BYTES as u64)?; + let mut seen = BTreeMap::new(); + let mut objects = Vec::with_capacity(manifest.layers.len() + 2); + for descriptor in std::iter::once(manifest_descriptor) + .chain(std::iter::once(manifest.config.clone())) + .chain(manifest.layers.iter().cloned()) + { + match seen.insert(descriptor.digest, (descriptor.size, descriptor.media_type.clone())) { + Some(previous) if previous != (descriptor.size, descriptor.media_type.clone()) => { + bail!("OCI digest has conflicting descriptors") + } + Some(_) => {} + None => objects.push(descriptor), + } + } + Ok(objects) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oci/src/tests.rs b/crates/oci/src/tests.rs new file mode 100644 index 00000000000..6fe4192bfa2 --- /dev/null +++ b/crates/oci/src/tests.rs @@ -0,0 +1,166 @@ +use super::*; +use serde_json::{json, Value}; + +fn platform() -> ImagePlatform { + ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + } +} +fn config() -> Value { + json!({"architecture":"arm64","os":"linux","config":{ + "Entrypoint":["node"],"Cmd":["server.js"],"User":"1000:1000","WorkingDir":"/app", + "Env":["PATH=/usr/bin","APP_KEY=image-secret"] + },"rootfs":{"type":"layers","diff_ids":[sha256(b"uncompressed layer")]}}) +} +fn fixture(config: &Value) -> (Vec, Vec, Descriptor) { + let config_bytes = serde_json::to_vec(config).unwrap(); + let manifest = serde_json::to_vec(&json!({ + "schemaVersion":2,"mediaType":OCI_MANIFEST, + "config":{"mediaType":OCI_CONFIG,"digest":sha256(&config_bytes),"size":config_bytes.len()}, + "layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":sha256(b"layer"),"size":5}] + })) + .unwrap(); + let descriptor = Descriptor { + media_type: OCI_MANIFEST.into(), + digest: sha256(&manifest), + size: manifest.len() as u64, + platform: Some(Platform { + os: "linux".into(), + architecture: "arm64".into(), + variant: Some("v8".into()), + os_version: None, + os_features: vec![], + }), + urls: vec![], + data: None, + artifact_type: None, + }; + (config_bytes, manifest, descriptor) +} + +#[test] +fn immutable_image_preserves_defaults_and_explicit_command_replaces_all_argv() { + let (bytes, raw, descriptor) = fixture(&config()); + verify_object(&descriptor, &raw).unwrap(); + let manifest = parse_manifest(&raw).unwrap(); + let image = parse_config(&bytes, &manifest, &platform()).unwrap(); + assert_eq!(image.config.argv(None).unwrap(), ["node", "server.js"]); + assert_eq!(image.config.argv(Some(&["/bin/sh".into()])).unwrap(), ["/bin/sh"]); + assert_eq!(image.config.user, "1000:1000"); + assert_eq!(image.config.working_directory, "/app"); + assert_eq!(image.config.environment().unwrap()["APP_KEY"], "image-secret"); + assert!(!format!("{image:?}").contains("image-secret")); + assert_eq!(object_closure(descriptor, &manifest).unwrap().len(), 3); +} + +#[test] +fn content_verification_checks_actual_sha256_and_length() { + assert_eq!( + sha256(b"abc").to_string(), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let (_, bytes, mut descriptor) = fixture(&config()); + assert!(verify_object(&descriptor, b"different").is_err()); + descriptor.size -= 1; + assert!(verify_object(&descriptor, &bytes).is_err()); +} + +#[test] +fn platform_selection_ignores_attestations_and_rejects_ambiguity_or_absence() { + let (_, _, descriptor) = fixture(&config()); + let mut attestation = descriptor.clone(); + attestation.platform = Some(Platform { + os: "unknown".into(), + architecture: "unknown".into(), + variant: None, + os_version: None, + os_features: vec![], + }); + let index = |descriptors: Vec| { + serde_json::to_vec(&json!({"schemaVersion":2,"mediaType":OCI_INDEX,"manifests":descriptors})).unwrap() + }; + assert_eq!( + select_platform(&index(vec![attestation.clone(), descriptor.clone()]), &platform()).unwrap(), + descriptor + ); + assert!(select_platform(&index(vec![descriptor.clone(), descriptor]), &platform()).is_err()); + assert!(select_platform(&index(vec![attestation]), &platform()).is_err()); +} + +#[test] +fn foreign_layers_urls_inline_data_and_artifacts_fail_closed() { + let (_, bytes, _) = fixture(&config()); + let baseline: Value = serde_json::from_slice(&bytes).unwrap(); + for (key, value) in [ + ("urls", json!(["http://169.254.169.254/latest/meta-data/"])), + ("data", json!("inline")), + ("artifactType", json!("application/test")), + ] { + let mut manifest = baseline.clone(); + manifest["layers"][0][key] = value; + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + } + let mut manifest = baseline.clone(); + manifest["layers"][0]["mediaType"] = json!("application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + let mut manifest = baseline; + manifest["artifactType"] = json!("application/test"); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); +} + +#[test] +fn config_rejects_volumes_wrong_platform_rootfs_and_reserved_environment() { + for (field, value) in [ + ("Volumes", json!({"/escape":{}})), + ("Env", json!(["SPACETIMEDB_IDENTITY=forged"])), + ("Env", json!(["KEY=a", "KEY=b"])), + ("Env", json!(["NO_EQUALS"])), + ("Env", json!(["KEY=contains\u{0}nul"])), + ("WorkingDir", json!("relative/path")), + ] { + let mut config = config(); + config["config"][field] = value; + let (bytes, raw, _) = fixture(&config); + assert!( + parse_config(&bytes, &parse_manifest(&raw).unwrap(), &platform()).is_err(), + "{field}" + ); + } + for (field, value) in [ + ("architecture", json!("amd64")), + ("rootfs", json!({"type":"layers","diff_ids":[]})), + ] { + let mut config = config(); + config[field] = value; + let (bytes, raw, _) = fixture(&config); + assert!(parse_config(&bytes, &parse_manifest(&raw).unwrap(), &platform()).is_err()); + } +} + +#[test] +fn size_limits_and_conflicting_descriptors_are_enforced() { + let (_, bytes, descriptor) = fixture(&config()); + let mut manifest: Value = serde_json::from_slice(&bytes).unwrap(); + manifest["layers"][0]["size"] = json!(MAX_IMAGE_BYTES); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + assert!(parse_manifest(&vec![b' '; MAX_MANIFEST_BYTES + 1]).is_err()); + let mut manifest = parse_manifest(&bytes).unwrap(); + let mut conflicting = manifest.layers[0].clone(); + conflicting.size += 1; + manifest.layers.push(conflicting); + assert!(object_closure(descriptor, &manifest).is_err()); +} + +#[test] +fn empty_scratch_image_requires_explicit_main_command() { + let config = ContainerConfig::default(); + assert!(config.argv(None).is_err()); + assert!(config.argv(Some(&[])).is_err()); + assert!(config.argv(Some(&["".into()])).is_err()); + assert_eq!(config.argv(Some(&["/main".into()])).unwrap(), ["/main"]); + assert!(valid_env_key("_A0")); + assert!(!valid_env_key("0A")); + assert!(valid_env_key(&"A".repeat(256))); + assert!(!valid_env_key(&"A".repeat(257))); +} diff --git a/crates/query/src/lib.rs b/crates/query/src/lib.rs index 4a1bea08c9c..23ec20c0701 100644 --- a/crates/query/src/lib.rs +++ b/crates/query/src/lib.rs @@ -29,7 +29,7 @@ pub fn compile_subscription( auth: &AuthCtx, ) -> Result<(Vec, TableId, TableName, bool)> { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") + bail!("SQL query exceeds maximum allowed length") } let (plan, mut has_param) = parse_and_type_sub(sql, tx, auth)?; @@ -59,11 +59,11 @@ pub fn compile_subscription( /// A utility for parsing and type checking a sql statement pub fn compile_sql_stmt(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Result { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") + bail!("SQL query exceeds maximum allowed length") } match parse_and_type_sql(sql, tx, auth)? { - stmt @ Statement::DML(_) => Ok(stmt), + stmt @ (Statement::DML(_) | Statement::Environment(_)) => Ok(stmt), Statement::Select(expr) => Ok(Statement::Select(resolve_views_for_sql(tx, expr, auth)?)), } } diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index f2fba813fa8..b5817afdbf8 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -204,6 +204,24 @@ pub struct AutoMigratePlan<'def> { } impl AutoMigratePlan<'_> { + /// Function authority changes are part of the published API comparison. + pub fn function_visibility_changes( + &self, + ) -> impl Iterator { + let reducers = self + .old + .reducers() + .filter(|old| old.lifecycle.is_none()) + .filter_map(|old| { + let new = self.new.reducer(&*old.name)?; + (old.visibility != new.visibility).then_some((&*old.name, &old.visibility, &new.visibility)) + }); + let procedures = self.old.procedures().filter_map(|old| { + let new = self.new.procedure(&*old.name)?; + (old.visibility != new.visibility).then_some((&*old.name, &old.visibility, &new.visibility)) + }); + reducers.chain(procedures) + } fn any_step(&self, f: impl Fn(&AutoMigrateStep) -> bool) -> bool { self.steps.iter().any(f) } @@ -467,6 +485,15 @@ pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> prechecks: Vec::new(), }; + let restricts_function_access = plan.function_visibility_changes().any(|(_, old, new)| { + [false, true] + .into_iter() + .any(|owner| old.allows_invocation(false, owner) && !new.allows_invocation(false, owner)) + }); + if restricts_function_access { + plan.ensure_disconnect_all_users(); + } + let views_ok = auto_migrate_views(&mut plan); let tables_ok = auto_migrate_tables(&mut plan); diff --git a/crates/schema/src/auto_migrate/formatter.rs b/crates/schema/src/auto_migrate/formatter.rs index 1f7377209bf..07128e20de5 100644 --- a/crates/schema/src/auto_migrate/formatter.rs +++ b/crates/schema/src/auto_migrate/formatter.rs @@ -17,6 +17,9 @@ use thiserror::Error; pub fn format_plan(f: &mut F, plan: &AutoMigratePlan) -> Result<(), FormattingErrors> { f.format_header()?; + for (name, old, new) in plan.function_visibility_changes() { + f.format_function_visibility(name, old, new)?; + } for step in &plan.steps { format_step(f, step, plan)?; @@ -149,6 +152,12 @@ pub enum Action { /// It allows for different implementations, such as ANSI formatting or plain text formatting. pub trait MigrationFormatter { fn format_header(&mut self) -> io::Result<()>; + fn format_function_visibility( + &mut self, + name: &str, + old: &crate::def::FunctionVisibility, + new: &crate::def::FunctionVisibility, + ) -> io::Result<()>; fn format_add_table(&mut self, table_info: &TableInfo) -> io::Result<()>; fn format_remove_table(&mut self, table_name: &Identifier) -> io::Result<()>; fn format_view(&mut self, view_info: &ViewInfo, action: Action) -> io::Result<()>; diff --git a/crates/schema/src/auto_migrate/termcolor_formatter.rs b/crates/schema/src/auto_migrate/termcolor_formatter.rs index 31807d14b30..bd2e3b60404 100644 --- a/crates/schema/src/auto_migrate/termcolor_formatter.rs +++ b/crates/schema/src/auto_migrate/termcolor_formatter.rs @@ -157,6 +157,15 @@ impl TermColorFormatter { } impl MigrationFormatter for TermColorFormatter { + fn format_function_visibility( + &mut self, + name: &str, + old: &crate::def::FunctionVisibility, + new: &crate::def::FunctionVisibility, + ) -> io::Result<()> { + self.write_bullet(&format!("Function {name} visibility: {old} -> {new}")) + } + fn format_header(&mut self) -> io::Result<()> { let line = "━".repeat(60); self.write_line(&line)?; diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index baae44ed76f..6f0417c4585 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -15,7 +15,7 @@ //! After validation, a `ModuleDef` can be converted to the `*Schema` types in `crate::schema` for use in the database. //! (Eventually, we may unify these types...) -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::{self, Debug, Write}; use std::hash::Hash; @@ -163,6 +163,9 @@ pub struct ModuleDef { /// was authored under. #[allow(unused)] raw_module_def_version: RawModuleDefVersion, + + /// Validated module bindings capabilities. Legacy modules have none. + capabilities: BTreeSet, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -174,6 +177,14 @@ pub enum RawModuleDefVersion { } impl ModuleDef { + pub fn supports_hosted_auth_v1(&self) -> bool { + self.capabilities.contains(&RawIdentifier::new("hosted_auth_v1")) + } + + pub fn capabilities(&self) -> impl Iterator { + self.capabilities.iter() + } + /// The raw module definition version this module was authored under. pub fn raw_module_def_version(&self) -> RawModuleDefVersion { self.raw_module_def_version @@ -184,6 +195,21 @@ impl ModuleDef { self.tables.values() } + /// The row type of a table or view, addressed by its canonical name. + pub fn type_ref_for_table_like(&self, name: &str) -> Option { + self.table(name) + .map(|table| table.product_type_ref) + .or_else(|| self.view(name).map(|view| view.product_type_ref)) + } + + /// Serialize without reinterpreting the definition's original version semantics. + pub fn into_raw(self) -> RawModuleDef { + match self.raw_module_def_version { + RawModuleDefVersion::V9OrEarlier => RawModuleDef::V9(self.try_into().expect("same-version conversion")), + RawModuleDefVersion::V10 => RawModuleDef::V10(self.into()), + } + } + /// The indexes of the module definition. pub fn indexes(&self) -> impl Iterator { self.tables().flat_map(|table| table.indexes.values()) @@ -469,7 +495,7 @@ impl TryFrom for ModuleDef { RawModuleDef::V8BackCompat(v8_mod) => Self::try_from(v8_mod), RawModuleDef::V9(v9_mod) => Self::try_from(v9_mod), RawModuleDef::V10(v10_mod) => Self::try_from(v10_mod), - _ => unimplemented!(), + _ => Err(crate::error::ValidationError::UnsupportedModuleVersion.into()), } } } @@ -489,8 +515,14 @@ impl TryFrom for ModuleDef { validate::v9::validate(v9_mod) } } -impl From for RawModuleDefV9 { - fn from(val: ModuleDef) -> Self { +impl TryFrom for RawModuleDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ModuleDef) -> Result { + if val.raw_module_def_version != RawModuleDefVersion::V9OrEarlier { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } let ModuleDef { tables, views, @@ -506,6 +538,7 @@ impl From for RawModuleDefV9 { http_handlers: _, http_routes: _, raw_module_def_version: _, + capabilities: _, } = val; // Extract column defaults from tables before consuming tables @@ -524,18 +557,26 @@ impl From for RawModuleDefV9 { }) .collect(); - RawModuleDefV9 { + let raw_reducers = reducers + .into_values() + .map(TryInto::try_into) + .collect::>()?; + let raw_procedures = procedures + .into_values() + .map(TryInto::try_into) + .collect::, _>>()?; + Ok(RawModuleDefV9 { tables: to_raw(tables), - reducers: reducers.into_iter().map(|(_, def)| def.into()).collect(), + reducers: raw_reducers, types: to_raw(types), misc_exports: column_defaults .into_iter() - .chain(procedures.into_iter().map(|(_, def)| def.into())) + .chain(raw_procedures) .chain(views.into_iter().map(|(_, def)| def.into())) .collect(), typespace, row_level_security: row_level_security_raw.into_iter().map(|(_, def)| def).collect(), - } + }) } } @@ -564,6 +605,7 @@ impl From for RawModuleDefV10 { http_handlers, http_routes, raw_module_def_version: _, + capabilities, } = val; let mut sections = Vec::new(); @@ -623,7 +665,15 @@ impl From for RawModuleDefV10 { RawIdentifier::from(rd.accessor_name.clone()), RawIdentifier::from(rd.name.clone()), ); - rd.into() + let public_scheduled = rd.visibility.is_client_callable() + && schedules + .iter() + .any(|schedule| schedule.function_name == RawIdentifier::from(rd.name.clone())); + let mut raw: RawReducerDefV10 = rd.into(); + if public_scheduled { + raw.visibility = RawFunctionVisibility::ExplicitClientCallable; + } + raw }) .collect(); if !raw_reducers.is_empty() { @@ -638,7 +688,15 @@ impl From for RawModuleDefV10 { RawIdentifier::from(pd.accessor_name.clone()), RawIdentifier::from(pd.name.clone()), ); - pd.into() + let public_scheduled = pd.visibility.is_client_callable() + && schedules + .iter() + .any(|schedule| schedule.function_name == RawIdentifier::from(pd.name.clone())); + let mut raw: RawProcedureDefV10 = pd.into(); + if public_scheduled { + raw.visibility = RawFunctionVisibility::ExplicitClientCallable; + } + raw }) .collect(); if !raw_procedures.is_empty() { @@ -698,6 +756,9 @@ impl From for RawModuleDefV10 { // Always emit ExplicitNames so canonical names survive the round-trip. sections.push(RawModuleDefV10Section::ExplicitNames(explicit_names)); + if !capabilities.is_empty() { + sections.push(RawModuleDefV10Section::Capabilities(capabilities.into_iter().collect())); + } RawModuleDefV10 { sections } } } @@ -1710,9 +1771,36 @@ pub enum FunctionVisibility { /// Callable from client code. ClientCallable, + + /// Callable only by a host-verified internal invocation. + Internal, +} + +impl fmt::Display for FunctionVisibility { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Private => "Private", + Self::ClientCallable => "Public", + Self::Internal => "Internal", + }) + } } impl FunctionVisibility { + pub fn is_client_callable(&self) -> bool { + matches!(self, Self::ClientCallable) + } + pub fn is_internal(&self) -> bool { + matches!(self, Self::Internal) + } + /// Lifecycle event dispatch is a separate restriction from this predicate. + pub fn allows_invocation(&self, is_internal: bool, is_authorized_private_caller: bool) -> bool { + match self { + Self::Internal => is_internal, + Self::Private => is_internal || is_authorized_private_caller, + Self::ClientCallable => true, + } + } pub fn is_private(&self) -> bool { matches!(self, FunctionVisibility::Private) } @@ -1723,16 +1811,26 @@ impl From for FunctionVisibility { fn from(val: RawFunctionVisibility) -> Self { match val { RawFunctionVisibility::Private => FunctionVisibility::Private, - RawFunctionVisibility::ClientCallable => FunctionVisibility::ClientCallable, + RawFunctionVisibility::ClientCallable | RawFunctionVisibility::ExplicitClientCallable => { + FunctionVisibility::ClientCallable + } + RawFunctionVisibility::Internal => FunctionVisibility::Internal, } } } +#[derive(Debug, Clone, thiserror::Error)] +#[error("schema cannot be represented as {target:?} without losing function visibility or source-version semantics; request schema version 10")] +pub struct SchemaConversionError { + pub target: RawModuleDefVersion, +} + impl From for RawFunctionVisibility { fn from(val: FunctionVisibility) -> Self { match val { - FunctionVisibility::Private => RawFunctionVisibility::Private, - FunctionVisibility::ClientCallable => RawFunctionVisibility::ClientCallable, + FunctionVisibility::Private => Self::Private, + FunctionVisibility::ClientCallable => Self::ClientCallable, + FunctionVisibility::Internal => Self::Internal, } } } @@ -1775,22 +1873,33 @@ pub struct ReducerDef { pub err_return_type: AlgebraicType, } -impl From for RawReducerDefV9 { - fn from(val: ReducerDef) -> Self { - RawReducerDefV9 { +impl TryFrom for RawReducerDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ReducerDef) -> Result { + if val.lifecycle.is_none() && !val.visibility.is_client_callable() { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } + Ok(RawReducerDefV9 { name: val.name.into(), params: val.params, lifecycle: val.lifecycle, - } + }) } } impl From for RawReducerDefV10 { fn from(val: ReducerDef) -> Self { + let visibility = if val.lifecycle.is_some() { + RawFunctionVisibility::Private + } else { + val.visibility.into() + }; RawReducerDefV10 { source_name: val.accessor_name.into(), params: val.params, - visibility: val.visibility.into(), + visibility, ok_return_type: val.ok_return_type, err_return_type: val.err_return_type, } @@ -1853,13 +1962,19 @@ pub struct HttpRouteDef { pub path: Box, } -impl From for RawProcedureDefV9 { - fn from(val: ProcedureDef) -> Self { - RawProcedureDefV9 { +impl TryFrom for RawProcedureDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ProcedureDef) -> Result { + if !val.visibility.is_client_callable() { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } + Ok(RawProcedureDefV9 { name: val.name.into(), params: val.params, return_type: val.return_type, - } + }) } } @@ -1874,9 +1989,10 @@ impl From for RawProcedureDefV10 { } } -impl From for RawMiscModuleExportV9 { - fn from(def: ProcedureDef) -> Self { - Self::Procedure(def.into()) +impl TryFrom for RawMiscModuleExportV9 { + type Error = SchemaConversionError; + fn try_from(def: ProcedureDef) -> Result { + Ok(Self::Procedure(def.try_into()?)) } } diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 5ea6370f2d0..7bf35c42898 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -71,9 +71,50 @@ impl From for ValidationCase { } } } -/// Validate a `RawModuleDefV9` and convert it into a `ModuleDef`, +/// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { + let mut seen_capabilities = false; + let mut capabilities = std::collections::BTreeSet::new(); + for section in &def.sections { + if let RawModuleDefV10Section::Capabilities(names) = section { + if seen_capabilities { + return Err(ValidationError::DuplicateModuleSection { + section: "Capabilities".into(), + } + .into()); + } + seen_capabilities = true; + if names.len() > 32 { + return Err(ValidationError::InvalidModuleCapabilities.into()); + } + for name in names { + if name.is_empty() + || name.len() > 64 + || !name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + || !capabilities.insert(name.clone()) + { + return Err(ValidationError::InvalidModuleCapabilities.into()); + } + } + } + } + // Retain the raw distinction until schedules are attached. Tag 1 has the + // historical contextual default; tag 3 is an explicit public declaration. + let raw_visibility: HashMap<_, _> = def + .reducers() + .into_iter() + .flatten() + .map(|function| (function.source_name.clone(), function.visibility)) + .chain( + def.procedures() + .into_iter() + .flatten() + .map(|function| (function.source_name.clone(), function.visibility)), + ) + .collect(); let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); let case_policy = def.case_conversion_policy().into(); @@ -270,7 +311,12 @@ pub fn validate(def: RawModuleDefV10) -> Result { attach_schedules_to_tables(&mut tables, schedules)?; check_scheduled_functions_exist(&mut tables, &reducers, &procedures)?; - change_scheduled_functions_and_lifetimes_visibility(&tables, &mut reducers, &mut procedures)?; + change_scheduled_functions_and_lifetimes_visibility( + &tables, + &mut reducers, + &mut procedures, + &raw_visibility, + )?; assign_query_view_primary_keys(&tables, &mut views); Ok((tables, types, reducers, procedures, views, http_handlers_and_routes)) @@ -315,16 +361,18 @@ pub fn validate(def: RawModuleDefV10) -> Result { procedures, http_handlers, http_routes, + capabilities, raw_module_def_version: RawModuleDefVersion::V10, }) } -/// Change the visibility of scheduled functions and lifecycle reducers to Internal. -/// +/// Apply historical schedule defaults only to the original ClientCallable tag. +/// Lifecycle reducers always retain their separate host-event restriction. fn change_scheduled_functions_and_lifetimes_visibility( tables: &HashMap, reducers: &mut IndexMap, procedures: &mut IndexMap, + raw_visibility: &HashMap, ) -> Result<()> { for sched_def in tables.iter().filter_map(|(_, t)| t.schedule.as_ref()) { match sched_def.function_kind { @@ -336,7 +384,12 @@ fn change_scheduled_functions_and_lifetimes_visibility( } })?; - def.visibility = crate::def::FunctionVisibility::Private; + if matches!( + raw_visibility.get(&RawIdentifier::from(def.accessor_name.clone())), + Some(RawFunctionVisibility::ClientCallable) + ) { + def.visibility = crate::def::FunctionVisibility::Private; + } } FunctionKind::Procedure => { @@ -347,7 +400,12 @@ fn change_scheduled_functions_and_lifetimes_visibility( } })?; - def.visibility = crate::def::FunctionVisibility::Private; + if matches!( + raw_visibility.get(&RawIdentifier::from(def.accessor_name.clone())), + Some(RawFunctionVisibility::ClientCallable) + ) { + def.visibility = crate::def::FunctionVisibility::Private; + } } FunctionKind::Unknown => {} @@ -356,7 +414,16 @@ fn change_scheduled_functions_and_lifetimes_visibility( for red_def in reducers.iter_mut().map(|(_, r)| r) { if red_def.lifecycle.is_some() { - red_def.visibility = crate::def::FunctionVisibility::Private; + if matches!( + raw_visibility.get(&RawIdentifier::from(red_def.accessor_name.clone())), + Some(RawFunctionVisibility::ExplicitClientCallable) + ) { + return Err(ValidationError::InvalidLifecycleVisibility { + function: red_def.accessor_name.clone().into(), + } + .into()); + } + red_def.visibility = crate::def::FunctionVisibility::Internal; } } @@ -1293,7 +1360,7 @@ mod tests { def.reducers[&check_deliveries_name].visibility, FunctionVisibility::Private, ); - assert_eq!(def.reducers[&init_name].visibility, FunctionVisibility::Private); + assert_eq!(def.reducers[&init_name].visibility, FunctionVisibility::Internal); assert_eq!( def.reducers[&extra_reducer_name].visibility, FunctionVisibility::ClientCallable @@ -2427,3 +2494,277 @@ mod tests { assert_eq!(view.param_columns[0].view_name, id("Level2Person")); } } + +#[cfg(test)] +mod visibility_tests { + use super::*; + use crate::def::FunctionVisibility; + use spacetimedb_lib::db::raw_def::v10; + use spacetimedb_lib::{db::raw_def::v9, RawModuleDef, ScheduleAt}; + use spacetimedb_sats::{AlgebraicType, ProductType}; + use v10::{FunctionVisibility as Declared, RawModuleDefV10Builder}; + + fn scheduled_module(visibility: Option, procedure: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "Jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") + .finish(); + let params = ProductType::from([("job", AlgebraicType::Ref(row))]); + if procedure { + builder.add_procedure_with_visibility("run_job", params, AlgebraicType::unit(), visibility); + } else { + builder.add_reducer_with_visibility("run_job", params, visibility); + } + builder.add_schedule("Jobs", 1, "run_job"); + builder.finish().try_into().unwrap() + } + + #[test] + fn explicit_scheduled_visibility_overrides_the_private_default() { + for procedure in [false, true] { + for (selection, expected) in [ + (None, FunctionVisibility::Private), + (Some(Declared::Private), FunctionVisibility::Private), + (Some(Declared::Internal), FunctionVisibility::Internal), + (Some(Declared::ClientCallable), FunctionVisibility::ClientCallable), + ] { + let module = scheduled_module(selection, procedure); + let visibility = if procedure { + &module.procedure("run_job").unwrap().visibility + } else { + &module.reducer("run_job").unwrap().visibility + }; + assert_eq!(visibility, &expected); + assert_eq!(module.raw_module_def_version(), RawModuleDefVersion::V10); + } + } + } + + #[test] + fn ordinary_defaults_and_lifecycle_restrictions() { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer("ordinary", ProductType::unit()); + builder.add_procedure("ordinary_procedure", ProductType::unit(), AlgebraicType::unit()); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "initialize", ProductType::unit()); + let module: ModuleDef = builder.finish().try_into().unwrap(); + assert!(module.reducer("ordinary").unwrap().visibility.is_client_callable()); + assert!(module + .procedure("ordinary_procedure") + .unwrap() + .visibility + .is_client_callable()); + assert!(module.reducer("initialize").unwrap().visibility.is_internal()); + let exported: RawModuleDefV10 = module.into(); + assert!(exported + .reducers() + .into_iter() + .flatten() + .all(|function| matches!(function.visibility, Declared::ClientCallable | Declared::Private))); + assert!(exported + .procedures() + .into_iter() + .flatten() + .all(|function| matches!(function.visibility, Declared::ClientCallable))); + for selection in [Declared::ClientCallable, Declared::ExplicitClientCallable] { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer_with_visibility( + v9::Lifecycle::Init, + "initialize", + ProductType::unit(), + Some(selection), + ); + assert!(ModuleDef::try_from(builder.finish()) + .unwrap_err() + .to_string() + .contains("must have Internal visibility")); + } + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer_with_visibility( + v9::Lifecycle::Init, + "initialize", + ProductType::unit(), + Some(Declared::Internal), + ); + assert!(ModuleDef::try_from(builder.finish()).is_ok()); + } + + #[test] + fn duplicate_definitions_sections_and_lifecycles_are_rejected() { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer("same", ProductType::unit()); + builder.add_procedure("same", ProductType::unit(), AlgebraicType::unit()); + assert!(ModuleDef::try_from(builder.finish()).is_err()); + let raw = v10::RawModuleDefV10 { + sections: vec![ + v10::RawModuleDefV10Section::Capabilities(vec![]), + v10::RawModuleDefV10Section::Capabilities(vec![]), + ], + }; + assert!(ModuleDef::try_from(raw) + .unwrap_err() + .to_string() + .contains("repeated V10 section")); + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "a", ProductType::unit()); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "b", ProductType::unit()); + assert!(ModuleDef::try_from(builder.finish()).is_err()); + } + + #[test] + fn resolved_v10_roundtrips_without_reapplying_defaults_and_rejects_v9_exports() { + for procedure in [false, true] { + for selection in [ + None, + Some(Declared::Private), + Some(Declared::Internal), + Some(Declared::ClientCallable), + ] { + let module = scheduled_module(selection, procedure); + assert!(v9::RawModuleDefV9::try_from(module.clone()).is_err()); + let RawModuleDef::V10(raw) = module.clone().into_raw() else { + panic!("lost source version") + }; + if matches!(selection, Some(Declared::ClientCallable)) { + assert!(raw + .reducers() + .into_iter() + .flatten() + .map(|function| &function.visibility) + .chain( + raw.procedures() + .into_iter() + .flatten() + .map(|function| &function.visibility) + ) + .all(|visibility| matches!(visibility, Declared::ExplicitClientCallable))); + } + let bytes = spacetimedb_lib::bsatn::to_vec(&RawModuleDef::V10(raw)).unwrap(); + let roundtrip: RawModuleDef = spacetimedb_lib::bsatn::from_slice(&bytes).unwrap(); + let roundtrip: ModuleDef = roundtrip.try_into().unwrap(); + if procedure { + assert_eq!( + roundtrip.procedure("run_job").unwrap().visibility, + module.procedure("run_job").unwrap().visibility + ); + } else { + assert_eq!( + roundtrip.reducer("run_job").unwrap().visibility, + module.reducer("run_job").unwrap().visibility + ); + } + assert_eq!(roundtrip.raw_module_def_version(), RawModuleDefVersion::V10); + } + } + } + + #[test] + fn legacy_v9_schedules_stay_public_and_v10_schedules_stay_private() { + let mut builder = v9::RawModuleDefV9Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index(v9::btree(0), "jobs_id_idx") + .with_schedule("run_job", 1) + .finish(); + builder.add_reducer("run_job", ProductType::from([("job", row.into())]), None); + let v9: ModuleDef = builder.finish().try_into().unwrap(); + assert!(v9.reducer("run_job").unwrap().visibility.is_client_callable()); + let upgraded: RawModuleDefV10 = v9.clone().into(); + assert!(matches!( + upgraded.reducers().unwrap()[0].visibility, + Declared::ExplicitClientCallable + )); + let upgraded: ModuleDef = upgraded.try_into().unwrap(); + assert!(upgraded.reducer("run_job").unwrap().visibility.is_client_callable()); + assert!(matches!(v9.into_raw(), RawModuleDef::V9(_))); + + let mut builder = v10::RawModuleDefV10Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") + .finish(); + builder.add_reducer("run_job", ProductType::from([("job", row.into())])); + builder.add_schedule("jobs", 1, "run_job"); + let v10: ModuleDef = builder.finish().try_into().unwrap(); + assert!(v10.reducer("run_job").unwrap().visibility.is_private()); + assert!(matches!(v10.into_raw(), RawModuleDef::V10(_))); + } + + #[test] + fn capabilities_are_explicit_bounded_and_preserved() { + let bare: ModuleDef = RawModuleDefV10Builder::new().finish().try_into().unwrap(); + assert!(!bare.supports_hosted_auth_v1()); + let mut builder = RawModuleDefV10Builder::new(); + builder.add_capability("hosted_auth_v1"); + let module: ModuleDef = builder.finish().try_into().unwrap(); + assert!(module.supports_hosted_auth_v1()); + let reloaded: ModuleDef = module.into_raw().try_into().unwrap(); + assert!(reloaded.supports_hosted_auth_v1()); + for names in [ + vec!["".to_string()], + vec!["Uppercase".to_string()], + vec!["with-dash".to_string()], + vec!["a".repeat(65)], + vec!["duplicate".to_string(); 2], + (0..33).map(|i| format!("cap_{i}")).collect(), + ] { + let mut builder = RawModuleDefV10Builder::new(); + for name in names { + builder.add_capability(name); + } + assert!(ModuleDef::try_from(builder.finish()).is_err()); + } + } + + #[test] + fn narrowing_function_visibility_is_a_reported_client_break() { + let module = |visibility| { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer_with_visibility("run_now", ProductType::unit(), Some(visibility)); + ModuleDef::try_from(builder.finish()).unwrap() + }; + let public = module(Declared::ClientCallable); + let internal = module(Declared::Internal); + let plan = crate::auto_migrate::ponder_migrate(&public, &internal).unwrap(); + assert!(plan.breaks_client()); + let display = plan + .pretty_print(crate::auto_migrate::PrettyPrintStyle::NoColor) + .unwrap(); + assert!(display.contains("run_now")); + assert!(display.contains("Internal")); + assert!(!crate::auto_migrate::ponder_migrate(&internal, &public) + .unwrap() + .breaks_client()); + } + + #[test] + fn visibility_authority_is_cumulative_without_elevating_the_owner() { + for (visibility, external, owner, internal) in [ + (FunctionVisibility::Internal, false, false, true), + (FunctionVisibility::Private, false, true, true), + (FunctionVisibility::ClientCallable, true, true, true), + ] { + assert_eq!(visibility.allows_invocation(false, false), external); + assert_eq!(visibility.allows_invocation(false, true), owner); + assert_eq!(visibility.allows_invocation(true, false), internal); + } + } +} diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 618f8e3c9c4..aa5e1099575 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -167,6 +167,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { procedures, http_handlers: IndexMap::new(), http_routes: Vec::new(), + capabilities: Default::default(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, }) } @@ -378,7 +379,11 @@ impl ModuleValidatorV9<'_> { recursive: false, // A ProductTypeDef not stored in a Typespace cannot be recursive. }, lifecycle, - visibility: FunctionVisibility::ClientCallable, + visibility: if lifecycle.is_some() { + FunctionVisibility::Internal + } else { + FunctionVisibility::ClientCallable + }, ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }; diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 33abb0c1866..5dfdcbe7f7e 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,6 +22,14 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { + #[error("unsupported module definition version")] + UnsupportedModuleVersion, + #[error("invalid module capabilities: at most 32 unique names of 1..64 lowercase ASCII letters, digits or underscores are allowed")] + InvalidModuleCapabilities, + #[error("lifecycle reducer `{function}` must have Internal visibility")] + InvalidLifecycleVisibility { function: RawIdentifier }, + #[error("module contains repeated V10 section `{section}`")] + DuplicateModuleSection { section: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, #[error("name `{name}` is used for multiple types")] diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index e62b9dc6963..e42b34cf6ee 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -773,7 +773,7 @@ impl TableSchema { .map(|(i, schema)| (ColId::from(i), schema)) .map(|(col_pos, schema)| ColumnSchema { col_pos, ..schema }) .collect(); - let view_primary_key = (module_def.raw_module_def_version() == RawModuleDefVersion::V10) + let view_primary_key = (module_def.raw_module_def_version() != RawModuleDefVersion::V9OrEarlier) .then_some(*primary_key) .flatten(); @@ -914,7 +914,7 @@ impl TableSchema { }; let mut constraints = vec![]; - let view_primary_key = (module_def.raw_module_def_version() == RawModuleDefVersion::V10) + let view_primary_key = (module_def.raw_module_def_version() != RawModuleDefVersion::V9OrEarlier) .then_some(primary_key.map(|pk| ColId::from(meta_cols + pk.idx()))) .flatten(); diff --git a/crates/sql-parser/src/ast/sql.rs b/crates/sql-parser/src/ast/sql.rs index be7b753f395..a7593cc4582 100644 --- a/crates/sql-parser/src/ast/sql.rs +++ b/crates/sql-parser/src/ast/sql.rs @@ -19,6 +19,15 @@ pub enum SqlAst { Set(SqlSet), /// SHOW var Show(SqlShow), + /// Administrative environment mutation, distinct from generic table DML. + Environment(SqlEnvironment), +} + +#[derive(Debug)] +pub struct SqlEnvironment { + pub key: SqlIdent, + /// None is DELETE; a string literal, including empty, is SET. + pub value: Option, } impl SqlAst { diff --git a/crates/sql-parser/src/parser/sql.rs b/crates/sql-parser/src/parser/sql.rs index e689817cc4c..762a56e3b80 100644 --- a/crates/sql-parser/src/parser/sql.rs +++ b/crates/sql-parser/src/parser/sql.rs @@ -133,11 +133,13 @@ use sqlparser::{ Value, Values, }, dialect::PostgreSqlDialect, + keywords::Keyword, parser::Parser, + tokenizer::Token, }; use crate::ast::{ - sql::{SqlAst, SqlDelete, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, + sql::{SqlAst, SqlDelete, SqlEnvironment, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, SqlIdent, }; @@ -148,7 +150,36 @@ use super::{ /// Parse a SQL string pub fn parse_sql(sql: &str) -> SqlParseResult { - let mut stmts = Parser::parse_sql(&PostgreSqlDialect {}, sql)?; + // DELETE env.KEY is a SpacetimeDB administrative statement, not the + // PostgreSQL DELETE FROM grammar. Use the same tokenizer and expression + // parser, including comments and SQL string escaping, for this extension. + let mut parser = Parser::new(&PostgreSqlDialect {}).try_with_sql(sql)?; + let verb = parser.peek_token().token; + let environment_prefix = matches!(parser.peek_nth_token(1).token, + Token::Word(word) if word.quote_style.is_none() && word.value.eq_ignore_ascii_case("env")) + && parser.peek_nth_token(2).token == Token::Period; + if environment_prefix + && matches!(&verb, Token::Word(word) if matches!(word.keyword, Keyword::SET | Keyword::DELETE)) + { + parser.next_token(); + parser.next_token(); + parser.next_token(); + let key = SqlIdent(parser.parse_identifier()?.value.into()); + let value = if matches!(verb, Token::Word(word) if word.keyword == Keyword::SET) { + if !parser.parse_keyword(Keyword::TO) { + parser.expect_token(&Token::Eq)?; + } + Some(parse_literal_expr(parser.parse_expr()?, SqlUnsupported::Assignment)?) + } else { + None + }; + let _ = parser.consume_token(&Token::SemiColon); + if parser.peek_token().token != Token::EOF { + return Err(SqlUnsupported::MultiStatement.into()); + } + return Ok(SqlAst::Environment(SqlEnvironment { key, value })); + } + let mut stmts = parser.parse_statements()?; if stmts.len() > 1 { return Err(SqlUnsupported::MultiStatement.into()); } diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index 9eeef5d3535..2b58326552a 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -27,6 +27,10 @@ directives = [ # Maximum number of JS procedure isolates per database. Omit to use the number # of cores reported by the OS. # procedure-instance-pool-size = 8 +# Wall-clock bound for each JS startup, schema description and function call. +# Defaults to 120 seconds; must be positive and no greater than 120 seconds. +# Timed-out reducers fail and roll back, including lifecycle calls during publish. +# execution-timeout = "120s" [v8-heap-policy] # Check the V8 heap after this many requests. Set to 0 to disable. diff --git a/crates/standalone/src/subcommands/extract_schema.rs b/crates/standalone/src/subcommands/extract_schema.rs index efc77960195..874f213c183 100644 --- a/crates/standalone/src/subcommands/extract_schema.rs +++ b/crates/standalone/src/subcommands/extract_schema.rs @@ -4,7 +4,7 @@ use anyhow::Context; use clap::{ArgMatches, CommandFactory, FromArgMatches}; use spacetimedb::host::extract_schema; use spacetimedb::messages::control_db; -use spacetimedb_lib::{db::raw_def::v10::RawModuleDefV10, sats, RawModuleDef}; +use spacetimedb_lib::sats; /// Extracts the module schema from a local module file. /// WARNING: This command is UNSTABLE and subject to breaking changes. @@ -67,7 +67,7 @@ pub async fn exec(args: &ArgMatches) -> anyhow::Result<()> { let module_def = extract_schema(program_bytes.into(), host_type.into()).await?; - let raw_def = RawModuleDef::V10(RawModuleDefV10::from(module_def)); + let raw_def = module_def.into_raw(); serde_json::to_writer(std::io::stdout().lock(), &sats::serde::SerdeWrapper(raw_def))?; diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index bf8c305526d..22cbcf943da 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -35,6 +35,8 @@ serde.workspace = true futures.workspace = true [dev-dependencies] +spacetimedb-auth.workspace = true +spacetimedb-datastore.workspace = true serial_test.workspace = true [lints] diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 21fea57fe96..5a07c3b6f60 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -61,6 +61,12 @@ pub struct ModuleHandle { } impl ModuleHandle { + /// Access the real standalone control/host environment for integration + /// tests that publish, migrate, or recover the running module. + pub fn environment(&self) -> &StandaloneEnv { + &self._env + } + async fn call_reducer(&self, reducer: &str, args: FunctionArgs) -> anyhow::Result<()> { let result = self .client diff --git a/crates/testing/tests/deployment_publish.rs b/crates/testing/tests/deployment_publish.rs new file mode 100644 index 00000000000..dd3b7323967 --- /dev/null +++ b/crates/testing/tests/deployment_publish.rs @@ -0,0 +1,204 @@ +//! Exercise publication against the real host and Wasm module, including the +//! unchanged-program path and retries after a later module has been installed. +use serial_test::serial; +use spacetimedb::db::deployment::{current_deployment, install_publication_fence, DeploymentCommit}; +use spacetimedb::host::{FunctionArgs, UpdateDatabaseResult}; +use spacetimedb::messages::control_db::HostType; +use spacetimedb_client_api::{ControlStateReadAccess, NodeDelegate}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::ST_DEPLOYMENT_OPERATION_ID; +use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; +use spacetimedb_lib::{hash_bytes, sats::product, ConnectionId, Hash, Identity, Uuid}; +use spacetimedb_schema::auto_migrate::{MigrationPolicy, MigrationToken}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +fn prepared(bytes: &[u8], epoch: u64, previous: Option, command: &str) -> DeploymentCommit { + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + DeploymentCommit { + operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(epoch)), + publication_epoch: epoch, + publisher: Identity::ZERO, + expected_revision: previous, + prepared_manifest_hash: hash_bytes(epoch.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(bytes), + }), + container: Some(ContainerSpec { + image_manifest: OciDigest::sha256([3; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec![command.into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 1000, + }), + }), + } +} + +fn fence(module: &spacetimedb::host::ModuleHost, request: &DeploymentCommit) { + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); +} + +async fn confirmed(result: UpdateDatabaseResult) { + let (offset, durable) = match result { + UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + } + | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { + tx_offset, + durable_offset, + } + | UpdateDatabaseResult::DeploymentAlreadyCommitted { + tx_offset, + durable_offset, + .. + } => (tx_offset, durable_offset), + other => panic!("expected a durable publication result, got {other:?}"), + }; + tokio::time::timeout(Duration::from_secs(10), async { + let offset = offset.await.unwrap(); + if let Some(mut durable) = durable { + durable.wait_for(offset).await.unwrap(); + } + }) + .await + .unwrap(); +} + +#[test] +#[serial] +fn container_only_changes_and_old_retries_preserve_the_current_module() { + let compiled = CompiledModule::compile("hosted-auth-test", CompilationMode::Debug); + let bytes = compiled.program_bytes(); + compiled.with_module_async(DEFAULT_CONFIG, |handle| async move { + let env = handle.environment(); + let database = env.get_database_by_identity(&handle.db_identity).await.unwrap().unwrap(); + let host = env.leader(database.id).await.unwrap(); + let module = host.module().await.unwrap(); + let first = prepared(&bytes, 1, None, "/app/first"); + fence(&module, &first); + // Even the first in-progress container publication fences the legacy + // raw API, including a byte-identical module update. + assert!(host.update(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible).await.is_err()); + // The rejected updater must restore the old host in its controller. + assert_eq!(host.module().await.unwrap().info.module_hash, hash_bytes(&bytes)); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); + confirmed(result).await; + let first_revision = first.deployment.revision().unwrap(); + let second = prepared(&bytes, 2, Some(first_revision), "/app/second"); + fence(&host.module().await.unwrap(), &second); + confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, second.clone()).await.unwrap()).await; + assert_ne!(first_revision, second.deployment.revision().unwrap()); + + // A valid custom section changes program bytes/hash without changing + // its definition. This exercises the actual Wasm migration and swap. + let mut newer_bytes = bytes.to_vec(); + newer_bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let third = prepared(&newer_bytes, 3, Some(second.deployment.revision().unwrap()), "/app/third"); + fence(&host.module().await.unwrap(), &third); + confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, third.clone()).await.unwrap()).await; + let newest = third.deployment.revision().unwrap(); + assert_eq!(host.module().await.unwrap().info.module_hash, hash_bytes(&newer_bytes)); + let before_retry = host.module().await.unwrap(); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); + assert!(matches!(&result, UpdateDatabaseResult::DeploymentAlreadyCommitted { result, .. } if result.revision == first_revision)); + confirmed(result).await; + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, before_retry.info.module_hash); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + }); + current.call_reducer(Identity::ZERO, None, None, None, None, "private_only", FunctionArgs::Nullary).await.unwrap().outcome.into_result().unwrap(); + + // A stale CAS and a forged association between bytes and declaration + // cannot change either component, and rejection leaves service usable. + let stale = prepared(&bytes, 4, Some(first_revision), "/app/stale"); + fence(¤t, &stale); + assert!(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, stale).await.is_err()); + let mismatched = prepared(&bytes, 5, Some(newest), "/app/mismatch"); + fence(¤t, &mismatched); + assert!(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, mismatched).await.is_err()); + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, hash_bytes(&newer_bytes)); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + }); + + let empty = spacetimedb::host::empty_module::program(1).unwrap(); + let mut remove_module = prepared(&empty.bytes, 6, Some(newest), "/app/image-only"); + let DeploymentSpec::V1(spec) = &mut remove_module.deployment; + spec.module = ModuleComponent::SystemEmpty(1); + // An init request cannot execute this instance's schema while storing + // the bytes of a different valid, prepared program. + assert!(current.init_database_with_deployment(empty.clone(), Some(remove_module.clone())).await.is_err()); + + // Removing a nonempty table fails during migration execution, after + // the tentative program and deployment/receipt writes. All roll back. + let observation_name = current.info.module_def.tables() + .find(|table| table.name.to_ascii_lowercase().contains("observation")) + .unwrap().name.to_string(); + let observation_table = current.relational_db().with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<_> { + let table = tx.table_id_from_name(&observation_name)?.unwrap(); + tx.insert_via_serialize_bsatn(table, &product![ConnectionId::from_u128(700), Identity::ZERO, false, Identity::ZERO, false])?; + Ok(table) + }).unwrap(); + let policy = MigrationPolicy::BreakClients(MigrationToken { + database_identity: handle.db_identity, + old_module_hash: current.info.module_hash, + new_module_hash: empty.hash, + }.hash()); + fence(¤t, &remove_module); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, empty.bytes.clone(), policy.clone(), remove_module.clone()).await.unwrap(); + assert!(matches!(result, UpdateDatabaseResult::ErrorExecutingMigration(ref error) if error.to_string().contains("table contains data")), "{result:?}"); + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, hash_bytes(&newer_bytes)); + assert_eq!(current.relational_db().program().unwrap().unwrap().hash, hash_bytes(&newer_bytes)); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + assert_eq!(tx.table_row_count(observation_table), Some(1)); + }); + current.relational_db().with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + tx.clear_table(observation_table)?; + Ok(()) + }).unwrap(); + // The same operation remains eligible after an execution rollback. + confirmed(host.update_with_deployment(database, HostType::Wasm, empty.bytes.clone(), policy, remove_module.clone()).await.unwrap()).await; + let current = host.module().await.unwrap(); + assert!(current.info.module_def.tables().next().is_none()); + assert!(spacetimedb::host::empty_module::matches_program(1, ¤t.relational_db().program().unwrap().unwrap())); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, remove_module.deployment.revision().unwrap()); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(4)); + }); + }); +} diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs new file mode 100644 index 00000000000..afa5c8f6f34 --- /dev/null +++ b/crates/testing/tests/environment.rs @@ -0,0 +1,160 @@ +//! Actual module calls exercise environment ABI, bindings, and snapshot semantics. +use serial_test::serial; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_lib::identity::AuthCtx; +use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; + +async fn sql(module: &ModuleHost, statement: String) -> Vec { + spacetimedb::sql::execute::run( + module.relational_db().clone(), + statement, + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await + .unwrap() + .rows +} + +async fn set_environment(module: &ModuleHost, key: &str, value: &str) { + sql(module, format!("SET env.{key} = '{}'", value.replace('\'', "''"))).await; +} + +fn exercise_fixture(name: &str) { + CompiledModule::compile(name, CompilationMode::Debug).with_module_async(DEFAULT_CONFIG, |handle| async move { + let module = handle.client.module(); + for (key, expected) in [ + ("MISSING", None), + ("EMPTY", Some("".to_string())), + ("UTF8", Some("héllo 🌍".to_string())), + ("NUL", Some("before\0after".to_string())), + ("MAXIMUM", Some("é".repeat(4096))), + ] { + if let Some(value) = &expected { + set_environment(&module, key, value).await; + } + let args = product![key, expected.clone()]; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + ) + .await; + let result = result + .map_err(anyhow::Error::from) + .and_then(|r| r.outcome.into_result()); + assert!( + result.is_ok(), + "{name} {key}: {result:?}; module log: {}", + handle.read_log(None).await + ); + let read = || FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()); + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(expected.clone())); + if expected.is_some() { + set_environment(&module, key, "updated").await; + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(Some("updated".to_string()))); + sql(&module, format!("DELETE env.{key}")).await; + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(None::)); + } + } + if name == "environment-test" { + // This view first reads a missing key. Its dependency must survive + // absence, and normal SQL mutations must invalidate its cached row. + let read_view = || "SELECT * FROM environment_value".to_string(); + assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); + set_environment(&module, "WATCHED", "first").await; + assert_eq!( + sql(&module, read_view()).await, + vec![product![Some("first".to_string())]] + ); + set_environment(&module, "WATCHED", "second").await; + assert_eq!( + sql(&module, read_view()).await, + vec![product![Some("second".to_string())]] + ); + sql(&module, "DELETE env.WATCHED".into()).await; + assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); + set_environment(&module, "LIMIT", &"x".repeat(8192)).await; + for _ in 0..2 { + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "bounded_environment_sources", + FunctionArgs::Nullary, + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + } + } + let args = product!["A=B", None::]; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + ) + .await; + assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); + }); +} + +#[test] +#[serial] +fn rust_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("environment-test"); +} + +#[test] +#[serial] +fn typescript_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-ts"); +} + +#[test] +#[serial] +fn cpp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-cpp"); +} + +#[test] +#[serial] +fn csharp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-cs"); +} diff --git a/crates/testing/tests/hosted_invocation.rs b/crates/testing/tests/hosted_invocation.rs new file mode 100644 index 00000000000..7c9b3c5528c --- /dev/null +++ b/crates/testing/tests/hosted_invocation.rs @@ -0,0 +1,261 @@ +//! Exercise Rust Wasm bindings and host admission using actual signed proofs. +use serial_test::serial; +use spacetimedb::auth::hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}; +use spacetimedb::auth::invocation::InvocationCaller; +use spacetimedb::auth::JwtKeys; +use spacetimedb::db::deployment::install_container_fence; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_auth::identity::ConnectionAuthCtx; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + StContainerFenceRow, ST_CLIENT_ID, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, +}; +use spacetimedb_lib::sats::{product, AlgebraicValue, ProductValue}; +use spacetimedb_lib::{bsatn, ConnectionId, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::{Duration, SystemTime}; + +fn authenticate(source: Identity, target: Identity, lifetime: Duration) -> ConnectionAuthCtx { + let keys = JwtKeys::generate().unwrap(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + source_database: source, + target_database: target, + generation: 1, + grant_revision: 1, + lease_expires_at: now + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "test.platform", + &binding, + now, + now + lifetime, + "wasm-integration", + ) + .unwrap(); + HostedTokenValidator::new([("test.platform".into(), keys.public)]) + .unwrap() + .validate_token(&token, target, now, |issuer, requested_source, requested_target| { + (issuer == "test.platform" && requested_source == source && requested_target == target).then_some(binding) + }) + .unwrap() + .into_connection_auth() + .unwrap() +} + +fn install_fence(module: &ModuleHost, source: Identity, generation: u64, allowed: bool) { + let db = module.relational_db(); + db.with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + install_container_fence( + db, + tx, + &StContainerFenceRow { + source_identity: source.into(), + generation, + target_grant_revision: generation, + target_set_hash: spacetimedb_lib::hash_bytes(b"configured targets"), + allowed, + }, + )?; + Ok(()) + }) + .unwrap(); +} + +fn assert_connection_count(module: &ModuleHost, expected: u64) { + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(expected)); + } + Ok(()) + }) + .unwrap(); +} + +fn arguments(values: ProductValue) -> FunctionArgs { + FunctionArgs::Bsatn(bsatn::to_vec(&values).unwrap().into()) +} + +async fn call( + module: &ModuleHost, + caller: impl Into + Send, + connection: Option, + reducer: &str, + args: ProductValue, +) -> anyhow::Result<()> { + module + .call_reducer(caller, connection, None, None, None, reducer, arguments(args)) + .await? + .outcome + .into_result() +} + +#[test] +#[serial] +fn hosted_wasm_calls_preserve_authority_and_disconnect_after_revocation() { + CompiledModule::compile("hosted-auth-test", CompilationMode::Debug).with_module_async( + DEFAULT_CONFIG, + |handle| async move { + let module = handle.client.module(); + let target = handle.db_identity; + let foreign = Identity::ONE; + let owner = Identity::ZERO; + let self_connection = ConnectionId::from_u128(101); + let foreign_connection = ConnectionId::from_u128(102); + let failed_disconnect = ConnectionId::from_u128(999); + let expired_connection = ConnectionId::from_u128(103); + install_fence(&module, target, 1, true); + install_fence(&module, foreign, 1, true); + let self_auth = authenticate(target, target, Duration::from_secs(30)); + let foreign_auth = authenticate(foreign, target, Duration::from_secs(30)); + let expiring_auth = authenticate(target, target, Duration::from_secs(3)); + for (auth, connection) in [ + (self_auth.clone(), self_connection), + (foreign_auth.clone(), foreign_connection), + (self_auth.clone(), failed_disconnect), + (expiring_auth.clone(), expired_connection), + ] { + module.call_identity_connected(auth, connection).await.unwrap(); + } + assert_connection_count(&module, 4); + for (auth, sender, connection, internal) in [ + (&self_auth, target, self_connection, true), + (&foreign_auth, foreign, foreign_connection, false), + ] { + call( + &module, + auth, + Some(connection), + "inspect_context", + product![sender, Some(connection), internal, true], + ) + .await + .unwrap(); + let result = module + .call_procedure( + auth, + Some(connection), + None, + "inspect_procedure", + arguments(product![sender, Some(connection), internal]), + ) + .await; + assert_eq!(result.result.unwrap().return_val, AlgebraicValue::Bool(true)); + for lifecycle in ["connected", "disconnected"] { + assert!(call(&module, auth, Some(connection), lifecycle, product![]) + .await + .is_err()); + } + } + // Ordinary owner calls and equal database identities remain external. + for ordinary in [target, owner] { + call( + &module, + ordinary, + None, + "inspect_context", + product![ordinary, Option::::None, false, false], + ) + .await + .unwrap(); + assert!(call(&module, ordinary, None, "internal_only", product![]) + .await + .is_err()); + } + call(&module, owner, None, "private_only", product![]).await.unwrap(); + call(&module, &self_auth, Some(self_connection), "internal_only", product![]) + .await + .unwrap(); + call(&module, &self_auth, Some(self_connection), "private_only", product![]) + .await + .unwrap(); + for reducer in ["internal_only", "private_only"] { + assert!( + call(&module, &foreign_auth, Some(foreign_connection), reducer, product![]) + .await + .is_err() + ); + } + call(&module, owner, None, "schedule_check", product![]).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let result = module + .call_procedure(owner, None, None, "scheduled_finished", FunctionArgs::Nullary) + .await; + if result.result.unwrap().return_val == AlgebraicValue::Bool(true) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("scheduled call did not observe internal authority"); + // Tokens remain cryptographically valid, but their persisted grants are revoked. + install_fence(&module, target, 2, false); + install_fence(&module, foreign, 2, false); + assert!( + call(&module, &self_auth, Some(self_connection), "internal_only", product![]) + .await + .is_err() + ); + let result = module + .call_procedure( + &self_auth, + Some(self_connection), + None, + "inspect_procedure", + arguments(product![target, Some(self_connection), true]), + ) + .await; + assert!(result.result.is_err()); + if let Ok(remaining) = expiring_auth + .hosted + .as_ref() + .unwrap() + .expires_at() + .duration_since(SystemTime::now()) + { + tokio::time::sleep(remaining + Duration::from_millis(20)).await; + } + assert!(call( + &module, + &expiring_auth, + Some(expired_connection), + "internal_only", + product![] + ) + .await + .is_err()); + // Host cleanup retains captured flags and JWT sender after revocation and expiry. + for (sender, connection) in [ + (target, self_connection), + (foreign, foreign_connection), + (target, failed_disconnect), + (target, expired_connection), + ] { + module.call_identity_disconnected(sender, connection).await.unwrap(); + } + assert_connection_count(&module, 0); + for (connection, sender, internal, disconnected) in [ + (self_connection, target, true, true), + (foreign_connection, foreign, false, true), + (failed_disconnect, target, true, false), + (expired_connection, target, true, true), + ] { + call( + &module, + owner, + None, + "inspect_observation", + product![connection, sender, internal, disconnected], + ) + .await + .unwrap(); + } + }, + ); +} diff --git a/modules/environment-test/Cargo.toml b/modules/environment-test/Cargo.toml new file mode 100644 index 00000000000..9ae6466b734 --- /dev/null +++ b/modules/environment-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "environment-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs new file mode 100644 index 00000000000..fd4d1159df9 --- /dev/null +++ b/modules/environment-test/src/lib.rs @@ -0,0 +1,54 @@ +use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; + +#[spacetimedb::reducer] +pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert_eq!(ctx.env.get(&key), expected); + assert_eq!(ctx.as_read_only().env.get(&key), expected); + assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); +} + +#[spacetimedb::procedure] +pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option { + let outside = ctx.env.get(&key); + ctx.with_tx(|tx| assert_eq!(tx.env.get(&key), outside)); + outside +} + +#[derive(SpacetimeType)] +pub struct EnvironmentValue { + pub value: Option, +} + +#[spacetimedb::view(accessor = environment_value, public)] +pub fn environment_value(ctx: &AnonymousViewContext) -> Option { + Some(EnvironmentValue { + value: ctx.env.get("WATCHED"), + }) +} + +/// Hand-written ABI callers cannot retain unbounded host allocations. +#[spacetimedb::reducer] +pub fn bounded_environment_sources(_ctx: &ReducerContext) { + use spacetimedb::sys::raw::{self, BytesSource}; + let mut sources = Vec::new(); + for i in 0..=256 { + let mut source = BytesSource::INVALID; + let status = unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }; + if i == 256 { + assert_eq!(status, 9); // NO_SPACE + } else { + assert_eq!(status, 0); + assert!(source != BytesSource::INVALID); + sources.push(source); + } + } + let mut buffer = [0u8; 8192]; + let mut len = buffer.len(); + let status = unsafe { raw::bytes_source_read(sources[0], buffer.as_mut_ptr(), &mut len) }; + assert_eq!(status, -1); + assert_eq!(len, buffer.len()); + let mut source = BytesSource::INVALID; + assert_eq!(unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }, 0); + assert!(source != BytesSource::INVALID); + // The remaining sources are released when this invocation ends. +} diff --git a/modules/hosted-auth-test/Cargo.toml b/modules/hosted-auth-test/Cargo.toml new file mode 100644 index 00000000000..b37b9b44332 --- /dev/null +++ b/modules/hosted-auth-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "hosted-auth-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/hosted-auth-test/src/lib.rs b/modules/hosted-auth-test/src/lib.rs new file mode 100644 index 00000000000..f6380f686eb --- /dev/null +++ b/modules/hosted-auth-test/src/lib.rs @@ -0,0 +1,141 @@ +//! Actual host integration fixture for verified container authentication. +use spacetimedb::{ConnectionId, Identity, ProcedureContext, ReducerContext, Table}; + +#[spacetimedb::table(accessor = observations)] +pub struct Observation { + #[primary_key] + connection: ConnectionId, + sender: Identity, + internal: bool, + jwt_identity: Identity, + disconnected: bool, +} + +#[spacetimedb::reducer(client_connected)] +pub fn connected(ctx: &ReducerContext) { + ctx.db.observations().insert(Observation { + connection: ctx.connection_id().unwrap(), + sender: ctx.sender(), + internal: ctx.sender_auth().is_internal(), + jwt_identity: ctx.sender_auth().jwt().unwrap().identity(), + disconnected: false, + }); +} + +#[spacetimedb::reducer(client_disconnected)] +pub fn disconnected(ctx: &ReducerContext) -> Result<(), String> { + let connection = ctx.connection_id().unwrap(); + let mut observation = ctx.db.observations().connection().find(connection).unwrap(); + assert_eq!(ctx.sender(), observation.sender); + assert_eq!(ctx.sender_auth().is_internal(), observation.internal); + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), observation.jwt_identity); + // Exercise host fallback cleanup after a user callback rejects disconnect. + if connection == ConnectionId::from_u128(999) { + return Err("intentional disconnect failure".into()); + } + observation.disconnected = true; + ctx.db.observations().connection().update(observation); + Ok(()) +} + +#[spacetimedb::reducer] +pub fn inspect_context( + ctx: &ReducerContext, + sender: Identity, + connection: Option, + internal: bool, + jwt: bool, +) { + assert_eq!(ctx.sender(), sender); + assert_eq!(ctx.connection_id(), connection); + assert_eq!(ctx.sender_auth().is_internal(), internal); + assert_eq!(ctx.sender_auth().has_jwt(), jwt); + if jwt { + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), sender); + } +} + +#[spacetimedb::reducer(internal)] +pub fn internal_only(ctx: &ReducerContext) { + assert!(ctx.sender_auth().is_internal()); +} + +#[spacetimedb::reducer(private)] +pub fn private_only(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer] +pub fn inspect_observation( + ctx: &ReducerContext, + connection: ConnectionId, + sender: Identity, + internal: bool, + disconnected: bool, +) { + let observation = ctx.db.observations().connection().find(connection).unwrap(); + assert_eq!(observation.sender, sender); + assert_eq!(observation.jwt_identity, sender); + assert_eq!(observation.internal, internal); + assert_eq!(observation.disconnected, disconnected); +} + +#[spacetimedb::procedure] +pub fn inspect_procedure( + ctx: &mut ProcedureContext, + sender: Identity, + connection: Option, + internal: bool, +) -> bool { + assert_eq!(ctx.sender(), sender); + assert_eq!(ctx.connection_id(), connection); + assert_eq!(ctx.sender_auth().is_internal(), internal); + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), sender); + ctx.with_tx(|tx| { + assert_eq!(tx.sender(), sender); + assert_eq!(tx.connection_id(), connection); + assert_eq!(tx.sender_auth().is_internal(), internal); + assert_eq!(tx.sender_auth().jwt().unwrap().identity(), sender); + }); + true +} + +#[spacetimedb::table(accessor = scheduled_checks, scheduled(scheduled_check))] +pub struct ScheduledCheck { + #[primary_key] + #[auto_inc] + id: u64, + scheduled_at: spacetimedb::ScheduleAt, +} + +#[spacetimedb::reducer] +pub fn schedule_check(ctx: &ReducerContext) { + ctx.db.scheduled_checks().insert(ScheduledCheck { + id: 0, + scheduled_at: ctx.timestamp.into(), + }); +} + +#[spacetimedb::reducer] +pub fn scheduled_check(ctx: &ReducerContext, _job: ScheduledCheck) { + assert_eq!(ctx.sender(), ctx.database_identity()); + assert_eq!(ctx.connection_id(), None); + assert!(ctx.sender_auth().is_internal()); + assert!(!ctx.sender_auth().has_jwt()); + ctx.db.observations().insert(Observation { + connection: ConnectionId::from_u128(777), + sender: ctx.sender(), + internal: true, + jwt_identity: ctx.sender(), + disconnected: false, + }); +} + +#[spacetimedb::procedure] +pub fn scheduled_finished(ctx: &mut ProcedureContext) -> bool { + ctx.with_tx(|tx| { + tx.db + .observations() + .connection() + .find(ConnectionId::from_u128(777)) + .is_some() + }) +} diff --git a/modules/module-test-cpp/src/lib.cpp b/modules/module-test-cpp/src/lib.cpp index bc5305b1f16..b5035c68257 100644 --- a/modules/module-test-cpp/src/lib.cpp +++ b/modules/module-test-cpp/src/lib.cpp @@ -703,3 +703,15 @@ SPACETIMEDB_PROCEDURE(std::string, get_my_schema_via_http, ProcedureContext ctx) return result.error(); } } + +SPACETIMEDB_REDUCER(expect_environment, ReducerContext ctx, std::string key, std::optional expected) { + if (ctx.env.get(key) != expected) LOG_PANIC("environment value mismatch"); + return Ok(); +} +SPACETIMEDB_PROCEDURE(std::optional, read_environment, ProcedureContext ctx, std::string key) { + const auto outside = ctx.env.get(key); + ctx.with_tx([&](TxContext& tx) { + if (tx.env.get(key) != outside) LOG_PANIC("transaction environment value mismatch"); + }); + return outside; +} diff --git a/modules/module-test-cs/EnvironmentTests.cs b/modules/module-test-cs/EnvironmentTests.cs new file mode 100644 index 00000000000..1798d7bda44 --- /dev/null +++ b/modules/module-test-cs/EnvironmentTests.cs @@ -0,0 +1,25 @@ +#pragma warning disable STDB_UNSTABLE +namespace SpacetimeDB.Modules.ModuleTestCs; + +using SpacetimeDB; + +public static partial class EnvironmentTests +{ + [Reducer] + public static void expect_environment(ReducerContext ctx, string key, string? expected) + { + if (ctx.Env.Get(key) != expected) throw new Exception("environment value mismatch"); + } + + [Procedure] + public static string? read_environment(ProcedureContext ctx, string key) + { + var outside = ctx.Env.Get(key); + ctx.WithTx(tx => + { + if (tx.Env.Get(key) != outside) throw new Exception("transaction environment value mismatch"); + return true; + }); + return outside; + } +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index b8f0c01d558..4388f8a41f1 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -510,7 +510,7 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; try { const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=9` + `http://localhost:3000/v1/database/${module_identity}/schema?version=10` ); return response.text(); } catch (e) { @@ -520,3 +520,26 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { throw e; } }); + +// Dedicated environment ABI integration exercised by crates/testing. +export const expect_environment = spacetimedb.reducer( + { key: t.string(), expected: t.option(t.string()) }, + (ctx, { key, expected }) => { + if (ctx.env.get(key) !== (expected ?? null)) { + throw new Error('environment value mismatch'); + } + } +); +export const read_environment = spacetimedb.procedure( + { key: t.string() }, + t.option(t.string()), + (ctx, { key }) => { + const outside = ctx.env.get(key); + ctx.withTx(tx => { + if (tx.env.get(key) !== outside) { + throw new Error('transaction environment value mismatch'); + } + }); + return outside ?? undefined; + } +); diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index 56e6b288e2d..3be87f37593 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -545,7 +545,7 @@ fn with_tx(ctx: &mut ProcedureContext) { fn get_my_schema_via_http(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=9" + "http://localhost:3000/v1/database/{module_identity}/schema?version=10" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => format!("{e}"), diff --git a/modules/sdk-test-procedure-ts/src/index.ts b/modules/sdk-test-procedure-ts/src/index.ts index f89aa76665a..e9e42163f17 100644 --- a/modules/sdk-test-procedure-ts/src/index.ts +++ b/modules/sdk-test-procedure-ts/src/index.ts @@ -93,7 +93,7 @@ export const will_panic = spacetimedb.procedure(t.unit(), _ctx => { export const read_my_schema = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=9` + `http://localhost:3000/v1/database/${module_identity}/schema?version=10` ); return response.text(); }); diff --git a/modules/sdk-test-procedure/src/lib.rs b/modules/sdk-test-procedure/src/lib.rs index 95ae9b523b1..71b6bd4fedd 100644 --- a/modules/sdk-test-procedure/src/lib.rs +++ b/modules/sdk-test-procedure/src/lib.rs @@ -44,7 +44,7 @@ fn will_panic(_ctx: &mut ProcedureContext) { fn read_my_schema(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=9" + "http://localhost:3000/v1/database/{module_identity}/schema?version=10" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => panic!("{e}"), diff --git a/sdks/rust/src/client_cache.rs b/sdks/rust/src/client_cache.rs index ab0a90980bd..afe8237ffa6 100644 --- a/sdks/rust/src/client_cache.rs +++ b/sdks/rust/src/client_cache.rs @@ -2,6 +2,9 @@ //! //! This module is internal, and may incompatibly change without warning. +#[cfg(test)] +mod uuid_tests; + use crate::callbacks::CallbackId; use crate::db_connection::{debug_log, PendingMutation, SharedCell}; use crate::spacetime_module::{InModule, SpacetimeModule, TableUpdate, WithBsatn}; @@ -449,7 +452,9 @@ impl TableHandle { /// See [`DbContextImpl::queue_mutation`]. fn queue_mutation(&self, mutation: PendingMutation) { - self.pending_mutations.unbounded_send(mutation).unwrap(); + // A retained table handle may outlive terminal connection cleanup. + // The closed queue rejects and releases its callback captures. + let _ = self.pending_mutations.unbounded_send(mutation); } /// Called by the autogenerated implementation of the [`crate::Table`] method of the same name. diff --git a/sdks/rust/src/client_cache/uuid_tests.rs b/sdks/rust/src/client_cache/uuid_tests.rs new file mode 100644 index 00000000000..a9516acc81b --- /dev/null +++ b/sdks/rust/src/client_cache/uuid_tests.rs @@ -0,0 +1,75 @@ +use super::*; +use spacetimedb_lib::Uuid; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Debug)] +struct CountedRow { + id: Uuid, + value: u64, + clones: Arc, +} +impl Clone for CountedRow { + fn clone(&self) -> Self { + self.clones.fetch_add(1, Ordering::SeqCst); + Self { + id: self.id, + value: self.value, + clones: self.clones.clone(), + } + } +} + +fn row(id: Uuid, value: u64, clones: &Arc) -> WithBsatn { + WithBsatn { + bsatn: spacetimedb_lib::bsatn::to_vec(&(id, value)).unwrap().into(), + row: CountedRow { + id, + value, + clones: clones.clone(), + }, + } +} + +#[test] +fn uuid_unique_lookup_clones_only_the_match_and_tracks_updates_and_deletes() { + let clones = Arc::new(AtomicUsize::new(0)); + let mut cache = TableCache::::new(None); + // This is the exact registration emitted by Rust codegen. + cache.add_unique_constraint::("id", |row| &row.id); + let ids = (0u8..128) + .map(|n| Uuid::from_random_bytes_v4([n; 16])) + .collect::>(); + cache.apply_diff(&TableUpdate { + inserts: ids + .iter() + .enumerate() + .map(|(n, id)| row(*id, n as u64, &clones)) + .collect(), + deletes: vec![], + }); + clones.store(0, Ordering::SeqCst); + // UniqueConstraintHandle::find delegates to this lookup and clones only + // its selected row. A whole-cache snapshot would increment this 128 times. + let found = cache.find_by_unique_index("id", &ids[63]).cloned().unwrap(); + assert_eq!((found.id, found.value), (ids[63], 63)); + assert_eq!(clones.load(Ordering::SeqCst), 1); + let absent = Uuid::from_random_bytes_v4([255; 16]); + assert!(cache.find_by_unique_index("id", &absent).cloned().is_none()); + assert_eq!(clones.load(Ordering::SeqCst), 1); + + cache.apply_diff(&TableUpdate { + inserts: vec![row(ids[63], 999, &clones)], + deletes: vec![row(ids[63], 63, &clones)], + }); + clones.store(0, Ordering::SeqCst); + assert_eq!(cache.find_by_unique_index("id", &ids[63]).cloned().unwrap().value, 999); + assert_eq!(clones.load(Ordering::SeqCst), 1); + cache.apply_diff(&TableUpdate { + inserts: vec![], + deletes: vec![row(ids[63], 999, &clones)], + }); + clones.store(0, Ordering::SeqCst); + assert!(cache.find_by_unique_index("id", &ids[63]).cloned().is_none()); + assert_eq!(clones.load(Ordering::SeqCst), 0); + assert_eq!(cache.find_by_unique_index("id", &ids[64]).unwrap().value, 64); +} diff --git a/sdks/rust/src/db_connection.rs b/sdks/rust/src/db_connection.rs index 332aac1b322..8d114f4cbce 100644 --- a/sdks/rust/src/db_connection.rs +++ b/sdks/rust/src/db_connection.rs @@ -55,6 +55,9 @@ use tokio::{ pub(crate) type SharedCell = Arc>; +#[cfg(all(test, not(feature = "browser")))] +mod builder_tests; + #[cfg(not(feature = "browser"))] type SharedAsyncCell = Arc>; #[cfg(feature = "browser")] @@ -136,6 +139,12 @@ impl DbContextImpl { /// applying its mutations to the client cache and invoking callbacks. fn process_message(&self, msg: ParsedMessage) -> crate::Result<()> { self.debug_log(|out| writeln!(out, "`process_message`: {msg:?}")); + if !self.is_active() && !matches!(&msg, ParsedMessage::Error(_)) { + // Local disconnect invalidated callbacks. Drain already received + // results without interpreting them as outstanding healthy calls; + // the socket's terminal event must still reach end_connection. + return Ok(()); + } match msg { // Error: route as a connection error if we never finished connecting, // otherwise treat it as an erroneous disconnect. @@ -317,17 +326,27 @@ impl DbContextImpl { /// /// Returns the terminal error that should be returned from `advance_*` methods. fn end_connection(&self, callback_error: Option) -> crate::Error { - let mut inner = self.inner.lock().unwrap(); let return_error = callback_error.clone().unwrap_or(crate::Error::Disconnected); + let (lifecycle, db_callbacks, mut subscriptions, connect_callback, disconnect_callback, connect_error_callback) = { + let mut inner = self.inner.lock().unwrap(); + let lifecycle = inner.connection_lifecycle; + inner.connection_lifecycle = ConnectionLifecycle::Ended; + ( + lifecycle, + std::mem::take(&mut inner.db_callbacks), + std::mem::take(&mut inner.subscriptions), + inner.on_connect.take(), + inner.on_disconnect.take(), + inner.on_connect_error.take(), + ) + }; - let lifecycle = inner.connection_lifecycle; - if lifecycle == ConnectionLifecycle::Ended { - return return_error; - } - inner.connection_lifecycle = ConnectionLifecycle::Ended; - - // Set `send_chan` to `None`, since `Self::is_active` checks that. - *self.send_chan.lock().unwrap() = None; + // Serialize request enqueueing with terminal closure. A retained + // DbConnection must not retain in-flight callback captures indefinitely. + let outgoing = self.send_chan.lock().unwrap().take(); + self.discard_pending_requests(); + drop((outgoing, db_callbacks, connect_callback)); + subscriptions.on_disconnect(&self.make_event_ctx(callback_error.clone())); match lifecycle { ConnectionLifecycle::Connecting => { @@ -335,26 +354,59 @@ impl DbContextImpl { source: InternalError::new("Connection closed before receiving the initial connection message"), }); let ctx: M::ErrorContext = self.make_event_ctx(Some(callback_error.clone())); - if let Some(connect_error_callback) = inner.on_connect_error.take() { + if let Some(connect_error_callback) = connect_error_callback { connect_error_callback(&ctx, callback_error.clone()); } callback_error } ConnectionLifecycle::Connected => { let ctx: M::ErrorContext = self.make_event_ctx(callback_error.clone()); - if let Some(disconnect_callback) = inner.on_disconnect.take() { + if let Some(disconnect_callback) = disconnect_callback { disconnect_callback(&ctx, callback_error.clone()); } - - // Call the `on_disconnect` method for all subscriptions. - inner.subscriptions.on_disconnect(&ctx); - return_error } ConnectionLifecycle::Ended => return_error, } } + fn discard_pending_requests(&self) { + let (reducer_callbacks, procedure_callbacks) = { + let mut inner = self.inner.lock().unwrap(); + ( + std::mem::take(&mut inner.reducer_callbacks), + std::mem::take(&mut inner.procedure_callbacks), + ) + }; + let mut queued = Vec::new(); + { + // There is exactly one supported advance_* caller, and get_message + // releases this guard before applying a terminal message. Avoid a + // blocking_lock here because advance_one_message_async runs in Tokio. + #[cfg(not(feature = "browser"))] + let mut pending = self + .pending_mutations_recv + .try_lock() + .expect("concurrent SDK message advancement"); + #[cfg(feature = "browser")] + let mut pending = self.pending_mutations_recv.lock().unwrap(); + pending.close(); + while let Ok(Some(mutation)) = pending.try_next() { + queued.push(mutation); + } + } + // Destructors may reenter SDK methods. Drop user captures only after + // releasing the connection and pending-queue locks. These calls have an + // unknown outcome; do not manufacture a successful reducer completion. + for mutation in queued { + if let PendingMutation::Subscribe { handle, .. } = &mutation { + handle.cancel_pending_callbacks(); + } + drop(mutation); + } + drop((reducer_callbacks, procedure_callbacks)); + } + fn make_event_ctx>(&self, event: E) -> Ctx { let imp = self.clone(); Ctx::new(imp, event) @@ -362,7 +414,9 @@ impl DbContextImpl { /// Apply all queued [`PendingMutation`]s. fn apply_pending_mutations(&self) -> crate::Result<()> { - while let Ok(Some(pending_mutation)) = get_lock_sync(&self.pending_mutations_recv).try_next() { + loop { + let pending = get_lock_sync(&self.pending_mutations_recv).try_next(); + let Ok(Some(pending_mutation)) = pending else { break }; self.apply_mutation(pending_mutation)?; } @@ -372,6 +426,16 @@ impl DbContextImpl { /// Apply an individual [`PendingMutation`]. fn apply_mutation(&self, mutation: PendingMutation) -> crate::Result<()> { self.debug_log(|out| writeln!(out, "`apply_mutation`: {mutation:?}")); + if !self.is_active() + && matches!( + &mutation, + PendingMutation::InvokeReducerWithCallback { .. } | PendingMutation::InvokeProcedureWithCallback { .. } + ) + { + // A call may have been queued behind the disconnect mutation. Drop + // its captures and continue driving the actual terminal event. + return Ok(()); + } match mutation { // Subscribe: register the subscription in the [`SubscriptionManager`] // and send the `Subscribe` WS message. @@ -491,6 +555,7 @@ impl DbContextImpl { // eventually resulting in disconnect callbacks being called // if the initial connection had completed. *self.send_chan.lock().unwrap() = None; + self.discard_pending_requests(); } // Callback stuff: these all do what you expect. @@ -605,13 +670,13 @@ impl DbContextImpl { // This may be unnecessary, but `tokio::select` does not document any ordering guarantees, // and if both `pending_mutations.next()` and `recv.next()` have values ready, // we want to process the pending mutation first. - if let Ok(pending_mutation) = pending_mutations.try_next() { - return Message::Local(pending_mutation.unwrap()); + if let Ok(Some(pending_mutation)) = pending_mutations.try_next() { + return Message::Local(pending_mutation); } #[cfg(not(feature = "browser"))] tokio::select! { - pending_mutation = pending_mutations.next() => Message::Local(pending_mutation.unwrap()), + Some(pending_mutation) = pending_mutations.next() => Message::Local(pending_mutation), incoming_message = recv.next() => Message::Ws(incoming_message), } @@ -621,7 +686,10 @@ impl DbContextImpl { pin_mut!(pending_fut, recv_fut); futures::select! { - pending_mutation = pending_fut => Message::Local(pending_mutation.unwrap()), + pending_mutation = pending_fut => match pending_mutation { + Some(pending_mutation) => Message::Local(pending_mutation), + None => Message::Ws(recv_fut.await), + }, incoming_message = recv_fut => Message::Ws(incoming_message), } } @@ -716,8 +784,7 @@ impl DbContextImpl { } self.pending_mutations_send .unbounded_send(PendingMutation::Disconnect) - .unwrap(); - Ok(()) + .map_err(|_| crate::Error::Disconnected) } /// Add a [`PendingMutation`] to the `pending_mutations` queue, @@ -725,8 +792,14 @@ impl DbContextImpl { /// /// This is used to defer operations which would otherwise need to hold a lock on `self.inner`, /// as otherwise running those operations within a callback would deadlock. - fn queue_mutation(&self, mutation: PendingMutation) { + fn queue_mutation(&self, mutation: PendingMutation) -> crate::Result<()> { + let outgoing = self.send_chan.lock().unwrap(); + if outgoing.is_none() { + drop(outgoing); + return Err(crate::Error::Disconnected); + } self.pending_mutations_send.unbounded_send(mutation).unwrap(); + Ok(()) } /// Called by autogenerated table access methods. @@ -757,8 +830,7 @@ impl DbContextImpl { self.queue_mutation(PendingMutation::InvokeReducerWithCallback { reducer: reducer.into(), callback: Box::new(callback), - }); - Ok(()) + }) } /// Called by the autogenerated `DbConnection` method of the same name. @@ -788,7 +860,7 @@ impl DbContextImpl { + Send + 'static, ) { - self.queue_mutation(PendingMutation::InvokeProcedureWithCallback { + let _ = self.queue_mutation(PendingMutation::InvokeProcedureWithCallback { procedure: procedure_name, args: bsatn::to_vec(&args).expect("Failed to BSATN serialize procedure args"), callback: Box::new(move |ctx, ret| { @@ -953,6 +1025,25 @@ but you must call one of them, or else the connection will never progress. Ok(::new(imp)) } + /// Open a connection asynchronously on the current Tokio runtime. + /// + /// Unlike [`Self::build`], the WebSocket handshake does not block this task. + /// Dropping this future while the handshake is pending closes its socket + /// and releases the builder's callbacks. Callers can impose their own + /// connection deadline with [`tokio::time::timeout`]. Background connection + /// tasks are started only after the handshake succeeds. + /// + /// Requires an active Tokio runtime. As with [`Self::build`], the returned + /// connection must be advanced explicitly to receive events. + #[cfg(not(feature = "browser"))] + pub async fn build_async(self) -> crate::Result { + let handle = runtime::Handle::try_current().map_err(|error| { + InternalError::new("DbConnectionBuilder::build_async requires a Tokio runtime").with_cause(error) + })?; + let imp = self.build_native_impl(handle).await?; + Ok(::new(imp)) + } + #[cfg(feature = "browser")] pub async fn build(self) -> crate::Result { let imp = self.build_impl().await?; @@ -963,6 +1054,19 @@ but you must call one of them, or else the connection will never progress. /// to construct a [`DbContextImpl`]. #[cfg(not(feature = "browser"))] fn build_impl(self) -> crate::Result> { + let (runtime, handle) = enter_or_create_runtime()?; + // Keep an SDK-owned runtime outside its own block_on future. If the + // handshake fails, dropping that runtime inside the future would panic. + let imp = tokio::task::block_in_place(|| handle.block_on(self.build_native_impl(handle.clone())))?; + imp.inner.lock().unwrap().runtime = runtime; + Ok(imp) + } + + /// Share native construction between the synchronous and asynchronous API. + /// The handshake is the only suspension point. Until it succeeds, this + /// future owns the socket and callbacks without spawning background tasks. + #[cfg(not(feature = "browser"))] + async fn build_native_impl(self, handle: runtime::Handle) -> crate::Result> { let extra_logging = self .additional_logging_path .map(|path| { @@ -973,18 +1077,15 @@ but you must call one of them, or else the connection will never progress. .transpose()? .map(|file| Arc::new(StdMutex::new(file))); - let (runtime, handle) = enter_or_create_runtime()?; - let connection_id_override = get_connection_id_override(); - let ws_connection = tokio::task::block_in_place(|| { - handle.block_on(WsConnection::connect( - self.uri.unwrap(), - self.database_name.as_ref().unwrap(), - self.token.as_deref(), - connection_id_override, - self.params, - )) - }) + let ws_connection = WsConnection::connect( + self.uri.unwrap(), + self.database_name.as_ref().unwrap(), + self.token.as_deref(), + connection_id_override, + self.params, + ) + .await .map_err(|source| crate::Error::FailedToConnect { source: InternalError::new("Failed to initiate WebSocket connection").with_cause(source), })?; @@ -998,7 +1099,7 @@ but you must call one of them, or else the connection will never progress. let (pending_mutations_send, pending_mutations_recv) = mpsc::unbounded(); let pending_mutations_recv = Arc::new(TokioMutex::new(pending_mutations_recv)); - let inner_ctx = build_db_ctx_inner(runtime, self.on_connect, self.on_connect_error, self.on_disconnect); + let inner_ctx = build_db_ctx_inner(None, self.on_connect, self.on_connect_error, self.on_disconnect); Ok(build_db_ctx( handle, inner_ctx, @@ -1622,3 +1723,6 @@ pub(crate) fn next_query_set_id() -> QuerySetId { id: NEXT_QUERY_SET_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed), } } + +#[cfg(all(test, not(feature = "browser")))] +mod terminal_tests; diff --git a/sdks/rust/src/db_connection/builder_tests.rs b/sdks/rust/src/db_connection/builder_tests.rs new file mode 100644 index 00000000000..7158c02b9e5 --- /dev/null +++ b/sdks/rust/src/db_connection/builder_tests.rs @@ -0,0 +1,89 @@ +use super::terminal_tests::bindings::RemoteModule; +use super::*; +use std::{net::SocketAddr, time::Duration}; +use tokio::{io::AsyncReadExt, net::TcpListener, sync::oneshot, task::JoinHandle}; + +async fn stalled_handshake_peer() -> (SocketAddr, oneshot::Receiver<()>, JoinHandle<()>) { + // This owned numeric-loopback listener never completes the HTTP upgrade. + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (ready_tx, ready_rx) = oneshot::channel(); + let peer = tokio::spawn(async move { + let (mut socket, remote) = listener.accept().await.unwrap(); + assert!(remote.ip().is_loopback()); + let mut request = Vec::new(); + let mut buffer = [0u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = socket.read(&mut buffer).await.unwrap(); + assert_ne!(count, 0, "client closed before sending the upgrade request"); + request.extend_from_slice(&buffer[..count]); + assert!(request.len() <= 16 * 1024); + } + ready_tx.send(()).unwrap(); + assert_eq!( + socket.read(&mut buffer).await.unwrap(), + 0, + "pending handshake socket was retained" + ); + }); + (address, ready_rx, peer) +} + +async fn check_pending_handshake_cleanup(use_timeout: bool) { + let (address, ready, peer) = stalled_handshake_peer().await; + let capture = Arc::new(()); + let weak_capture = Arc::downgrade(&capture); + let mut connect = Box::pin( + DbConnectionBuilder::::new() + .with_uri(format!("http://{address}")) + .with_database_name("disposable-builder-handshake-test") + .on_connect(move |_, _, _| drop(capture)) + .build_async(), + ); + tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + result = &mut connect => panic!("stalled handshake unexpectedly completed: {:?}", result.err()), + result = ready => result.unwrap(), + } + }) + .await + .expect("async builder blocked the executor or never initiated its handshake"); + assert!(weak_capture.upgrade().is_some()); + + if use_timeout { + assert!(tokio::time::timeout(Duration::from_millis(30), connect).await.is_err()); + } else { + drop(connect); + } + assert!( + weak_capture.upgrade().is_none(), + "cancelled builder retained callback captures" + ); + tokio::time::timeout(Duration::from_secs(2), peer) + .await + .expect("cancelled handshake did not close its socket") + .unwrap(); +} + +// A current-thread runtime also proves the asynchronous builder does not use +// block_in_place or block_on while waiting for the peer's handshake response. +#[tokio::test] +async fn async_builder_timeout_closes_stalled_handshake_and_releases_callbacks() { + check_pending_handshake_cleanup(true).await; +} + +#[tokio::test] +async fn async_builder_cancellation_closes_stalled_handshake_and_releases_callbacks() { + check_pending_handshake_cleanup(false).await; +} + +#[test] +fn sync_builder_returns_connect_errors_without_dropping_its_runtime_inside_block_on() { + // The SDK rejects a query in the host URI before attempting any network I/O. + // Running outside Tokio exercises the synchronous API's owned runtime. + let result = DbConnectionBuilder::::new() + .with_uri("http://127.0.0.1:1/?unexpected=query") + .with_database_name("disposable-builder-error-test") + .build(); + assert!(matches!(result, Err(crate::Error::FailedToConnect { .. }))); +} diff --git a/sdks/rust/src/db_connection/terminal_tests.rs b/sdks/rust/src/db_connection/terminal_tests.rs new file mode 100644 index 00000000000..397c36a34ae --- /dev/null +++ b/sdks/rust/src/db_connection/terminal_tests.rs @@ -0,0 +1,250 @@ +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[path = "../../tests/connect_disconnect_client/src/module_bindings/mod.rs"] +pub(super) mod bindings; +use bindings::RemoteModule; + +#[derive(spacetimedb_lib::ser::Serialize)] +#[sats(crate = spacetimedb_lib)] +struct Args {} +impl InModule for Args { + type Module = RemoteModule; +} +impl From for bindings::Reducer { + fn from(_: Args) -> Self { + Self::IdentityConnected + } +} + +struct DropProbe { + context: DbContextImpl, + drops: Arc, +} +impl Drop for DropProbe { + fn drop(&mut self) { + assert!( + self.context.inner.try_lock().is_ok(), + "capture dropped under inner lock" + ); + assert!( + self.context.send_chan.try_lock().is_ok(), + "capture dropped under send lock" + ); + assert!( + self.context.pending_mutations_recv.try_lock().is_ok(), + "capture dropped under queue lock" + ); + self.drops.fetch_add(1, Ordering::SeqCst); + } +} + +fn fixture( + runtime: &Runtime, + disconnects: Arc, +) -> ( + DbContextImpl, + mpsc::UnboundedSender>, +) { + let inner = build_db_ctx_inner::( + None, + None, + None, + Some(Box::new(move |_, error| { + assert!(error.is_none()); + disconnects.fetch_add(1, Ordering::SeqCst); + })), + ); + inner.lock().unwrap().connection_lifecycle = ConnectionLifecycle::Connected; + let (outgoing, outgoing_recv) = mpsc::unbounded(); + // Keep the transport receiver alive without any network or server fixture. + let (incoming, incoming_recv) = mpsc::unbounded(); + let (pending, pending_recv) = mpsc::unbounded(); + let context = build_db_ctx( + runtime.handle().clone(), + inner, + outgoing, + Arc::new(TokioMutex::new(incoming_recv)), + pending, + Arc::new(TokioMutex::new(pending_recv)), + Some(ConnectionId::from_u128(1)), + None, + ); + runtime.spawn(async move { + let mut outgoing_recv = outgoing_recv; + while outgoing_recv.next().await.is_some() {} + }); + (context, incoming) +} + +fn queue_calls(context: &DbContextImpl, drops: &Arc) { + let reducer = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context + .invoke_reducer_with_callback(Args {}, move |_, _| { + let _capture = reducer; + panic!("unknown-outcome reducer must not report completion"); + }) + .unwrap(); + let procedure = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context.invoke_procedure_with_callback::<_, ()>("procedure", Args {}, move |_, _| { + let _capture = procedure; + panic!("unknown-outcome procedure must not report completion"); + }); +} + +#[test] +fn terminal_disconnect_releases_inflight_and_queued_requests_with_retained_context() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, _incoming) = fixture(&runtime, disconnects.clone()); + queue_calls(&context, &drops); + context.frame_tick().unwrap(); + queue_calls(&context, &drops); + assert_eq!(drops.load(Ordering::SeqCst), 0); + assert!(matches!(context.end_connection(None), crate::Error::Disconnected)); + assert_eq!(drops.load(Ordering::SeqCst), 4); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); + assert!(!context.is_active()); + + let late = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + assert!(matches!( + context.invoke_reducer_with_callback(Args {}, move |_, _| drop(late)), + Err(crate::Error::Disconnected) + )); + let late = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context.invoke_procedure_with_callback::<_, ()>("procedure", Args {}, move |_, _| drop(late)); + assert_eq!(drops.load(Ordering::SeqCst), 6); + context.end_connection(None); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); +} + +#[test] +fn queued_disconnect_releases_calls_and_drains_late_results_until_terminal_event() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, incoming) = fixture(&runtime, disconnects.clone()); + queue_calls(&context, &drops); + context.frame_tick().unwrap(); + context.disconnect().unwrap(); + queue_calls(&context, &drops); + incoming + .unbounded_send(ParsedMessage::ReducerResult { + request_id: u32::MAX, + timestamp: Timestamp::UNIX_EPOCH, + result: Ok(Ok(bindings::DbUpdate::default())), + }) + .unwrap(); + incoming + .unbounded_send(ParsedMessage::ProcedureResult { + request_id: u32::MAX, + result: Ok(Bytes::new()), + }) + .unwrap(); + context.frame_tick().unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 4); + assert_eq!( + disconnects.load(Ordering::SeqCst), + 0, + "local request is not the terminal callback" + ); + assert!(!context.is_active()); + drop(incoming); + assert!(matches!( + runtime.block_on(context.advance_one_message_async()), + Err(crate::Error::Disconnected) + )); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); +} + +#[test] +fn retained_table_and_subscription_handles_cannot_retain_callbacks_after_terminal() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, _incoming) = fixture(&runtime, disconnects); + let table = context.get_table::("connected"); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + table.on_insert(move |_, _| { + let _ = &capture; + }); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let registered = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.frame_tick().unwrap(); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let queued = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.end_connection(None); + assert_eq!(drops.load(Ordering::SeqCst), 3); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(®istered)); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&queued)); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + table.on_insert(move |_, _| { + let _ = &capture; + }); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let subscription = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + assert_eq!(drops.load(Ordering::SeqCst), 5); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&subscription)); + assert_eq!(table.iter().count(), 0); +} + +#[test] +fn cancelling_before_initial_message_releases_registered_subscription_without_callbacks() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, incoming) = fixture(&runtime, disconnects.clone()); + context.inner.lock().unwrap().connection_lifecycle = ConnectionLifecycle::Connecting; + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let subscription = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.frame_tick().unwrap(); + context.disconnect().unwrap(); + context.frame_tick().unwrap(); + drop(incoming); + assert!(matches!( + runtime.block_on(context.advance_one_message_async()), + Err(crate::Error::Disconnected) + )); + assert_eq!(disconnects.load(Ordering::SeqCst), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&subscription)); +} diff --git a/sdks/rust/src/lib.rs b/sdks/rust/src/lib.rs index d99447d440f..2a8ec759600 100644 --- a/sdks/rust/src/lib.rs +++ b/sdks/rust/src/lib.rs @@ -10,6 +10,9 @@ // code generated by the CLI's codegen references them, // but users should not. +#[cfg(test)] +extern crate self as spacetimedb_sdk; + mod callbacks; mod client_cache; mod compression; diff --git a/sdks/rust/src/subscription.rs b/sdks/rust/src/subscription.rs index 2204985ea15..3fb4be3b5b2 100644 --- a/sdks/rust/src/subscription.rs +++ b/sdks/rust/src/subscription.rs @@ -47,23 +47,11 @@ pub(crate) enum PendingUnsubscribeResult { impl SubscriptionManager { pub(crate) fn on_disconnect(&mut self, _ctx: &M::ErrorContext) { - // We need to clear all the subscriptions. - // TODO: is this correct? We don't remove them from the client cache, - // we may want to resume them in the future if we impl reconnecting, - // and users can already register on-disconnect callbacks which will run in this case. - - // NOTE(cloutiertyler) - // This function previously invoke `on_error` for all subscriptions. - // However, this is inconsistent behavior given that `on_disconnect` for - // connections no longer always has an error argument and that the user - // can add an `on_ended` callback when unsubscribing. - // - // We propose instead that `on_ended` be added to the subscription - // builder so that it can be invoked when the subscription is ended - // because of a normal disconnect, but without the user calling - // `unsubscribe_then`. This can be done in a non-breaking way. - // - // For now, we will just do nothing when a subscription ends normally. + // Disconnect does not synthesize on_error/on_ended callbacks. Retained + // handles must nevertheless stop retaining callback captures. + for (_, handle) in std::mem::take(&mut self.subscriptions) { + handle.cancel_pending_callbacks(); + } } /// Register a new subscription. This does not send the subscription to the server. @@ -187,13 +175,17 @@ impl SubscriptionBuilder { self.on_applied, self.on_error, )); - self.conn + if self + .conn .pending_mutations_send .unbounded_send(PendingMutation::Subscribe { query_set_id, handle: handle.clone(), }) - .unwrap(); + .is_err() + { + handle.cancel_pending_callbacks(); + } M::SubscriptionHandle::new(handle) } @@ -471,6 +463,16 @@ impl SubscriptionHandleImpl { } } + pub(crate) fn cancel_pending_callbacks(&self) { + let callbacks = { + let mut inner = self.inner.lock().unwrap(); + inner.status = SubscriptionServerState::Ended; + (inner.on_applied.take(), inner.on_error.take(), inner.on_ended.take()) + }; + // Callback destructors may inspect or unsubscribe this same handle. + drop(callbacks); + } + pub(crate) fn start(&self) -> Option { let mut inner = self.inner.lock().unwrap(); inner.start() diff --git a/sdks/rust/tests/procedure-client/src/test_handlers.rs b/sdks/rust/tests/procedure-client/src/test_handlers.rs index d3f75c0698a..2dda622bf21 100644 --- a/sdks/rust/tests/procedure-client/src/test_handlers.rs +++ b/sdks/rust/tests/procedure-client/src/test_handlers.rs @@ -1,7 +1,7 @@ use crate::module_bindings::*; use anyhow::Context; use core::time::Duration; -use spacetimedb_lib::db::raw_def::v9::{RawMiscModuleExportV9, RawModuleDefV9}; +use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; use spacetimedb_sdk::{DbConnectionBuilder, DbContext, Table}; use test_counter::TestCounter; @@ -249,7 +249,7 @@ async fn exec_insert_with_tx_rollback(db_name: &str) { /// Test that a procedure can perform an HTTP request and return a string derived from the response. /// /// Invoke the procedure `read_my_schema`, -/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV9`], +/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV10`], /// then (in the client) deserialize the response and assert that it contains a description of that procedure. async fn exec_procedure_http_ok(db_name: &str) { let test_counter = TestCounter::new(); @@ -263,12 +263,14 @@ async fn exec_procedure_http_ok(db_name: &str) { #[allow(clippy::redundant_closure_call)] (|| { anyhow::ensure!(res.is_ok(), "Expected Ok result but got {res:?}"); - let module_def: RawModuleDefV9 = spacetimedb_lib::de::serde::deserialize_from( + let module_def: RawModuleDefV10 = spacetimedb_lib::de::serde::deserialize_from( &mut serde_json::Deserializer::from_str(&res.unwrap()), )?; - anyhow::ensure!(module_def.misc_exports.iter().any(|misc_export| { - if let RawMiscModuleExportV9::Procedure(procedure_def) = misc_export { - &*procedure_def.name == "read_my_schema" + anyhow::ensure!(module_def.sections.iter().any(|section| { + if let RawModuleDefV10Section::Procedures(procedures) = section { + procedures + .iter() + .any(|procedure| &*procedure.source_name == "read_my_schema") } else { false }