From bb49112515858895f7242f200e8ca92408901f36 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 16:19:52 -0400 Subject: [PATCH 01/16] Add Container Hosting Stage 1 runtime and publishing foundation --- Cargo.lock | 34 + Cargo.toml | 2 + crates/auth/src/hosted.rs | 393 +++++++ crates/auth/src/hosted/expiration_tests.rs | 136 +++ crates/auth/src/identity.rs | 22 +- crates/auth/src/lib.rs | 1 + crates/cli/Cargo.toml | 10 + crates/cli/docs/container-build.md | 226 ++++ crates/cli/docs/container-operations.md | 39 + crates/cli/src/container/config.rs | 122 +++ crates/cli/src/container/mod.rs | 395 +++++++ crates/cli/src/container/oci.rs | 262 +++++ crates/cli/src/container/process.rs | 212 ++++ crates/cli/src/container/publish/client.rs | 434 ++++++++ crates/cli/src/container/publish/journal.rs | 336 ++++++ crates/cli/src/container/publish/mod.rs | 315 ++++++ crates/cli/src/container/publish/tests.rs | 628 +++++++++++ crates/cli/src/container/tests.rs | 720 +++++++++++++ crates/cli/src/lib.rs | 20 + crates/cli/src/main.rs | 15 + crates/cli/src/spacetime_config.rs | 21 + crates/cli/src/subcommands/container.rs | 186 ++++ .../src/subcommands/container/operations.rs | 315 ++++++ crates/cli/src/subcommands/container/url.rs | 172 +++ .../src/subcommands/container/url/tests.rs | 198 ++++ crates/cli/src/subcommands/dev.rs | 4 + crates/cli/src/subcommands/mod.rs | 1 + crates/cli/src/subcommands/publish.rs | 108 +- crates/cli/src/subcommands/publish/managed.rs | 975 ++++++++++++++++++ crates/cli/src/subcommands/subscribe.rs | 37 +- .../container_build_acceptance/README.md | 77 ++ .../container_build_acceptance/acceptance.py | 469 +++++++++ .../test_fixture.py | 64 ++ .../container_build_acceptance/tool-lock.json | 31 + crates/cli/tests/real_container_builder.rs | 80 ++ crates/client-api/src/auth.rs | 47 +- crates/client-api/src/lib.rs | 41 +- crates/client-api/src/routes/database.rs | 15 +- crates/codegen/src/util.rs | 5 +- crates/codegen/tests/codegen.rs | 42 + crates/core/Cargo.toml | 1 + crates/core/src/auth/hosted_tokens.rs | 332 ++++++ crates/core/src/auth/invocation.rs | 127 +++ crates/core/src/auth/invocation/tests.rs | 199 ++++ crates/core/src/auth/mod.rs | 2 + crates/core/src/auth/token_validation.rs | 13 + crates/core/src/client/client_connection.rs | 528 +++++++++- crates/core/src/config.rs | 51 + crates/core/src/db/container_environment.rs | 354 +++++++ .../src/db/container_environment/tests.rs | 427 ++++++++ crates/core/src/db/deployment.rs | 586 +++++++++++ crates/core/src/db/deployment/tests.rs | 633 ++++++++++++ crates/core/src/db/mod.rs | 9 + crates/core/src/db/retained_shutdown_tests.rs | 50 + crates/core/src/error.rs | 2 + crates/core/src/host/container_environment.rs | 188 ++++ .../container_environment/durability_tests.rs | 291 ++++++ crates/core/src/host/container_fence.rs | 204 ++++ crates/core/src/host/container_fence/tests.rs | 392 +++++++ crates/core/src/host/empty_module.rs | 47 + crates/core/src/host/empty_module/README.md | 23 + crates/core/src/host/empty_module/generate.py | 113 ++ crates/core/src/host/empty_module/tests.rs | 137 +++ .../src/host/empty_module/v1.schema.bsatn | Bin 0 -> 33 bytes crates/core/src/host/empty_module/v1.sha256 | 1 + crates/core/src/host/empty_module/v1.wasm | Bin 0 -> 250 bytes crates/core/src/host/host_controller.rs | 757 +++++++------- .../host/host_controller/deployment_tests.rs | 215 ++++ .../execution_deadline_tests.rs | 413 ++++++++ .../src/host/host_controller/lifecycle.rs | 157 +++ .../host/host_controller/lifecycle_tests.rs | 403 ++++++++ .../core/src/host/host_controller/registry.rs | 333 ++++++ crates/core/src/host/instance_env.rs | 144 ++- crates/core/src/host/mod.rs | 5 +- crates/core/src/host/module_common.rs | 2 +- crates/core/src/host/module_host.rs | 557 +++++++--- .../core/src/host/module_host/drain_tests.rs | 397 +++++++ .../core/src/host/module_host/operations.rs | 126 +++ crates/core/src/host/scheduler.rs | 91 ++ crates/core/src/host/v8/execution_deadline.rs | 272 +++++ crates/core/src/host/v8/mod.rs | 295 ++++-- crates/core/src/host/v8/syscall/common.rs | 21 +- crates/core/src/host/v8/syscall/v1.rs | 1 + crates/core/src/host/v8/syscall/v2.rs | 1 + .../src/host/wasm_common/module_host_actor.rs | 188 +++- .../src/host/wasmtime/wasm_instance_env.rs | 81 +- .../core/src/host/wasmtime/wasmtime_module.rs | 2 + crates/core/src/sql/execute.rs | 48 +- .../subscription/module_subscription_actor.rs | 86 +- .../module_subscription_manager.rs | 8 +- .../locking_tx_datastore/committed_state.rs | 3 + .../src/locking_tx_datastore/datastore.rs | 51 +- .../src/locking_tx_datastore/mut_tx.rs | 15 +- crates/datastore/src/system_tables.rs | 22 +- .../datastore/src/system_tables/deployment.rs | 253 +++++ .../src/system_tables/environment.rs | 4 +- crates/engine/src/durability.rs | 20 +- crates/engine/src/error.rs | 2 + crates/engine/src/hosted_admission.rs | 282 +++++ crates/engine/src/lib.rs | 1 + crates/engine/src/relational_db.rs | 105 +- .../src/relational_db/shutdown_tests.rs | 65 ++ crates/engine/src/snapshot.rs | 81 +- crates/engine/src/sql/ast.rs | 3 + crates/lib/Cargo.toml | 1 + crates/lib/src/container.rs | 478 +++++++++ crates/lib/src/container/endpoints.rs | 57 + crates/lib/src/container/operations.rs | 199 ++++ crates/lib/src/container/tests.rs | 246 +++++ crates/lib/src/container_environment.rs | 40 + crates/lib/src/deployment.rs | 256 +++++ crates/lib/src/deployment/api.rs | 117 +++ crates/lib/src/deployment/manifest.rs | 172 +++ .../lib/src/deployment/system_empty_v1.wasm | Bin 0 -> 250 bytes crates/lib/src/deployment/tests.rs | 128 +++ crates/lib/src/lib.rs | 3 + crates/oci/Cargo.toml | 20 + crates/oci/src/layers.rs | 373 +++++++ crates/oci/src/layers/tests.rs | 130 +++ crates/oci/src/lib.rs | 377 +++++++ crates/oci/src/tests.rs | 166 +++ crates/standalone/config.toml | 4 + crates/testing/Cargo.toml | 2 + crates/testing/src/modules.rs | 6 + crates/testing/tests/deployment_publish.rs | 204 ++++ crates/testing/tests/hosted_invocation.rs | 261 +++++ modules/hosted-auth-test/Cargo.toml | 13 + modules/hosted-auth-test/src/lib.rs | 141 +++ sdks/rust/src/client_cache.rs | 7 +- sdks/rust/src/client_cache/uuid_tests.rs | 75 ++ sdks/rust/src/db_connection.rs | 180 +++- sdks/rust/src/db_connection/builder_tests.rs | 89 ++ sdks/rust/src/db_connection/terminal_tests.rs | 250 +++++ sdks/rust/src/lib.rs | 3 + sdks/rust/src/subscription.rs | 40 +- 135 files changed, 20629 insertions(+), 839 deletions(-) create mode 100644 crates/auth/src/hosted.rs create mode 100644 crates/auth/src/hosted/expiration_tests.rs create mode 100644 crates/cli/docs/container-build.md create mode 100644 crates/cli/docs/container-operations.md create mode 100644 crates/cli/src/container/config.rs create mode 100644 crates/cli/src/container/mod.rs create mode 100644 crates/cli/src/container/oci.rs create mode 100644 crates/cli/src/container/process.rs create mode 100644 crates/cli/src/container/publish/client.rs create mode 100644 crates/cli/src/container/publish/journal.rs create mode 100644 crates/cli/src/container/publish/mod.rs create mode 100644 crates/cli/src/container/publish/tests.rs create mode 100644 crates/cli/src/container/tests.rs create mode 100644 crates/cli/src/subcommands/container.rs create mode 100644 crates/cli/src/subcommands/container/operations.rs create mode 100644 crates/cli/src/subcommands/container/url.rs create mode 100644 crates/cli/src/subcommands/container/url/tests.rs create mode 100644 crates/cli/src/subcommands/publish/managed.rs create mode 100644 crates/cli/tests/container_build_acceptance/README.md create mode 100644 crates/cli/tests/container_build_acceptance/acceptance.py create mode 100644 crates/cli/tests/container_build_acceptance/test_fixture.py create mode 100644 crates/cli/tests/container_build_acceptance/tool-lock.json create mode 100644 crates/cli/tests/real_container_builder.rs create mode 100644 crates/core/src/auth/hosted_tokens.rs create mode 100644 crates/core/src/auth/invocation.rs create mode 100644 crates/core/src/auth/invocation/tests.rs create mode 100644 crates/core/src/db/container_environment.rs create mode 100644 crates/core/src/db/container_environment/tests.rs create mode 100644 crates/core/src/db/deployment.rs create mode 100644 crates/core/src/db/deployment/tests.rs create mode 100644 crates/core/src/db/retained_shutdown_tests.rs create mode 100644 crates/core/src/host/container_environment.rs create mode 100644 crates/core/src/host/container_environment/durability_tests.rs create mode 100644 crates/core/src/host/container_fence.rs create mode 100644 crates/core/src/host/container_fence/tests.rs create mode 100644 crates/core/src/host/empty_module.rs create mode 100644 crates/core/src/host/empty_module/README.md create mode 100644 crates/core/src/host/empty_module/generate.py create mode 100644 crates/core/src/host/empty_module/tests.rs create mode 100644 crates/core/src/host/empty_module/v1.schema.bsatn create mode 100644 crates/core/src/host/empty_module/v1.sha256 create mode 100644 crates/core/src/host/empty_module/v1.wasm create mode 100644 crates/core/src/host/host_controller/deployment_tests.rs create mode 100644 crates/core/src/host/host_controller/execution_deadline_tests.rs create mode 100644 crates/core/src/host/host_controller/lifecycle.rs create mode 100644 crates/core/src/host/host_controller/lifecycle_tests.rs create mode 100644 crates/core/src/host/host_controller/registry.rs create mode 100644 crates/core/src/host/module_host/drain_tests.rs create mode 100644 crates/core/src/host/module_host/operations.rs create mode 100644 crates/core/src/host/v8/execution_deadline.rs create mode 100644 crates/datastore/src/system_tables/deployment.rs create mode 100644 crates/engine/src/hosted_admission.rs create mode 100644 crates/engine/src/relational_db/shutdown_tests.rs create mode 100644 crates/lib/src/container.rs create mode 100644 crates/lib/src/container/endpoints.rs create mode 100644 crates/lib/src/container/operations.rs create mode 100644 crates/lib/src/container/tests.rs create mode 100644 crates/lib/src/container_environment.rs create mode 100644 crates/lib/src/deployment.rs create mode 100644 crates/lib/src/deployment/api.rs create mode 100644 crates/lib/src/deployment/manifest.rs create mode 100644 crates/lib/src/deployment/system_empty_v1.wasm create mode 100644 crates/lib/src/deployment/tests.rs create mode 100644 crates/oci/Cargo.toml create mode 100644 crates/oci/src/layers.rs create mode 100644 crates/oci/src/layers/tests.rs create mode 100644 crates/oci/src/lib.rs create mode 100644 crates/oci/src/tests.rs create mode 100644 crates/testing/tests/deployment_publish.rs create mode 100644 crates/testing/tests/hosted_invocation.rs create mode 100644 modules/hosted-auth-test/Cargo.toml create mode 100644 modules/hosted-auth-test/src/lib.rs create mode 100644 sdks/rust/src/client_cache/uuid_tests.rs create mode 100644 sdks/rust/src/db_connection/builder_tests.rs create mode 100644 sdks/rust/src/db_connection/terminal_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 0020e4b5c2b..b1a6cab1682 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3109,6 +3109,13 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hosted-auth-test" +version = "0.0.0" +dependencies = [ + "spacetimedb", +] + [[package]] name = "http" version = "1.3.1" @@ -7919,6 +7926,7 @@ name = "spacetimedb-cli" version = "2.10.0" dependencies = [ "anyhow", + "axum", "base64 0.21.7", "bytes", "cargo_metadata", @@ -7953,20 +7961,24 @@ dependencies = [ "rolldown_common", "rolldown_error", "rolldown_utils", + "rustix 1.1.2", "rustyline", "serde", "serde_json", "serde_with", + "sha2", "spacetimedb-auth", "spacetimedb-client-api-messages", "spacetimedb-codegen", "spacetimedb-data-structures", "spacetimedb-fs-utils", "spacetimedb-lib", + "spacetimedb-oci", "spacetimedb-paths", "spacetimedb-schema", "syntect", "tabled", + "tar", "tempfile", "termcolor", "termtree", @@ -7974,8 +7986,11 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-tungstenite 0.27.0", + "tokio-util", "toml 0.8.23", "toml_edit 0.22.27", + "url", + "uuid", "walkdir", "wasmbin", "webbrowser", @@ -8200,6 +8215,7 @@ dependencies = [ "tracing-subscriber", "tracing-tracy", "url", + "uuid", "v8", "wasmtime", "wasmtime-internal-fiber", @@ -8420,6 +8436,7 @@ dependencies = [ "spacetimedb-metrics", "spacetimedb-primitives", "spacetimedb-sats", + "thiserror 1.0.69", ] [[package]] @@ -8442,6 +8459,20 @@ dependencies = [ "prometheus", ] +[[package]] +name = "spacetimedb-oci" +version = "2.10.0" +dependencies = [ + "anyhow", + "flate2", + "serde", + "serde_json", + "sha2", + "spacetimedb-lib", + "tar", + "zstd", +] + [[package]] name = "spacetimedb-paths" version = "2.10.0" @@ -8817,11 +8848,13 @@ dependencies = [ "serde", "serde_json", "serial_test", + "spacetimedb-auth", "spacetimedb-cli", "spacetimedb-client-api", "spacetimedb-client-api-messages", "spacetimedb-core", "spacetimedb-data-structures", + "spacetimedb-datastore", "spacetimedb-guard", "spacetimedb-lib", "spacetimedb-paths", @@ -10205,6 +10238,7 @@ checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.4", "js-sys", + "serde", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index ba682e45ba5..f0e3c77b2f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/fs-utils", "crates/index-scan-gate", "crates/lib", + "crates/oci", "crates/metrics", "crates/paths", "crates/pg", @@ -46,6 +47,7 @@ members = [ "modules/keynote-benchmarks", "modules/perf-test", "modules/module-test", + "modules/hosted-auth-test", "modules/environment-test", "modules/invocation-flags-test", "templates/basic-rs/spacetimedb", diff --git a/crates/auth/src/hosted.rs b/crates/auth/src/hosted.rs new file mode 100644 index 00000000000..8601448bc87 --- /dev/null +++ b/crates/auth/src/hosted.rs @@ -0,0 +1,393 @@ +//! Target-bound credentials for a database's hosted container. +//! +//! Decoded claims are untrusted. Only signature verification against an explicitly +//! configured platform issuer and comparison with an authoritative instance/grant +//! binding can produce [`VerifiedHostedAuth`]. Receiving hosts must additionally +//! recheck their durable target fence at each transaction and subscription admission. + +use crate::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; +use anyhow::{bail, ensure, Context}; +use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, EncodingKey, Header, Validation}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::Identity; +use std::fmt; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +pub const HOSTED_TOKEN_KIND: &str = "spacetimedb_hosted_v1"; +pub const HOSTED_TOKEN_TYPE: &str = "spacetimedb-hosted+jwt"; +pub const MAX_HOSTED_TOKEN_LIFETIME: Duration = Duration::from_secs(30); +pub const MAX_HOSTED_TOKEN_BYTES: usize = 8192; + +/// Wire claims, deliberately distinct from ordinary issuer/subject-derived Identity claims. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostedTokenClaims { + pub kind: Box, + #[serde(rename = "iss")] + pub issuer: Box, + #[serde(rename = "sub")] + pub subject: Box, + #[serde(with = "identity_hex")] + pub source_database: Identity, + /// A scalar, canonical database Identity. Lists and database names are not accepted. + #[serde(rename = "aud", with = "identity_hex")] + pub target_database: Identity, + pub generation: u64, + pub grant_revision: u64, + #[serde(rename = "iat")] + pub issued_at: u64, + #[serde(rename = "exp")] + pub expires_at: u64, + #[serde(rename = "jti")] + pub token_id: Box, +} + +/// Trusted input obtained from current source registration, placement and target grant state. +/// Never construct this by copying the incoming token's claims. Admission must be open, +/// and the assigned node/incarnation and required module capability must already be checked. +#[derive(Clone, Copy, Debug)] +pub struct HostedTokenBinding { + pub source_database: Identity, + pub target_database: Identity, + pub generation: u64, + pub grant_revision: u64, + pub lease_expires_at: SystemTime, +} + +/// Authentication proof. It cannot be deserialized or constructed from decoded claims. +#[derive(Clone)] +pub struct VerifiedHostedAuth { + claims: HostedTokenClaims, + // Local lifetime state only. Never serialized into signed claims. + monotonic_deadline: Instant, +} + +impl fmt::Debug for VerifiedHostedAuth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VerifiedHostedAuth") + .field("source_database", &self.source_database()) + .field("target_database", &self.target_database()) + .field("generation", &self.generation()) + .field("grant_revision", &self.grant_revision()) + .field("expires_at", &self.expires_at()) + .finish_non_exhaustive() + } +} + +impl VerifiedHostedAuth { + pub fn source_database(&self) -> Identity { + self.claims.source_database + } + pub fn target_database(&self) -> Identity { + self.claims.target_database + } + pub fn generation(&self) -> u64 { + self.claims.generation + } + pub fn grant_revision(&self) -> u64 { + self.claims.grant_revision + } + pub fn issued_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.claims.issued_at) + } + pub fn expires_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.claims.expires_at) + } + pub fn token_id(&self) -> &str { + &self.claims.token_id + } + pub fn issuer(&self) -> &str { + &self.claims.issuer + } + pub fn is_internal(&self) -> bool { + self.source_database() == self.target_database() + } + + /// Expiry has no positive leeway. This does not replace generation/grant fencing. + pub fn check_at(&self, now: SystemTime) -> anyhow::Result<()> { + self.check_at_clocks(now, Instant::now()) + } + + fn check_at_clocks(&self, now: SystemTime, monotonic_now: Instant) -> anyhow::Result<()> { + ensure!(now >= self.issued_at(), "hosted credential is not yet valid"); + ensure!( + now < self.expires_at() && monotonic_now < self.monotonic_deadline, + "hosted credential expired" + ); + Ok(()) + } + + /// Cap the local lifetime using a fresh authority confirmation. The caller + /// records `confirmation_started` before sending the request and obtains + /// `confirmed_time` from the successful authoritative response. Charging the + /// entire round trip against the signed remaining lifetime is conservative. + /// Neither a receiving clock behind authority nor a later confirmation can + /// extend this proof. The original signed claims are preserved exactly. + pub fn constrain_expiration( + mut self, + confirmed_time: SystemTime, + confirmation_started: Instant, + ) -> anyhow::Result { + let now = Instant::now(); + ensure!(confirmation_started <= now, "invalid hosted confirmation clock"); + self.check_at_clocks(confirmed_time, now)?; + let remaining = self.expires_at().duration_since(confirmed_time)?; + let confirmed_deadline = confirmation_started + .checked_add(remaining) + .context("hosted confirmation deadline overflow")?; + self.monotonic_deadline = self.monotonic_deadline.min(confirmed_deadline); + self.check_at_clocks(confirmed_time, now)?; + Ok(self) + } + + /// Remaining connection lifetime, bounded by both signed wall-clock expiry + /// and the already captured local deadline. Repeated calls never reset it. + pub fn remaining_lifetime(&self, now: SystemTime) -> Duration { + self.remaining_lifetime_at_clocks(now, Instant::now()) + } + + fn remaining_lifetime_at_clocks(&self, now: SystemTime, monotonic_now: Instant) -> Duration { + if self.check_at_clocks(now, monotonic_now).is_err() { + return Duration::ZERO; + } + self.expires_at() + .duration_since(now) + .unwrap_or_default() + .min(self.monotonic_deadline.saturating_duration_since(monotonic_now)) + } + + pub fn into_connection_auth(self) -> anyhow::Result { + // Keep the actual claims, including the source/target/generation restrictions. + // Normalizing JSON whitespace does not alter any signed claim values. + let jwt_payload = serde_json::to_string(&self.claims)?.into_boxed_str(); + let mut extra = serde_json::to_value(&self.claims)?; + let extra = extra.as_object_mut().expect("hosted claims serialize as an object"); + for key in ["iss", "sub", "aud", "iat", "exp"] { + extra.remove(key); + } + let claims = SpacetimeIdentityClaims { + identity: self.source_database(), + subject: self.claims.subject.clone(), + issuer: self.claims.issuer.clone(), + audience: [self.target_database().to_hex().to_string().into_boxed_str()].into(), + iat: self.issued_at(), + exp: Some(self.expires_at()), + extra: Some( + extra + .iter() + .map(|(key, value)| (key.clone().into_boxed_str(), value.clone())) + .collect(), + ), + }; + Ok(ConnectionAuthCtx { + claims, + jwt_payload, + hosted: Some(self), + }) + } +} + +/// Classifies the reserved namespace only. A positive result grants no authority. +/// All reserved versions are rejected by ordinary OIDC validation and token exchange. +pub fn has_reserved_hosted_token_kind(token: &str) -> anyhow::Result { + classify_reserved_token(token, is_reserved_hosted_type, is_reserved_hosted_kind) +} + +/// Operational container proofs are never client Identity credentials. Reserve +/// their entire versioned namespace so a lease/registry proof cannot enter +/// OIDC discovery, ordinary JWT validation, or the Identity token exchange. +pub fn has_reserved_platform_token_kind(token: &str) -> anyhow::Result { + classify_reserved_token( + token, + |kind| is_reserved_hosted_type(kind) || kind.starts_with("spacetimedb-container-"), + |kind| is_reserved_hosted_kind(kind) || kind.starts_with("spacetimedb_container_"), + ) +} + +fn classify_reserved_token( + token: &str, + reserved_type: impl FnOnce(&str) -> bool, + reserved_kind: impl FnOnce(&str) -> bool, +) -> anyhow::Result { + let header = decode_header(token)?; + if header.typ.as_deref().is_some_and(reserved_type) { + return Ok(true); + } + let data = jsonwebtoken::dangerous::insecure_decode::(token)?; + Ok(data + .claims + .get("kind") + .and_then(serde_json::Value::as_str) + .is_some_and(reserved_kind)) +} + +pub fn is_reserved_hosted_kind(kind: &str) -> bool { + kind.starts_with("spacetimedb_hosted_") +} +fn is_reserved_hosted_type(kind: &str) -> bool { + kind.starts_with("spacetimedb-hosted") +} + +/// Decode routing hints only, never authentication. The caller must use these hints to +/// find trusted registration/grant state and then call [`verify_hosted_token`]. +pub fn unverified_hosted_token_claims(token: &str) -> anyhow::Result { + ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); + Ok(jsonwebtoken::dangerous::insecure_decode::(token)?.claims) +} + +/// Verify against a configured key and issuer, never a JWT-supplied key or JWKS URL. +/// `binding` must be authoritative state for that issuer's registered source database. +pub fn verify_hosted_token( + token: &str, + public_key: &DecodingKey, + trusted_issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, +) -> anyhow::Result { + let verification_started = Instant::now(); + ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); + let header = decode_header(token)?; + ensure!(header.alg == Algorithm::ES256, "hosted credential requires ES256"); + ensure!( + header.typ.as_deref() == Some(HOSTED_TOKEN_TYPE), + "invalid hosted credential type" + ); + let mut validation = Validation::new(Algorithm::ES256); + validation.set_required_spec_claims(&["iss", "sub", "aud", "exp"]); + validation.set_issuer(&[trusted_issuer]); + validation.set_audience(&[binding.target_database.to_hex().to_string()]); + validation.leeway = 0; + // Check time below against the caller's trusted clock, including exact expiry equality. + validation.validate_exp = false; + let claims = decode::(token, public_key, &validation)?.claims; + validate_claims(&claims, trusted_issuer, binding, now)?; + let remaining = (UNIX_EPOCH + Duration::from_secs(claims.expires_at)).duration_since(now)?; + let monotonic_deadline = verification_started + .checked_add(remaining) + .context("hosted credential deadline overflow")?; + let proof = VerifiedHostedAuth { + claims, + monotonic_deadline, + }; + proof.check_at(now)?; + Ok(proof) +} + +/// Mint from the broker's authoritative binding, with no guest-selected sender or generation. +pub fn sign_hosted_token( + private_key: &EncodingKey, + trusted_issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, + expires_at: SystemTime, + token_id: &str, +) -> anyhow::Result { + let claims = HostedTokenClaims { + kind: HOSTED_TOKEN_KIND.into(), + issuer: trusted_issuer.into(), + subject: binding.source_database.to_hex().to_string().into_boxed_str(), + source_database: binding.source_database, + target_database: binding.target_database, + generation: binding.generation, + grant_revision: binding.grant_revision, + issued_at: unix_seconds(now)?, + expires_at: unix_seconds(expires_at)?, + token_id: token_id.into(), + }; + validate_claims(&claims, trusted_issuer, binding, now)?; + let mut header = Header::new(Algorithm::ES256); + header.typ = Some(HOSTED_TOKEN_TYPE.into()); + Ok(jsonwebtoken::encode(&header, &claims, private_key)?) +} + +fn validate_claims( + claims: &HostedTokenClaims, + issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, +) -> anyhow::Result<()> { + ensure!( + !issuer.is_empty() && issuer.len() <= 128, + "invalid trusted hosted issuer" + ); + ensure!( + claims.kind.as_ref() == HOSTED_TOKEN_KIND, + "unsupported hosted credential kind" + ); + ensure!(claims.issuer.as_ref() == issuer, "untrusted hosted credential issuer"); + ensure!( + claims.source_database == binding.source_database, + "hosted credential source mismatch" + ); + ensure!( + claims.target_database == binding.target_database, + "hosted credential target mismatch" + ); + ensure!( + claims.subject.as_ref() == claims.source_database.to_hex().as_str(), + "hosted credential subject mismatch" + ); + ensure!( + claims.generation == binding.generation, + "hosted credential generation mismatch" + ); + ensure!( + claims.grant_revision == binding.grant_revision, + "hosted credential grant revision mismatch" + ); + ensure!( + !claims.token_id.is_empty() + && claims.token_id.len() <= 128 + && claims + .token_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'), + "invalid hosted credential token ID" + ); + let Some(lifetime) = claims.expires_at.checked_sub(claims.issued_at) else { + bail!("invalid hosted credential lifetime") + }; + ensure!( + lifetime > 0 && lifetime <= MAX_HOSTED_TOKEN_LIFETIME.as_secs(), + "hosted credential lifetime exceeds limit" + ); + let issued_at = UNIX_EPOCH + .checked_add(Duration::from_secs(claims.issued_at)) + .context("invalid hosted issue time")?; + let expires_at = UNIX_EPOCH + .checked_add(Duration::from_secs(claims.expires_at)) + .context("invalid hosted expiry")?; + ensure!( + issued_at <= now && now < expires_at, + "hosted credential outside validity interval" + ); + ensure!( + expires_at <= binding.lease_expires_at, + "hosted credential exceeds confirmed lease" + ); + Ok(()) +} + +fn unix_seconds(time: SystemTime) -> anyhow::Result { + Ok(time.duration_since(UNIX_EPOCH)?.as_secs()) +} + +mod identity_hex { + use super::*; + pub fn serialize(value: &Identity, serializer: S) -> Result { + serializer.serialize_str(value.to_hex().as_str()) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(serde::de::Error::custom( + "expected canonical 64-character lowercase database Identity", + )); + } + Identity::from_hex(value).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +#[path = "hosted/expiration_tests.rs"] +mod expiration_tests; diff --git a/crates/auth/src/hosted/expiration_tests.rs b/crates/auth/src/hosted/expiration_tests.rs new file mode 100644 index 00000000000..1624bfc474f --- /dev/null +++ b/crates/auth/src/hosted/expiration_tests.rs @@ -0,0 +1,136 @@ +use super::*; + +fn proof() -> (VerifiedHostedAuth, SystemTime, Instant) { + let wall = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let monotonic = Instant::now(); + ( + VerifiedHostedAuth { + claims: HostedTokenClaims { + kind: HOSTED_TOKEN_KIND.into(), + issuer: "platform.test".into(), + subject: Identity::ZERO.to_hex().to_string().into(), + source_database: Identity::ZERO, + target_database: Identity::ZERO, + generation: 1, + grant_revision: 2, + issued_at: 1_700_000_000, + expires_at: 1_700_000_030, + token_id: "expiration-test".into(), + }, + monotonic_deadline: monotonic + Duration::from_secs(30), + }, + wall, + monotonic, + ) +} + +#[test] +fn receiving_clock_behind_control_cannot_extend_confirmed_lifetime() { + let (proof, wall, monotonic) = proof(); + let request_started = monotonic - Duration::from_secs(2); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), request_started) + .unwrap(); + let deadline = request_started + Duration::from_secs(5); + assert_eq!(proof.monotonic_deadline, deadline); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(3), deadline - Duration::from_nanos(1)) + .is_ok()); + assert!(proof.check_at_clocks(wall + Duration::from_secs(3), deadline).is_err()); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(3), deadline), + Duration::ZERO + ); +} + +#[test] +fn confirmation_round_trip_consumes_remaining_signed_lifetime() { + let (proof, wall, monotonic) = proof(); + assert!(proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic - Duration::from_secs(6)) + .is_err()); +} + +#[test] +fn later_confirmation_cannot_extend_a_previous_deadline() { + let (proof, wall, monotonic) = proof(); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic) + .unwrap(); + let first_deadline = proof.monotonic_deadline; + // Even a second response with an earlier authority timestamp cannot extend + // the stricter deadline already held by this authentication proof. + let proof = proof + .constrain_expiration(wall + Duration::from_secs(20), Instant::now()) + .unwrap(); + assert_eq!(proof.monotonic_deadline, first_deadline); +} + +#[test] +fn backward_wall_clock_does_not_restart_the_local_lifetime() { + let (proof, wall, monotonic) = proof(); + let later = monotonic + Duration::from_secs(29); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(1), later), + Duration::from_secs(1) + ); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(1), monotonic + Duration::from_secs(30)) + .is_err()); +} + +#[test] +fn wall_clock_expiration_remains_an_independent_limit() { + let (proof, wall, monotonic) = proof(); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(29), monotonic), + Duration::from_secs(1) + ); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(30), monotonic) + .is_err()); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall - Duration::from_secs(1), monotonic), + Duration::ZERO + ); +} + +#[test] +fn invalid_confirmation_clocks_are_rejected() { + let (proof, wall, monotonic) = proof(); + assert!(proof + .clone() + .constrain_expiration(wall - Duration::from_secs(1), monotonic) + .is_err()); + assert!(proof + .clone() + .constrain_expiration(wall + Duration::from_secs(30), monotonic) + .is_err()); + assert!(proof + .constrain_expiration(wall, Instant::now() + Duration::from_secs(60)) + .is_err()); +} + +#[test] +fn cloning_and_connection_conversion_preserve_the_cap_and_signed_claims() { + let (proof, wall, monotonic) = proof(); + let original_claims = serde_json::to_value(&proof.claims).unwrap(); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic) + .unwrap(); + let deadline = proof.monotonic_deadline; + let cloned = proof.clone(); + let connection = proof.into_connection_auth().unwrap(); + let connection = connection.clone(); + let retained = connection.hosted.as_ref().unwrap(); + assert_eq!(cloned.monotonic_deadline, deadline); + assert_eq!(retained.monotonic_deadline, deadline); + assert_eq!(connection.claims.exp, Some(wall + Duration::from_secs(30))); + assert_eq!( + serde_json::from_str::(&connection.jwt_payload).unwrap(), + original_claims + ); + assert!(retained + .check_at_clocks(wall + Duration::from_secs(1), deadline) + .is_err()); +} diff --git a/crates/auth/src/identity.rs b/crates/auth/src/identity.rs index 41b38d00cb7..380945f0bfb 100644 --- a/crates/auth/src/identity.rs +++ b/crates/auth/src/identity.rs @@ -7,10 +7,20 @@ use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::Identity; use std::time::SystemTime; -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ConnectionAuthCtx { pub claims: SpacetimeIdentityClaims, pub jwt_payload: Box, + pub hosted: Option, +} + +impl std::fmt::Debug for ConnectionAuthCtx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConnectionAuthCtx") + .field("identity", &self.claims.identity) + .field("hosted", &self.hosted) + .finish_non_exhaustive() + } } impl TryFrom for ConnectionAuthCtx { @@ -20,6 +30,7 @@ impl TryFrom for ConnectionAuthCtx { Ok(ConnectionAuthCtx { claims, jwt_payload: payload.into(), + hosted: None, }) } } @@ -103,6 +114,15 @@ impl TryInto for IncomingClaims { type Error = anyhow::Error; fn try_into(self) -> anyhow::Result { + if self + .extra + .as_ref() + .and_then(|extra| extra.get("kind")) + .and_then(serde_json::Value::as_str) + .is_some_and(crate::hosted::is_reserved_hosted_kind) + { + anyhow::bail!("hosted credentials require dedicated target-bound validation"); + } // The issuer and subject must be less than 128 bytes. if self.issuer.len() > 128 { return Err(anyhow::anyhow!("Issuer too long: {:?}", self.issuer)); diff --git a/crates/auth/src/lib.rs b/crates/auth/src/lib.rs index db53a0c9064..b8b0976f40d 100644 --- a/crates/auth/src/lib.rs +++ b/crates/auth/src/lib.rs @@ -1 +1,2 @@ +pub mod hosted; pub mod identity; diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 5ea9f718833..983b59b2e90 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -27,6 +27,7 @@ spacetimedb-codegen.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-fs-utils.workspace = true spacetimedb-lib.workspace = true +spacetimedb-oci = { path = "../oci" } spacetimedb-paths.workspace = true spacetimedb-schema.workspace = true @@ -59,13 +60,18 @@ serde_json = { workspace = true, features = ["raw_value", "preserve_order", "arb serde_with = { workspace = true, features = ["chrono_0_4"] } syntect.workspace = true tabled.workspace = true +tar.workspace = true tempfile.workspace = true +sha2 = "0.10" +tokio-util.workspace = true termcolor.workspace = true termtree.workspace = true thiserror.workspace = true tokio.workspace = true tokio-tungstenite.workspace = true toml_edit.workspace = true +url.workspace = true +uuid = { workspace = true, features = ["std", "serde", "v7"] } walkdir.workspace = true wasmbin.workspace = true webbrowser.workspace = true @@ -83,11 +89,15 @@ notify.workspace = true path-clean = "1.0.1" [dev-dependencies] +axum.workspace = true pretty_assertions.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { workspace = true } +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +rustix = { version = "1", features = ["fs", "process"] } + [target.'cfg(windows)'.dependencies] windows-sys = { workspace = true, features = ["Win32_System_Console"] } diff --git a/crates/cli/docs/container-build.md b/crates/cli/docs/container-build.md new file mode 100644 index 00000000000..71749eda520 --- /dev/null +++ b/crates/cli/docs/container-build.md @@ -0,0 +1,226 @@ +# Local container image preparation + +`spacetime container build` reads one database's `container` declaration from +`spacetime.json` and prepares a verified local OCI image. It does not resolve a +database through a server or publish anything. The optional `DATABASE` argument +selects an exact local configuration target. Omit it only when the configuration +contains one container target. Container declarations do not pass to children. +The command is dispatched before saved CLI server settings or credentials are +opened; only project configuration and explicitly selected build credentials +are read. + +The same verified preparation feeds managed `publish`, described below. +Container lifecycle commands remain separate integration work. + +## Configuration + +```json +{ + "database": "example", + "container": { + "image": { + "build": { + "builder": "dockerfile", + "context": ".", + "dockerfile": "Dockerfile" + } + }, + "env_keys": ["API_KEY"], + "resources": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824, + "scratch_bytes": 1073741824, + "pids_max": 256 + } + } +} +``` + +`image` accepts exactly one source: + +- `{"build":{"context":"."}}` selects Dockerfile by default. +- `{"build":{"builder":"railpack","context":"."}}` explicitly selects Railpack. +- `{"oci_ref":"registry.example/team/image:tag"}` imports a registry image. +- `{"oci_ref":"oci:./existing-layout"}` imports a local OCI image-layout directory. + +Build contexts and local layout paths are relative to the configuration +directory. The Dockerfile path is relative to its build context. A tag is +resolved once during import; the result records the selected immutable manifest +digest. Registry import requires Skopeo. A local directory import requires no +external image tools. + +The image's `Entrypoint` followed by `Cmd` supplies the command by default. +`command` replaces the complete argv. Image `User` and `WorkingDir` are preserved +unless `user` or `working_directory` overrides them. An omitted or empty image +`WorkingDir` normalizes to `/`. Image `Env` remains in the +immutable configuration blob. `env_keys` contains runtime store references, +never runtime values. Reserved `SPACETIMEDB_` keys are rejected. Mounts must be +empty in Stage 1. Shared resource, startup-string, port, and environment-key +limits apply to the normalized specification. + +## Tools and credentials + +Source builds require an explicitly selected local BuildKit Unix socket: + +```sh +spacetime container build example \ + --project-path ./project \ + --platform linux/amd64 \ + --out-dir ./prepared-image \ + --buildkit-host unix:///absolute/path/to/disposable-buildkit.sock +``` + +Use an endpoint you own and have verified. The command never selects a saved +Docker daemon or BuildKit endpoint. This implementation invokes `buildctl` +directly and currently runs external tools on Linux and macOS. An existing local +OCI directory can also be imported without those subprocesses on Windows. +`--platform` is required and accepts `linux/amd64` or `linux/arm64`; it never +silently uses the build computer's platform. `--out-dir` must not already exist, +and its parent must exist. + +`--buildctl`, `--skopeo`, and `--railpack` select absolute executable paths or +command names on PATH. Missing tools +produce installation/path errors. No executable is automatically downloaded. +Railpack is pinned to `0.35.0`, paired with +`ghcr.io/railwayapp/railpack-frontend:v0.35.0`. Detection or version failures end +the build without changing builders. Its `prepare` output is given to BuildKit's +matching gateway frontend, following the [Railpack production integration](https://railpack.com/platforms/running-railpack-in-production). + +Registry access is anonymous unless `--registry-auth-file FILE` explicitly names +an auth JSON file. The CLI creates an isolated auth directory and does not load +saved Docker credentials. Registry credentials are distinct from Spacetime +credentials and are never copied into prepared metadata. + +Pass build secrets as `--build-secret NAME=FILE`. Files enter BuildKit's secret +interface; Railpack receives only their names during plan generation. Builds +with secrets disable cached build results. Build secrets are separate from +`env_keys`, and prebuilt imports reject them. Credential and secret files are +bounded to 1 MiB each. Subprocess environments are cleared except for basic +executable/temporary-directory paths and the explicitly isolated tool paths. +Do not put secret values in a Dockerfile, image command, or image defaults. + +## Output and failure behavior + +The new output directory contains an OCI image layout plus `prepared.json`: + +- `container` is the normalized container specification. +- `manifest` is the selected immutable manifest descriptor. +- `objects` contains the executable manifest/config/layer closure, with digest, + media type, size, purpose, and a path relative to the layout. + +Prepared metadata omits image environment values and build credentials. The +image configuration blob necessarily retains its image defaults. Consumers must +verify descriptors when reopening output; the local metadata is not an +admission receipt from a server. + +`prepare_container` returns `PreparedContainer`, which owns its temporary +layout. Dropping it removes the temporary output. `persist` transfers the verified +layout with an atomic operation that does not replace an existing output. +Credentials, plans, and intermediate builder files remain in the temporary +workspace and are removed. Failed verification never returns a prepared object. +The shared `Runner` interface permits structured fake builders in tests and +reuse by a later managed publication path. + +Verification checks manifest/config digests, platform, exact compressed layer +digests, uncompressed diff IDs, and bounded tar structure. Layers are inspected +as streams and are never unpacked into the project. Imports reject archive +links, special entries, unknown paths, and duplicate object paths. Bounds include +256 layers, 64 GiB compressed image data, 128 GiB expanded tar data, and one +million expanded entries, with the shared per-object/decompression limits. +Two image tools and two verification workers may run concurrently. Tool calls +have a 30-minute deadline; verification has a 5-minute deadline. Captured stdout +and stderr each have a 4 MiB limit and are not echoed, avoiding accidental +secret disclosure in tool diagnostics. + +Cancellation retains the tool's workspace until its Unix process group has been +signalled and its leader reaped. The group leader remains unreaped until the +signal, preventing reuse of its numeric process-group identity. This cleanup is +for locally trusted tools; it is not a sandbox for a tool that deliberately +escapes its group. BuildKit daemon work is subject to the selected daemon's own +client-disconnect cancellation and retention policies. Blocking verification +retains its worker permit and workspace while checking cancellation between +bounded reads. + +Tests use generated local OCI fixtures and fake tool invocations, including an +owned shell fixture for process cleanup. They do not execute Docker, BuildKit, +Railpack, Skopeo, or any server operation. Actual supported-builder acceptance +remains a separate integration check. + +## Managed publication + +`spacetime publish` publishes a selected target's container declaration through +managed publication. Select the image platform explicitly. Server selection uses +the normal CLI URL, configured alias, or default: + +```sh +spacetime publish my-db --server https://your-test-server.example \ + --container-platform linux/amd64 \ + --artifact-endpoint https://your-test-artifacts.example +``` + +The URLs above are placeholders. Managed transport requires HTTPS for remote +servers and also supports HTTP loopback servers. The CLI does not send the publisher's Bearer credential to an +advertised artifact origin unless it is the same origin as the selected server +or the exact URL is explicitly approved with `--artifact-endpoint`. It does not +follow HTTP redirects or inherit HTTP proxy settings for managed publication. +Image registry credentials remain separate and require `--registry-auth-file`. + +A target with a container declaration and no module source preserves its existing +module. A new container-only database uses the immutable versioned empty module. +An explicit `module-path`, `bin-path`, or `js-path` replaces the module. An omitted +container preserves it; `--remove-container` removes it. `--remove-module` selects +the empty module and runs the existing authorized migration preflight. A selected +container declaration conflicts with `--remove-container`, and a configured +module source conflicts with `--remove-module`. Manual migrations and data-clear +publication are rejected. Precompiled NativeAOT modules can use `--bin-path`; +managed source compilation with `--native-aot` is not yet supported. + +Ordinary module-only publications use the legacy path. When deployment inspection +finds an existing managed revision, module-only updates use managed publication +with `ContainerAction::Keep`. `--managed` explicitly selects managed publication +for a new module-only deployment. A managed error never falls back to a raw +module publication. Existing databases use their exact current deployment +revision as a compare-and-set precondition. + +New managed databases first reserve a server-generated Identity under the +publisher and creation options. A caller cannot select an unreserved new Identity. +`--parent` resolves an accessible existing database; `--organization` currently +requires the organization's Identity. Requested database naming runs separately +after activation. If naming fails or its response is lost, the command reports +the successfully created Identity and does not repeat publication or overwrite +names on a later resume. + +Before staging artifacts, the CLI saves a private operation directory under +`.spacetime/publications/` beside the project configuration, or the explicit +`--publication-state-dir`. The directory retains exact request bytes, immutable +module/OCI artifacts and upload receipts. It contains no publisher credential or +resolved database environment values. Image blobs can contain environment +values baked into the image, so treat the directory as private build output. + +If a request fails or the command is interrupted, use the printed directory: + +```sh +spacetime publish --resume-publication .spacetime/publications/OPERATION_UUID \ + --server https://your-test-server.example \ + --artifact-endpoint https://your-test-artifacts.example +``` + +Resume authenticates the original publisher and server, observes accepted state, +and reuses the exact operation bytes. It does not read `spacetime.json`, rebuild, +or resolve a changed image tag. Ambiguous uploads query the same upload receipt +before continuing. If current database-read access was revoked, resume uses the +exact stored PUT to recover the original publisher's admitted result before +requiring local artifacts. A lost publication response never creates a second +operation. The coordinator PUT has a bounded 30-minute timeout covering its +separate schema, image, storage, and confirmation steps; interruption retains +the journal while the outcome is uncertain. Exact public PUT replay remains +limited to the operation's seven-day retry window; server-side recovery has its +own durable lifetime. +The default activation wait is 60 seconds; `--publication-wait 0` returns after +the first confirmed status, and a pending status prints its resume instruction. +Keep the directory while an outcome is uncertain; a confirmed terminal operation +can be removed locally when its artifacts are no longer needed. + +Container declarations remain local to their exact database target. They do not +inherit to nested targets. `spacetime dev` currently rejects selected container +targets because it does not yet supervise their local runtime. diff --git a/crates/cli/docs/container-operations.md b/crates/cli/docs/container-operations.md new file mode 100644 index 00000000000..3aa680d8167 --- /dev/null +++ b/crates/cli/docs/container-operations.md @@ -0,0 +1,39 @@ +# Container status and lifecycle requests + +`spacetime container status DATABASE` reads the current control state without +opening database storage. Viewer access is sufficient. `--json` returns the +typed status, including desired and observed state, deployment revision, +generation, fixed diagnostics, and public endpoint availability. An absent +current instance is distinct from an old instance's terminal report. Endpoint +allocation may be pending while the rest of the status is available. + +`spacetime container start DATABASE`, `stop DATABASE`, and `restart DATABASE` +require Admin access. Start requests execution, stop requests a stop, and restart +requests a new instance with a fresh environment snapshot. Acceptance records +the desired action; physical stop and readiness complete asynchronously. Use +status to inspect progress. None of these commands changes the published image +or its declaration. + +Each lifecycle request uses a UUIDv7. Before sending it, the CLI prints structured +retry parameters to stderr: the action, resolved database Identity, request ID, +and selected server URL. Keep these parameters if the response is lost. Retry +with that Identity, the same action, `--request-id UUID`, and `--server URL`. +`--request-id` requires an Identity so a changed database name cannot redirect a +retry. Never create a fresh request ID merely to resolve an unknown outcome. + +An accepted exact retry returns the original result generation, even if another +request has since advanced the database generation. Reusing an ID for another +action fails. Current Admin access is checked again on every retry. The retry +window is seven days from the UUID's timestamp. `--json` writes only a verified +receipt to stdout; its generation is a decimal string to preserve all 64 bits. + +Server selection and login use the normal CLI configuration, with an explicit +`--server` override supported on every command. Authenticated requests disable +redirects and inherited proxies. Public `container url` remains anonymous, and +local `container build` continues to run without loading server credentials. + +The HTTP counterparts are authenticated `GET +/v1/database/DATABASE/container/status` and `POST` to the `start`, `stop`, or +`restart` suffix with `{"request_id":"UUID"}`. Successful mutations return HTTP +202. Responses are not cacheable. The operational API rejects hosted container +credentials and uses current ordinary database roles. diff --git a/crates/cli/src/container/config.rs b/crates/cli/src/container/config.rs new file mode 100644 index 00000000000..b9f12a6cf66 --- /dev/null +++ b/crates/cli/src/container/config.rs @@ -0,0 +1,122 @@ +//! Per-database declarations. Values and build credentials are never part of this configuration. +use anyhow::{ensure, Result}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::container::{ + ContainerMode, ContainerMount, ContainerPort, ContainerResources, ContainerSpec, ContainerSpecLimits, + ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_oci::ContainerConfig as ImageConfig; +use std::path::PathBuf; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerConfig { + pub image: ImageSource, + pub command: Option>, + pub user: Option, + pub working_directory: Option, + #[serde(default = "default_mode")] + pub mode: ContainerMode, + #[serde(default = "default_restart")] + pub restart: RestartPolicy, + #[serde(default)] + pub env_keys: Vec, + pub resources: ContainerResources, + #[serde(default)] + pub ports: Vec, + #[serde(default)] + pub mounts: Vec, + #[serde(default = "default_grace")] + pub stop_grace_ms: u32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ImageSource { + Build(BuildImage), + Prebuilt(PrebuiltImage), +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BuildImage { + #[serde(deserialize_with = "source_build")] + pub build: SourceBuild, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PrebuiltImage { + pub oci_ref: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "builder", rename_all = "lowercase", deny_unknown_fields)] +pub enum SourceBuild { + Dockerfile { + #[serde(default = "default_context")] + context: PathBuf, + #[serde(default = "default_dockerfile")] + dockerfile: PathBuf, + }, + Railpack { + #[serde(default = "default_context")] + context: PathBuf, + }, +} + +// A missing builder is Dockerfile. Keep deserialization strict after applying +// that one default, including rejection of Dockerfile-only fields on Railpack. +fn source_build<'de, D: serde::Deserializer<'de>>(deserializer: D) -> std::result::Result { + let mut value = serde_json::Value::deserialize(deserializer)?; + if let Some(object) = value.as_object_mut() { + object.entry("builder").or_insert_with(|| "dockerfile".into()); + } + serde_json::from_value(value).map_err(serde::de::Error::custom) +} +fn default_context() -> PathBuf { + PathBuf::from(".") +} +fn default_dockerfile() -> PathBuf { + PathBuf::from("Dockerfile") +} +fn default_mode() -> ContainerMode { + ContainerMode::Service +} +fn default_restart() -> RestartPolicy { + RestartPolicy::OnFailure +} +fn default_grace() -> u32 { + 30_000 +} + +impl ContainerConfig { + pub fn normalize( + &self, + manifest: OciDigest, + platform: ImagePlatform, + image: &ImageConfig, + ) -> Result { + ensure!(self.mounts.is_empty(), "container mounts are not supported in Stage 1"); + let argv = image.argv(self.command.as_deref())?; + spacetimedb_lib::container::validate_exec_size(&argv, image.env.as_deref().unwrap_or_default())?; + Ok(ContainerSpec { + image_manifest: manifest, + image_platform: platform, + argv, + user: self.user.clone().unwrap_or_else(|| image.user.clone()), + working_directory: self.working_directory.clone().unwrap_or_else(|| { + if image.working_directory.is_empty() { + "/".into() + } else { + image.working_directory.clone() + } + }), + mode: self.mode, + restart: self.restart, + env_keys: self.env_keys.clone(), + resources: self.resources, + ports: self.ports.clone(), + mounts: vec![], + stop_grace_ms: self.stop_grace_ms, + } + .normalize(&ContainerSpecLimits::default())?) + } +} diff --git a/crates/cli/src/container/mod.rs b/crates/cli/src/container/mod.rs new file mode 100644 index 00000000000..02af7005921 --- /dev/null +++ b/crates/cli/src/container/mod.rs @@ -0,0 +1,395 @@ +//! Local image preparation shared by build-only commands and managed publication. +pub mod config; +pub mod oci; +pub mod process; +pub mod publish; + +#[cfg(test)] +pub(crate) mod tests; + +use anyhow::{ensure, Context, Result}; +use config::{ContainerConfig, ImageSource, SourceBuild}; +use oci::{LocalArtifact, VerifiedImage}; +use process::{Invocation, Runner}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::container::{ContainerSpec, ImagePlatform}; +use spacetimedb_oci::Descriptor; +use std::{ + ffi::OsString, + fs, + io::Read, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +pub const RAILPACK_VERSION: &str = "0.35.0"; +pub const RAILPACK_FRONTEND: &str = "ghcr.io/railwayapp/railpack-frontend:v0.35.0"; +const BUILD_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const VERIFY_TIMEOUT: Duration = Duration::from_secs(5 * 60); +static VERIFIERS: std::sync::LazyLock> = + std::sync::LazyLock::new(|| Arc::new(tokio::sync::Semaphore::new(2))); + +pub struct BuildSecret { + pub name: String, + pub file: PathBuf, +} +pub struct BuildTools { + pub buildctl: PathBuf, + pub buildkit_host: Option, + pub railpack: PathBuf, + pub skopeo: PathBuf, + pub registry_auth_file: Option, + pub secrets: Vec, +} +impl Default for BuildTools { + fn default() -> Self { + Self { + buildctl: "buildctl".into(), + buildkit_host: None, + railpack: "railpack".into(), + skopeo: "skopeo".into(), + registry_auth_file: None, + secrets: vec![], + } + } +} + +/// Public metadata contains immutable descriptors and relative paths, not image +/// environment values or credentials. Reopening output must verify the bytes. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PreparedMetadata { + pub version: u32, + pub container: ContainerSpec, + pub manifest: Descriptor, + pub objects: Vec, +} +/// Owns all temporary output. Drop removes it; persist transfers only verified +/// artifacts. A failed/cancelled prepare never yields this type. +pub struct PreparedContainer { + workspace: Arc, + pub metadata: PreparedMetadata, +} +impl PreparedContainer { + pub fn layout(&self) -> PathBuf { + self.workspace.path().join("verified") + } + pub fn persist(self, output: &Path) -> Result<()> { + ensure!( + !output.exists(), + "output directory already exists: {}", + output.display() + ); + // The final transition must not replace a directory created after the + // initial check. The workspace is placed alongside the requested output. + #[cfg(any(target_os = "linux", target_os = "macos"))] + rustix::fs::renameat_with( + rustix::fs::CWD, + self.layout(), + rustix::fs::CWD, + output, + rustix::fs::RenameFlags::NOREPLACE, + ) + .context("could not publish local OCI output without replacing an existing path")?; + #[cfg(windows)] + fs::rename(self.layout(), output) + .context("could not publish local OCI output without replacing an existing path")?; + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + anyhow::bail!("atomic container output is supported on Linux, macOS, and Windows"); + Ok(()) + } +} +struct CancelOnDrop(CancellationToken); +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel(); + } +} +fn path_argument(key: &str, path: &Path) -> Result { + let path = path.to_str().context("image tools require UTF-8 paths")?; + Ok(format!("{key}={path}").into()) +} +fn csv_argument(key: &str, path: &Path) -> Result { + let path = path.to_str().context("image tools require UTF-8 paths")?; + Ok(format!("\"{key}={}\"", path.replace('"', "\"\""))) +} +fn read_credential(path: &Path) -> Result> { + let mut bytes = vec![]; + fs::File::open(path)?.take(1024 * 1024 + 1).read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= 1024 * 1024, + "explicit build credential file exceeds 1 MiB" + ); + Ok(bytes) +} +fn local_buildkit_host(value: Option<&str>) -> Result<&str> { + let value = value.context("source builds require --buildkit-host unix:///absolute/path/to/buildkitd.sock")?; + let path = value + .strip_prefix("unix://") + .context("local source builds require an explicit BuildKit Unix socket")?; + ensure!( + Path::new(path).is_absolute() && !path.contains(['\0', '\n', '\r']), + "invalid local BuildKit socket path" + ); + Ok(value) +} + +pub async fn prepare_container( + declaration: &ContainerConfig, + config_dir: &Path, + platform: ImagePlatform, + tools: &BuildTools, + workspace_parent: &Path, + runner: &impl Runner, + cancel: CancellationToken, +) -> Result { + ensure!( + platform.os == "linux" && matches!(platform.architecture.as_str(), "amd64" | "arm64"), + "choose linux/amd64 or linux/arm64 explicitly" + ); + ensure!( + declaration.mounts.is_empty(), + "container mounts are not supported in Stage 1" + ); + let cancel = cancel.child_token(); + let _cancel = CancelOnDrop(cancel.clone()); + let workspace = Arc::new( + tempfile::Builder::new() + .prefix(".spacetime-image-") + .tempdir_in(workspace_parent)?, + ); + let base = workspace.path(); + fs::create_dir(base.join("auth"))?; + let auth_file = base.join("auth/config.json"); + let auth = tools + .registry_auth_file + .as_deref() + .map(read_credential) + .transpose()? + .unwrap_or_else(|| br#"{"auths":{}}"#.to_vec()); + fs::write(&auth_file, auth)?; + let environment = vec![ + ("DOCKER_CONFIG".into(), base.join("auth").into_os_string()), + ("REGISTRY_AUTH_FILE".into(), auth_file.clone().into_os_string()), + ("XDG_CONFIG_HOME".into(), base.join("tool-config").into_os_string()), + ("XDG_CACHE_HOME".into(), base.join("tool-cache").into_os_string()), + ]; + let invoke = |tool: PathBuf, label, args, cwd: PathBuf| Invocation { + tool, + label, + args, + env: environment.clone(), + cwd, + workspace: workspace.clone(), + timeout: BUILD_TIMEOUT, + cancel: cancel.clone(), + }; + let input = base.join("input"); + let mut archive = None; + match &declaration.image { + ImageSource::Prebuilt(image) => { + ensure!( + tools.secrets.is_empty(), + "build secrets cannot be supplied with a prebuilt image" + ); + if let Some(path) = image.oci_ref.strip_prefix("oci:") { + let path = config_dir + .join(path) + .canonicalize() + .context("prebuilt OCI layout does not exist")?; + ensure!(path.is_dir(), "oci: must name an OCI layout directory"); + return verify(declaration.clone(), platform, workspace.clone(), path, None, cancel).await; + } + ensure!( + !image.oci_ref.is_empty() + && !image.oci_ref.starts_with('-') + && !image.oci_ref.contains(['\0', '\n', '\r']), + "invalid OCI registry reference" + ); + let reference = image.oci_ref.strip_prefix("docker://").unwrap_or(&image.oci_ref); + ensure!( + !reference.contains("://") + && (!reference.contains('@') + || reference.rsplit_once('@').is_some_and(|(_, digest)| digest + .parse::() + .is_ok())), + "invalid OCI registry reference" + ); + let args = vec![ + "--override-os".into(), + platform.os.clone().into(), + "--override-arch".into(), + platform.architecture.clone().into(), + "copy".into(), + "--preserve-digests".into(), + "--authfile".into(), + auth_file.into_os_string(), + format!("docker://{reference}").into(), + format!("oci:{}:prepared", input.display()).into(), + ]; + runner + .run(invoke( + tools.skopeo.clone(), + "Skopeo image import", + args, + config_dir.to_path_buf(), + )) + .await?; + } + ImageSource::Build(image) => { + let endpoint = local_buildkit_host(tools.buildkit_host.as_deref())?; + let (context, dockerfile, railpack) = match &image.build { + SourceBuild::Dockerfile { context, dockerfile } => (context, Some(dockerfile), false), + SourceBuild::Railpack { context } => (context, None, true), + }; + let context = config_dir + .join(context) + .canonicalize() + .context("build context does not exist")?; + ensure!(context.is_dir(), "build context must be a directory"); + let mut secret_arguments = vec![]; + let mut secret_names = std::collections::BTreeSet::new(); + for (index, secret) in tools.secrets.iter().enumerate() { + spacetimedb_lib::container::validate_env_key(&secret.name)?; + ensure!(secret_names.insert(&secret.name), "duplicate build secret name"); + let path = base.join(format!("secret-{index}")); + fs::write(&path, read_credential(&secret.file)?)?; + secret_arguments.extend([ + OsString::from("--secret"), + format!("id={},{}", secret.name, csv_argument("src", &path)?).into(), + ]); + } + let dockerfile = if railpack { + let version = runner + .run(invoke( + tools.railpack.clone(), + "Railpack version check", + vec!["--version".into()], + context.clone(), + )) + .await?; + let version = std::str::from_utf8(&version.stdout).context("invalid Railpack version response")?; + ensure!( + version + .split_whitespace() + .any(|word| word.trim_start_matches('v') == RAILPACK_VERSION), + "install Railpack {RAILPACK_VERSION} to match the pinned frontend" + ); + let plan = base.join("railpack-plan.json"); + let mut args = vec![ + "prepare".into(), + context.clone().into_os_string(), + "--plan-out".into(), + plan.clone().into_os_string(), + "--info-out".into(), + base.join("railpack-info.json").into_os_string(), + ]; + // Only names enter the plan; BuildKit receives the actual files. + for name in secret_names { + args.extend(["--env".into(), format!("{name}=").into()]); + } + runner + .run(invoke( + tools.railpack.clone(), + "Railpack detection", + args, + context.clone(), + )) + .await?; + ensure!( + plan.is_file() && plan.metadata()?.len() <= 4 * 1024 * 1024, + "Railpack did not produce a bounded build plan" + ); + plan + } else { + context + .join(dockerfile.unwrap()) + .canonicalize() + .context("Dockerfile does not exist")? + }; + ensure!( + dockerfile.is_file(), + "Dockerfile or Railpack plan must be a regular file" + ); + let output = base.join("image.tar"); + let mut args = vec![ + "--addr".into(), + endpoint.into(), + "build".into(), + "--frontend".into(), + if railpack { + "gateway.v0".into() + } else { + "dockerfile.v0".into() + }, + "--local".into(), + path_argument("context", &context)?, + "--local".into(), + path_argument("dockerfile", dockerfile.parent().unwrap())?, + "--opt".into(), + path_argument("filename", Path::new(dockerfile.file_name().unwrap()))?, + "--opt".into(), + format!("platform={}/{}", platform.os, platform.architecture).into(), + "--output".into(), + format!("type=oci,{}", csv_argument("dest", &output)?).into(), + ]; + if railpack { + args.extend(["--opt".into(), format!("source={RAILPACK_FRONTEND}").into()]); + } + if !secret_arguments.is_empty() { + args.push("--no-cache".into()); + args.extend(secret_arguments); + } + runner + .run(invoke(tools.buildctl.clone(), "BuildKit OCI build", args, context)) + .await?; + archive = Some(output); + } + } + verify(declaration.clone(), platform, workspace.clone(), input, archive, cancel).await +} + +async fn verify( + declaration: ContainerConfig, + platform: ImagePlatform, + workspace: Arc, + input: PathBuf, + archive: Option, + cancel: CancellationToken, +) -> Result { + let permit = VERIFIERS + .clone() + .try_acquire_owned() + .context("two OCI images are already being verified")?; + let owner = workspace.clone(); + let metadata = tokio::task::spawn_blocking(move || { + let _permit = permit; + let deadline = Instant::now() + VERIFY_TIMEOUT; + if let Some(archive) = archive { + oci::extract_archive(&archive, &input, &cancel, deadline)?; + } + let VerifiedImage { + manifest, + config, + objects, + } = oci::verify_layout(&input, &owner.path().join("verified"), &platform, &cancel, deadline)?; + let container = declaration.normalize(manifest.digest, platform, &config.config)?; + let metadata = PreparedMetadata { + version: 1, + container, + manifest, + objects, + }; + fs::write( + owner.path().join("verified/prepared.json"), + serde_json::to_vec_pretty(&metadata)?, + )?; + Ok::<_, anyhow::Error>(metadata) + }) + .await + .context("OCI verification worker stopped")??; + Ok(PreparedContainer { workspace, metadata }) +} diff --git a/crates/cli/src/container/oci.rs b/crates/cli/src/container/oci.rs new file mode 100644 index 00000000000..3e5ec888a57 --- /dev/null +++ b/crates/cli/src/container/oci.rs @@ -0,0 +1,262 @@ +//! Bounded OCI import. Layers are inspected as streams, never unpacked into the project. +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::container::{ImagePlatform, OciDigest}; +use spacetimedb_oci::{ + self as oci, + layers::{verify_layer_with_check, LayerLimits}, + Descriptor, ImageConfig, +}; +use std::{ + collections::BTreeSet, + fs::{self, File}, + io::{Read, Seek, Write}, + path::{Path, PathBuf}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; + +pub const MAX_EXPANDED_IMAGE_BYTES: u64 = 128 * 1024 * 1024 * 1024; +const MAX_ARCHIVE_ENTRIES: usize = 1024; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactKind { + Manifest, + Config, + Layer, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LocalArtifact { + pub kind: ArtifactKind, + pub descriptor: Descriptor, + /// Relative to the owned OCI layout directory. + pub path: PathBuf, +} +pub(crate) struct VerifiedImage { + pub manifest: Descriptor, + pub config: ImageConfig, + pub objects: Vec, +} + +pub(crate) fn check(cancel: &CancellationToken, deadline: Instant) -> std::io::Result<()> { + if cancel.is_cancelled() || Instant::now() >= deadline { + Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "container preparation cancelled or timed out", + )) + } else { + Ok(()) + } +} +fn blob_path(digest: OciDigest) -> PathBuf { + PathBuf::from("blobs/sha256").join(digest.to_string().strip_prefix("sha256:").expect("SHA-256 digest")) +} +fn bounded_file(path: &Path, limit: u64) -> Result { + ensure!( + fs::symlink_metadata(path)?.is_file(), + "OCI object must be a regular file" + ); + let file = File::open(path)?; + ensure!(file.metadata()?.len() <= limit, "OCI object exceeds its size bound"); + Ok(file) +} +fn read_small(path: &Path, limit: usize) -> Result> { + let mut bytes = vec![]; + bounded_file(path, limit as u64)? + .take(limit as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!(bytes.len() <= limit, "OCI metadata exceeds its size bound"); + Ok(bytes) +} + +pub(crate) fn extract_archive( + archive: &Path, + output: &Path, + cancel: &CancellationToken, + deadline: Instant, +) -> Result<()> { + fs::create_dir_all(output.join("blobs/sha256"))?; + let mut seen = BTreeSet::new(); + let mut total = 0u64; + let archive = bounded_file(archive, oci::MAX_IMAGE_BYTES + 16 * 1024 * 1024)?; + for (index, entry) in tar::Archive::new(archive).entries()?.enumerate() { + check(cancel, deadline)?; + ensure!(index < MAX_ARCHIVE_ENTRIES, "too many OCI archive entries"); + let mut entry = entry?; + let path = entry.path()?.into_owned(); + let name = path.to_str().context("OCI archive path is not UTF-8")?; + if entry.header().entry_type().is_dir() { + ensure!( + matches!(name.trim_end_matches('/'), "." | "blobs" | "blobs/sha256"), + "unexpected OCI archive directory" + ); + continue; + } + ensure!( + entry.header().entry_type().is_file(), + "OCI archive links and special files are unsupported" + ); + let blob = name.strip_prefix("blobs/sha256/"); + ensure!( + matches!(name, "index.json" | "oci-layout") + || blob.is_some_and( + |v| v.len() == 64 && v.bytes().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()) + ), + "unexpected OCI archive path" + ); + ensure!(seen.insert(path.clone()), "duplicate OCI archive path"); + let size = entry.size(); + total = total.checked_add(size).context("OCI archive size overflow")?; + ensure!( + total <= oci::MAX_IMAGE_BYTES + 16 * 1024 * 1024, + "OCI archive exceeds image size bound" + ); + if blob.is_none() { + ensure!(size <= oci::MAX_MANIFEST_BYTES as u64, "OCI archive metadata too large"); + } + let mut file = File::options().write(true).create_new(true).open(output.join(&path))?; + let mut buffer = [0u8; 64 * 1024]; + loop { + check(cancel, deadline)?; + let n = entry.read(&mut buffer)?; + if n == 0 { + break; + } + file.write_all(&buffer[..n])?; + } + } + Ok(()) +} + +pub(crate) fn verify_layout( + input: &Path, + output: &Path, + platform: &ImagePlatform, + cancel: &CancellationToken, + deadline: Instant, +) -> Result { + check(cancel, deadline)?; + let layout: serde_json::Value = serde_json::from_slice(&read_small(&input.join("oci-layout"), 1024)?)?; + ensure!( + layout.get("imageLayoutVersion").and_then(|v| v.as_str()) == Some("1.0.0"), + "unsupported OCI layout version" + ); + let index = read_small(&input.join("index.json"), oci::MAX_MANIFEST_BYTES)?; + // Layout metadata is not a hashed image object and may omit mediaType. + // Defaults apply only here, never to immutable registry index bytes. + let mut index: serde_json::Value = serde_json::from_slice(&index)?; + index + .as_object_mut() + .context("OCI layout index must be an object")? + .entry("mediaType") + .or_insert_with(|| oci::OCI_INDEX.into()); + let index = serde_json::to_vec(&index)?; + let parsed: oci::ImageIndex = serde_json::from_slice(&index)?; + ensure!( + parsed.media_type == oci::OCI_INDEX, + "unsupported OCI layout index media type" + ); + ensure!( + parsed.schema_version == 2 && parsed.manifests.len() <= oci::MAX_INDEX_ENTRIES, + "invalid OCI layout index" + ); + // A local layout index commonly names one manifest without platform metadata. + // The immutable image config below must still match the explicit platform. + let descriptor = if parsed.manifests.len() == 1 && parsed.manifests[0].platform.is_none() { + parsed.manifests.into_iter().next().unwrap() + } else { + oci::select_platform(&index, platform)? + }; + // A layout can name a multi-platform index; select exactly one executable manifest. + let source = read_small(&input.join(blob_path(descriptor.digest)), oci::MAX_MANIFEST_BYTES)?; + oci::verify_object(&descriptor, &source)?; + let (descriptor, bytes) = if matches!(descriptor.media_type.as_str(), oci::OCI_INDEX | oci::DOCKER_INDEX) { + let selected = oci::select_platform(&source, platform)?; + let bytes = read_small(&input.join(blob_path(selected.digest)), oci::MAX_MANIFEST_BYTES)?; + oci::verify_object(&selected, &bytes)?; + (selected, bytes) + } else { + (descriptor, source) + }; + ensure!( + matches!(descriptor.media_type.as_str(), oci::OCI_MANIFEST | oci::DOCKER_MANIFEST), + "OCI layout does not select an executable image" + ); + let manifest = oci::parse_manifest(&bytes)?; + let config_bytes = read_small(&input.join(blob_path(manifest.config.digest)), oci::MAX_CONFIG_BYTES)?; + let config = oci::parse_config(&config_bytes, &manifest, platform)?; + let mut compressed = 0u64; + let mut expanded = 0u64; + let mut entries = 0u64; + for (layer, diff_id) in manifest.layers.iter().zip(&config.rootfs.diff_ids) { + check(cancel, deadline)?; + compressed = compressed.checked_add(layer.size).context("image size overflow")?; + ensure!(compressed <= oci::MAX_IMAGE_BYTES, "compressed image exceeds bound"); + let file = bounded_file(&input.join(blob_path(layer.digest)), layer.size)?; + let size = verify_layer_with_check(file, layer, *diff_id, LayerLimits::default(), || { + check(cancel, deadline) + })?; + expanded = expanded + .checked_add(size.uncompressed_tar_bytes) + .context("expanded image size overflow")?; + entries = entries + .checked_add(size.entries) + .context("image entry count overflow")?; + ensure!( + expanded <= MAX_EXPANDED_IMAGE_BYTES && entries <= 1_000_000, + "expanded image exceeds bound" + ); + } + fs::create_dir_all(output.join("blobs/sha256"))?; + let mut objects = vec![]; + for object in oci::object_closure(descriptor.clone(), &manifest)? { + check(cancel, deadline)?; + let path = blob_path(object.digest); + let mut source = bounded_file(&input.join(&path), object.size)?; + source.rewind()?; + let mut target = File::options().write(true).create_new(true).open(output.join(&path))?; + let mut hash = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + check(cancel, deadline)?; + let n = source.read(&mut buffer)?; + if n == 0 { + break; + } + total = total.checked_add(n as u64).context("object size overflow")?; + ensure!(total <= object.size, "OCI object changed while copying"); + hash.update(&buffer[..n]); + target.write_all(&buffer[..n])?; + } + ensure!( + total == object.size && OciDigest::sha256(hash.finalize().into()) == object.digest, + "OCI object changed while copying" + ); + objects.push(LocalArtifact { + kind: if object.digest == descriptor.digest { + ArtifactKind::Manifest + } else if object.digest == manifest.config.digest { + ArtifactKind::Config + } else { + ArtifactKind::Layer + }, + descriptor: object, + path, + }); + } + fs::write(output.join("oci-layout"), br#"{"imageLayoutVersion":"1.0.0"}"#)?; + fs::write( + output.join("index.json"), + serde_json::to_vec( + &serde_json::json!({"schemaVersion":2,"mediaType":oci::OCI_INDEX,"manifests":[descriptor]}), + )?, + )?; + Ok(VerifiedImage { + manifest: descriptor, + config, + objects, + }) +} diff --git a/crates/cli/src/container/process.rs b/crates/cli/src/container/process.rs new file mode 100644 index 00000000000..3d548833178 --- /dev/null +++ b/crates/cli/src/container/process.rs @@ -0,0 +1,212 @@ +//! Local trusted tools run in a dedicated Unix process group. Cancellation +//! retains the workspace until that group is signalled and its leader reaped. +//! This is process cleanup, not containment of tools that deliberately escape +//! their process group. +use anyhow::Result; +use std::{ffi::OsString, future::Future, path::PathBuf, sync::Arc, time::Duration}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +pub struct Invocation { + pub tool: PathBuf, + pub label: &'static str, + pub args: Vec, + pub env: Vec<(OsString, OsString)>, + pub cwd: PathBuf, + pub workspace: Arc, + pub timeout: Duration, + pub cancel: CancellationToken, +} +pub struct Output { + pub stdout: Vec, +} +pub trait Runner: Sync { + fn run(&self, invocation: Invocation) -> impl Future> + Send; +} +pub struct LocalRunner; + +impl Runner for LocalRunner { + async fn run(&self, invocation: Invocation) -> Result { + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + local::run(invocation).await + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = invocation; + anyhow::bail!( + "local image tools currently require Linux or macOS; import an existing oci: directory instead" + ) + } + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod local { + use super::*; + use anyhow::{bail, ensure, Context}; + use rustix::process::{Pid, WaitIdOptions}; + use std::process::Stdio; + use tokio::{ + io::AsyncReadExt, + process::Command, + sync::{oneshot, Semaphore}, + }; + + static PROCESSES: std::sync::LazyLock> = std::sync::LazyLock::new(|| Arc::new(Semaphore::new(2))); + const MAX_OUTPUT: usize = 4 * 1024 * 1024; + + async fn read_bounded(mut reader: impl tokio::io::AsyncRead + Unpin) -> Result> { + let mut bytes = vec![]; + (&mut reader) + .take(MAX_OUTPUT as u64 + 1) + .read_to_end(&mut bytes) + .await?; + ensure!(bytes.len() <= MAX_OUTPUT, "builder diagnostic output exceeded 4 MiB"); + Ok(bytes) + } + + // Keep the group leader unreaped until after killpg. Its waitable PID pins + // the numeric process-group identity even when it exits before descendants. + struct ProcessGroup(Option); + impl ProcessGroup { + fn kill(&mut self) -> std::io::Result<()> { + if let Some(pid) = self.0.take() { + match rustix::process::kill_process_group(pid, rustix::process::Signal::KILL) { + Ok(()) | Err(rustix::io::Errno::SRCH) => (), + #[cfg(target_os = "macos")] + Err(rustix::io::Errno::PERM) if exited_leader_is_sole_member(pid) => (), + Err(error) => return Err(error.into()), + } + } + Ok(()) + } + async fn observe_exit(&mut self) -> Result<()> { + let pid = self.0.context("local image tool ownership lost")?; + loop { + match rustix::process::waitid( + rustix::process::WaitId::Pid(pid), + WaitIdOptions::EXITED | WaitIdOptions::NOWAIT | WaitIdOptions::NOHANG, + ) { + Ok(Some(_)) => return Ok(()), + Ok(None) | Err(rustix::io::Errno::INTR) => (), + Err(error) => { + // An external reaper invalidates numeric PID ownership. + // Never signal that group after this boundary. + if error == rustix::io::Errno::CHILD { + self.0 = None; + } + return Err(error).context("could not observe local image tool exit"); + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + } + + #[cfg(target_os = "macos")] + fn exited_leader_is_sole_member(pid: Pid) -> bool { + // XNU excludes zombies from killpg's eligible processes, and reports + // EPERM when only an unreaped leader remains. Do not generalize EPERM: + // prove this exact child has exited and its group contains only it. + // WNOWAIT keeps the PID/PGID pinned across the bounded inventory. + if !matches!( + rustix::process::waitid( + rustix::process::WaitId::Pid(pid), + WaitIdOptions::EXITED | WaitIdOptions::NOWAIT | WaitIdOptions::NOHANG, + ), + Ok(Some(_)) + ) { + return false; + } + #[link(name = "proc")] + unsafe extern "C" { + fn proc_listpgrppids( + pgrpid: std::ffi::c_int, + buffer: *mut std::ffi::c_void, + buffersize: std::ffi::c_int, + ) -> std::ffi::c_int; + } + // libproc returns the number of PIDs copied, including zombies. Two + // slots distinguish the sole leader from any extra/truncated members. + let mut members = [0i32; 2]; + // SAFETY: writable aligned storage and its exact byte size are passed; + // the group ID belongs to the still-unreaped child observed above. + let count = unsafe { + proc_listpgrppids( + pid.as_raw_pid(), + members.as_mut_ptr().cast(), + std::mem::size_of_val(&members) as std::ffi::c_int, + ) + }; + count == 1 && members[0] == pid.as_raw_pid() + } + impl Drop for ProcessGroup { + fn drop(&mut self) { + let _ = self.kill(); + } + } + + pub(super) async fn run(invocation: Invocation) -> Result { + let permit = PROCESSES + .clone() + .try_acquire_owned() + .context("two local image tools are already running")?; + let (mut send, receive) = oneshot::channel(); + tokio::spawn(async move { + let _permit = permit; + let result = async { + ensure!(!send.is_closed() && !invocation.cancel.is_cancelled(), "container build cancelled"); + let _workspace = invocation.workspace; + let mut command = Command::new(&invocation.tool); + command.args(&invocation.args).current_dir(&invocation.cwd).env_clear(); + // No implicit registry, Spacetime, proxy or builder credentials. + for key in ["PATH", "SystemRoot", "TMPDIR", "TEMP", "TMP"] { + if let Some(value) = std::env::var_os(key) { command.env(key, value); } + } + command.envs(invocation.env).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped()).process_group(0); + let mut child = command.spawn().with_context(|| format!("could not run {}; install it or provide its executable path", invocation.tool.display()))?; + let mut group = ProcessGroup(Some(Pid::from_raw(child.id().context("builder PID unavailable")? as i32).context("invalid builder PID")?)); + let stdout = child.stdout.take().context("builder stdout unavailable")?; + let stderr = child.stderr.take().context("builder stderr unavailable")?; + let outcome = { + let completion = async { + let (status, stdout, _) = tokio::try_join!( + async { + group.observe_exit().await?; + group.kill().context("failed to stop builder descendants")?; + Ok::<_, anyhow::Error>(child.wait().await?) + }, + read_bounded(stdout), + read_bounded(stderr), + )?; + ensure!(status.success(), "{} failed ({status}); no prepared output was accepted", invocation.label); + Ok(Output { stdout }) + }; + tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("container build caller closed")), + _ = invocation.cancel.cancelled() => Err(anyhow::anyhow!("container build cancelled")), + _ = tokio::time::sleep(invocation.timeout) => Err(anyhow::anyhow!("{} exceeded its build deadline", invocation.label)), + result = completion => result, + } + }; + // Disarmed before every reap, including completion above. Child + // wait is cached if completion already reaped it. + group.kill().context("failed to stop local image tool group")?; + match tokio::time::timeout(Duration::from_secs(5), child.wait()).await { + Ok(result) => { result.context("local image tool could not be reaped")?; }, + Err(_) => { + // The owner retains the workspace and permit throughout + // delayed physical cleanup, even if the caller is gone. + child.wait().await.context("local image tool could not be reaped")?; + bail!("local image tool required delayed physical cleanup"); + } + } + outcome + }.await; + let _ = send.send(result); + }); + receive.await.context("local image tool owner stopped")? + } +} diff --git a/crates/cli/src/container/publish/client.rs b/crates/cli/src/container/publish/client.rs new file mode 100644 index 00000000000..ebea3d20545 --- /dev/null +++ b/crates/cli/src/container/publish/client.rs @@ -0,0 +1,434 @@ +//! Ordinary publisher transport. Credentials are held in memory and never +//! redirected, inherited from a proxy, or sent to an unapproved artifact URL. +use anyhow::{bail, ensure, Context, Result}; +use reqwest::{header::HeaderValue, Client, Method, StatusCode, Url}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use spacetimedb_client_api_messages::name::{DomainName, PrePublishResult, SetDomainsResult}; +use spacetimedb_lib::{container::OciDigest, deployment::api::*, Identity, Uuid}; +use std::time::Duration; + +pub const MAX_RESPONSE_BYTES: usize = 1024 * 1024; +// A submission composes the 120s schema worker, 300s image verification, +// two independently bounded 300s artifact writes, and read/pin/control calls. +// This caller deadline does not extend those server resource or credential +// bounds. Cancellation/timeout leaves the exact operation in its journal. +const COORDINATOR_REQUEST_TIMEOUT: Duration = Duration::from_secs(30 * 60); +pub const UPLOAD_CHUNK_BYTES: usize = 8 * 1024 * 1024; + +#[derive(Debug, thiserror::Error)] +#[error("{action} returned HTTP {status}")] +pub struct HttpFailure { + pub action: &'static str, + pub status: StatusCode, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UploadKind { + Manifest, + Config, + Layer, + Module, +} +impl UploadKind { + fn header(self) -> &'static str { + match self { + Self::Manifest => "manifest", + Self::Config => "config", + Self::Layer => "layer", + Self::Module => "module", + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObjectRef { + pub digest: OciDigest, + pub size: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UploadStatus { + pub id: uuid::Uuid, + pub object: ObjectRef, + pub offset: u64, + pub expires_at: u64, + pub complete: bool, +} +impl UploadStatus { + pub fn validate(&self, object: ObjectRef, id: Option) -> Result<()> { + ensure!( + self.id.get_version_num() == 4 && id.is_none_or(|id| self.id == id), + "artifact upload session changed" + ); + ensure!( + self.object == object && self.offset <= object.size && (!self.complete || self.offset == object.size), + "artifact upload descriptor or offset changed" + ); + Ok(()) + } +} + +pub struct ArtifactEndpoint(Url); +impl ArtifactEndpoint { + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +pub struct PublisherClient { + http: Client, + server: Url, + authorization: HeaderValue, +} +impl PublisherClient { + pub fn new(server: &str, mut authorization: HeaderValue) -> Result { + let server = endpoint(server)?; + ensure!( + authorization.as_bytes().starts_with(b"Bearer "), + "managed publication requires ordinary Bearer authentication" + ); + authorization.set_sensitive(true); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { + http, + server, + authorization, + }) + } + pub fn server(&self) -> &Url { + &self.server + } + fn request(&self, method: Method, url: Url) -> reqwest::RequestBuilder { + self.http + .request(method, url) + .header(reqwest::header::AUTHORIZATION, self.authorization.clone()) + } + /// A response cannot grant itself permission to receive this credential. + /// Same-origin paths are accepted; another origin requires an explicit URL. + pub fn artifact_endpoint(&self, advertised: &str, approved: Option<&str>) -> Result { + let artifact = endpoint(advertised)?; + if artifact.origin() != self.server.origin() { + let approved=approved.context("artifact service uses another origin; pass --artifact-endpoint with its exact trusted URL before sending publisher credentials")?; + ensure!( + endpoint(approved)? == artifact, + "advertised artifact endpoint differs from the explicitly approved endpoint" + ); + } else if let Some(approved) = approved { + ensure!( + endpoint(approved)? == artifact, + "advertised artifact endpoint differs from the explicitly approved endpoint" + ); + } + Ok(ArtifactEndpoint(artifact)) + } + pub async fn capabilities(&self) -> Result> { + let response = self + .http + .get(route(&self.server, &["v1", "containers", "capabilities"])?) + .send() + .await?; + optional_json(response, "publication capabilities").await + } + pub async fn permission(&self) -> Result { + json( + self.request( + Method::GET, + route(&self.server, &["v1", "containers", "publish-permission"])?, + ) + .send() + .await?, + "publication permission", + ) + .await + } + pub async fn deployment(&self, name: &str) -> Result> { + optional_json( + self.request( + Method::GET, + route(&self.server, &["v1", "database", name, "deployment"])?, + ) + .send() + .await?, + "deployment inspection", + ) + .await + } + pub async fn reserve(&self, request: &ReserveDatabaseRequest) -> Result { + json( + self.request( + Method::POST, + route(&self.server, &["v1", "containers", "reservations"])?, + ) + .json(request) + .send() + .await?, + "database reservation", + ) + .await + } + pub async fn submit(&self, database: Identity, request: &PublishRequest) -> Result { + let bytes = serde_json::to_vec(request)?; + self.submit_bytes(database, &bytes).await + } + /// Resume sends these same immutable bytes, without reconstructing a request + /// from a changed tag, project configuration, or current deployment. + pub async fn submit_bytes(&self, database: Identity, bytes: &[u8]) -> Result { + ensure!(bytes.len() <= 256 * 1024, "publication HTTP body exceeds 256 KiB"); + json( + self.request( + Method::PUT, + route(&self.server, &["v1", "database", &database.to_hex(), "deployment"])?, + ) + .timeout(COORDINATOR_REQUEST_TIMEOUT) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(bytes.to_vec()) + .send() + .await?, + "publication submission", + ) + .await + } + pub async fn status(&self, database: Identity, operation: Uuid) -> Result> { + optional_json( + self.request( + Method::GET, + route( + &self.server, + &[ + "v1", + "database", + &database.to_hex(), + "deployment", + "operations", + &operation.to_string(), + ], + )?, + ) + .send() + .await?, + "publication status", + ) + .await + } + pub async fn preflight(&self, database: Identity, kind: &str, bytes: &[u8]) -> Result { + json( + self.request( + Method::POST, + route(&self.server, &["v1", "database", &database.to_hex(), "pre_publish"])?, + ) + .query(&[("host_type", kind), ("pretty_print_style", "NoColor")]) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(bytes.to_vec()) + .send() + .await?, + "module migration preflight", + ) + .await + } + pub async fn set_name(&self, database: Identity, name: &str) -> Result<()> { + let name: DomainName = name.parse()?; + let result: SetDomainsResult = json( + self.request( + Method::PUT, + route(&self.server, &["v1", "database", &database.to_hex(), "names"])?, + ) + .json(&[name]) + .send() + .await?, + "database naming", + ) + .await?; + ensure!( + matches!(result, SetDomainsResult::Success), + "database was created, but assigning its requested name was not confirmed" + ); + Ok(()) + } + pub async fn begin_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + kind: UploadKind, + object: ObjectRef, + ) -> Result { + let status: UploadStatus = json( + self.request( + Method::POST, + route(&artifact.0, &["v1", "databases", &database.to_hex(), "uploads"])?, + ) + .header("x-spacetimedb-artifact-kind", kind.header()) + .json(&serde_json::json!({"kind":kind,"object":object})) + .send() + .await?, + "artifact upload creation", + ) + .await?; + status.validate(object, None)?; + Ok(status) + } + pub async fn upload_status( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + id: uuid::Uuid, + object: ObjectRef, + ) -> Result { + let status: UploadStatus = json( + self.request( + Method::GET, + route( + &artifact.0, + &["v1", "databases", &database.to_hex(), "uploads", &id.to_string()], + )?, + ) + .send() + .await?, + "artifact upload status", + ) + .await?; + status.validate(object, Some(id))?; + Ok(status) + } + pub async fn append_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + status: &UploadStatus, + chunk: Vec, + ) -> Result { + ensure!( + !chunk.is_empty() + && chunk.len() <= UPLOAD_CHUNK_BYTES + && status + .offset + .checked_add(chunk.len() as u64) + .is_some_and(|end| end <= status.object.size), + "invalid artifact chunk" + ); + let next: UploadStatus = json( + self.request( + Method::PATCH, + route( + &artifact.0, + &["v1", "databases", &database.to_hex(), "uploads", &status.id.to_string()], + )?, + ) + .query(&[("offset", status.offset)]) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(chunk) + .send() + .await?, + "artifact upload append", + ) + .await?; + next.validate(status.object, Some(status.id))?; + Ok(next) + } + pub async fn complete_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + status: &UploadStatus, + ) -> Result { + let completed: ObjectRef = json( + self.request( + Method::POST, + route( + &artifact.0, + &[ + "v1", + "databases", + &database.to_hex(), + "uploads", + &status.id.to_string(), + "complete", + ], + )?, + ) + .send() + .await?, + "artifact upload completion", + ) + .await?; + // The completion endpoint confirms the immutable object, rather than + // returning session status. The route already binds the original UUID; + // retain that session only after checking the exact digest and size. + ensure!(completed == status.object, "artifact completion descriptor changed"); + let mut next = status.clone(); + next.offset = completed.size; + next.complete = true; + next.validate(status.object, Some(status.id))?; + Ok(next) + } +} + +pub(crate) fn is_loopback(url: &Url) -> bool { + match url.host() { + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + Some(url::Host::Domain(domain)) => domain.eq_ignore_ascii_case("localhost"), + None => false, + } +} +pub fn endpoint(value: &str) -> Result { + let mut url = Url::parse(value).context("resolved server URL must use HTTP(S)")?; + ensure!( + url.username().is_empty() && url.password().is_none() && url.query().is_none() && url.fragment().is_none(), + "endpoint must not contain credentials, query, or fragment" + ); + let local = is_loopback(&url); + ensure!( + url.scheme() == "https" || (url.scheme() == "http" && local), + "use HTTPS, or HTTP for a local loopback server" + ); + ensure!(url.host_str().is_some(), "endpoint host is missing"); + if !url.path().ends_with('/') { + let path = format!("{}/", url.path()); + url.set_path(&path); + } + Ok(url) +} +fn route(base: &Url, segments: &[&str]) -> Result { + let mut url = base.clone(); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid base URL"))? + .pop_if_empty() + .extend(segments); + Ok(url) +} +async fn optional_json(response: reqwest::Response, action: &'static str) -> Result> { + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + json(response, action).await.map(Some) +} +async fn json(mut response: reqwest::Response, action: &'static str) -> Result { + if !response.status().is_success() { + return Err(HttpFailure { + action, + status: response.status(), + } + .into()); + } + if response + .content_length() + .is_some_and(|length| length > MAX_RESPONSE_BYTES as u64) + { + bail!("{action} response exceeds its bound"); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + chunk.len() <= MAX_RESPONSE_BYTES.saturating_sub(bytes.len()), + "{action} response exceeds its bound" + ); + bytes.extend_from_slice(&chunk); + } + serde_json::from_slice(&bytes).with_context(|| format!("invalid {action} response")) +} diff --git a/crates/cli/src/container/publish/journal.rs b/crates/cli/src/container/publish/journal.rs new file mode 100644 index 00000000000..adb8f9521d8 --- /dev/null +++ b/crates/cli/src/container/publish/journal.rs @@ -0,0 +1,336 @@ +//! A publication resume record contains exact request bytes and object +//! descriptors, never credentials or resolved runtime environment values. +use super::client::{ObjectRef, UploadKind, UploadStatus}; +use crate::container::{oci, PreparedContainer}; +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::{ + container::OciDigest, + deployment::{api::*, PUBLISH_PROTOCOL_VERSION}, + Identity, Uuid, +}; +use std::{ + fs::{self, File}, + io::{Read, Write}, + path::{Path, PathBuf}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; + +const MAX_RECORD_BYTES: usize = 1024 * 1024; +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UploadRecord { + pub kind: UploadKind, + pub object: ObjectRef, + pub session: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Record { + pub version: u32, + pub server: String, + pub artifact_endpoint: String, + pub publisher: Identity, + pub database: Option, + pub reservation: Option, + pub requested_name: Option, + /// UTF-8 JSON sent verbatim on every submission attempt. + pub request_json: String, + pub request_digest: OciDigest, + pub uploads: Vec, + pub submitted: bool, + pub status: Option, + pub name_confirmed: bool, + pub naming_attempted: bool, +} +impl Record { + pub fn request(&self) -> Result { + ensure!( + self.version == 1 && self.request_json.len() <= 256 * 1024, + "unsupported or oversized publication resume record" + ); + ensure!( + spacetimedb_oci::sha256(self.request_json.as_bytes()) == self.request_digest, + "publication request bytes changed" + ); + let request: PublishRequest = serde_json::from_str(&self.request_json)?; + let envelope = &request.manifest.current().envelope; + ensure!( + envelope.version == PUBLISH_PROTOCOL_VERSION, + "unsupported publication protocol" + ); + ensure!( + envelope.operation_id.get_version() == Some(spacetimedb_lib::sats::uuid::Version::V7), + "publication operation must be UUIDv7" + ); + ensure!(self.uploads.len() <= 259, "too many retained publication objects"); + if let Some(reservation) = &self.reservation { + ensure!( + reservation.operation_id == envelope.operation_id + && reservation.version == PUBLISH_PROTOCOL_VERSION + && request.creation.as_ref() == Some(&reservation.options), + "reservation differs from the immutable publication request" + ); + } else { + ensure!( + self.database.is_some() && request.creation.is_none(), + "publication database binding is missing" + ); + } + for upload in &self.uploads { + ensure!(upload.object.size > 0, "empty publication artifact"); + if let Some(status) = &upload.session { + status.validate(upload.object, None)?; + } + } + if let Some(status) = &self.status { + self.check_status(&request, status)?; + } + Ok(request) + } + pub fn check_status(&self, request: &PublishRequest, status: &PublicationStatus) -> Result<()> { + let envelope = &request.manifest.current().envelope; + ensure!( + Some(status.database_identity) == self.database + && status.operation_id == envelope.operation_id + && status.expected_revision == envelope.expected_revision + && status.proposed_revision == request.manifest.current().deployment.revision()?, + "publication response belongs to another operation or deployment" + ); + Ok(()) + } +} + +pub struct Journal { + directory: PathBuf, + _lock: File, + pub record: Record, +} +struct IncompleteDirectory(Option); +impl Drop for IncompleteDirectory { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + let _ = fs::remove_dir_all(path); + } + } +} +impl Journal { + pub fn create( + base: &Path, + record: Record, + image: Option, + module: Option<&[u8]>, + ) -> Result { + let request = record.request()?; + let has_image = image.is_some(); + let directory = base.join(request.manifest.current().envelope.operation_id.to_string()); + fs::create_dir_all(base)?; + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder + .create(&directory) + .context("publication resume directory already exists; resume it rather than reusing its operation")?; + let mut incomplete = IncompleteDirectory(Some(directory.clone())); + let lock = Self::lock(&directory, true)?; + if let Some(image) = image { + image.persist(&directory.join("image"))?; + } + if let Some(module) = module { + let mut file = File::options() + .write(true) + .create_new(true) + .open(directory.join("module.blob"))?; + file.write_all(module)?; + file.sync_all()?; + } + let journal = Self { + directory, + _lock: lock, + record, + }; + journal.sync_retained_artifacts(has_image)?; + journal.save()?; + sync_directory_chain(&journal.directory)?; + incomplete.0 = None; + Ok(journal) + } + pub fn open(directory: &Path) -> Result { + let directory = directory + .canonicalize() + .context("publication resume directory not found")?; + let lock = Self::lock(&directory, false)?; + let path = directory.join("publication.json"); + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "publication resume record must be a regular file" + ); + let mut bytes = vec![]; + File::open(path)? + .take(MAX_RECORD_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= MAX_RECORD_BYTES, + "publication resume record is too large" + ); + let record: Record = serde_json::from_slice(&bytes)?; + record.request()?; + // Creation saves JSON only after artifact flush. Reconfirm the parent + // link if its creator stopped before finishing that last barrier. + sync_directory_chain(&directory)?; + Ok(Self { + directory, + _lock: lock, + record, + }) + } + fn sync_retained_artifacts(&self, has_image: bool) -> Result<()> { + let mut files = self + .record + .uploads + .iter() + .map(|upload| self.object_path(upload)) + .collect::>(); + if has_image { + files.extend( + ["oci-layout", "index.json", "prepared.json"].map(|name| self.directory.join("image").join(name)), + ); + } + let mut directories = std::collections::BTreeSet::new(); + for path in files { + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "retained publication artifact must be a regular file" + ); + // Write access also lets Windows flush an owned immutable artifact. + File::options().read(true).write(true).open(&path)?.sync_all()?; + let mut parent = path.parent(); + while let Some(path) = parent { + directories.insert(path.to_owned()); + if path == self.directory { + break; + } + parent = path.parent(); + } + } + // Flush children before the directories that link them. + for directory in directories.into_iter().rev() { + sync_directory(&directory)?; + } + Ok(()) + } + fn lock(directory: &Path, create: bool) -> Result { + let file = File::options() + .read(true) + .write(true) + .create_new(create) + .open(directory.join("publication.lock"))?; + file.try_lock() + .context("another process is using this publication resume directory")?; + Ok(file) + } + pub fn directory(&self) -> &Path { + &self.directory + } + pub fn operation(&self) -> Result { + Ok(self.record.request()?.manifest.current().envelope.operation_id) + } + pub fn save(&self) -> Result<()> { + self.record.request()?; + let bytes = serde_json::to_vec_pretty(&self.record)?; + ensure!( + bytes.len() <= MAX_RECORD_BYTES, + "publication resume record is too large" + ); + let mut file = tempfile::NamedTempFile::new_in(&self.directory)?; + file.write_all(&bytes)?; + file.as_file().sync_all()?; + file.persist(self.directory.join("publication.json"))?; + #[cfg(unix)] + File::open(&self.directory)?.sync_all()?; + Ok(()) + } + pub fn object_path(&self, upload: &UploadRecord) -> PathBuf { + match upload.kind { + UploadKind::Module => self.directory.join("module.blob"), + _ => self.directory.join("image/blobs/sha256").join( + upload + .object + .digest + .to_string() + .strip_prefix("sha256:") + .expect("SHA256"), + ), + } + } + /// Reopening state never trusts stale local object bytes. This repeats only + /// their digest check; server preparation remains the admission authority. + pub async fn verify_objects(&self, cancel: CancellationToken) -> Result<()> { + let permit = crate::container::VERIFIERS + .clone() + .try_acquire_owned() + .context("two OCI verifications are already running")?; + let objects = self + .record + .uploads + .iter() + .map(|upload| (self.object_path(upload), upload.object)) + .collect::>(); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let deadline = Instant::now() + crate::container::VERIFY_TIMEOUT; + for (path, object) in objects { + oci::check(&cancel, deadline)?; + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "retained publication artifact must be a regular file" + ); + let mut file = File::open(path)?; + ensure!(file.metadata()?.len() == object.size, "retained artifact size changed"); + let mut hash = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + oci::check(&cancel, deadline)?; + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + total = total.checked_add(read as u64).context("artifact size overflow")?; + ensure!(total <= object.size, "retained artifact grew"); + hash.update(&buffer[..read]); + } + ensure!( + total == object.size && OciDigest::sha256(hash.finalize().into()) == object.digest, + "retained publication artifact digest changed" + ); + } + Ok::<_, anyhow::Error>(()) + }) + .await + .context("publication verification worker stopped")??; + Ok(()) + } +} + +fn sync_directory(directory: &Path) -> Result<()> { + #[cfg(unix)] + File::open(directory)?.sync_all()?; + // As in paths::utils::write_atomic, Windows directory handles cannot be + // synced through std. File flushes still precede publishing the journal. + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} +fn sync_directory_chain(directory: &Path) -> Result<()> { + let canonical = directory.canonicalize()?; + for ancestor in canonical.ancestors() { + sync_directory(ancestor)?; + } + Ok(()) +} diff --git a/crates/cli/src/container/publish/mod.rs b/crates/cli/src/container/publish/mod.rs new file mode 100644 index 00000000000..35ac356322e --- /dev/null +++ b/crates/cli/src/container/publish/mod.rs @@ -0,0 +1,315 @@ +//! Managed publication uses one immutable operation and a durable local resume +//! directory. Ambiguous HTTP responses never select a new operation or fall +//! back to the legacy module publication endpoint. +pub mod client; +pub mod journal; + +#[cfg(test)] +pub(crate) mod tests; + +use anyhow::{bail, ensure, Context, Result}; +use client::{ArtifactEndpoint, PublisherClient, UploadStatus, UPLOAD_CHUNK_BYTES}; +use journal::Journal; +use spacetimedb_lib::deployment::{ + api::{PublicationPhase, PublicationStatus}, + operation_expiry_ms, +}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio_util::sync::CancellationToken; + +#[derive(Debug)] +pub enum Outcome { + Complete(PublicationStatus), + Pending(PublicationStatus), + Aborted(PublicationStatus), + /// Database creation succeeded. Never describe a naming failure as a + /// rollback or send another publication operation to compensate for it. + NamingUnconfirmed(PublicationStatus), +} + +pub async fn run( + client: &PublisherClient, + journal: &mut Journal, + approved_artifact: Option<&str>, + wait: Duration, + cancel: CancellationToken, +) -> Result { + let cancel = cancel.child_token(); + let _cancel = crate::container::CancelOnDrop(cancel.clone()); + ensure!( + client.server().as_str() == journal.record.server, + "resume server differs from the original publication endpoint" + ); + let artifact = client.artifact_endpoint(&journal.record.artifact_endpoint, approved_artifact)?; + let request = journal.record.request()?; + let operation = request.manifest.current().envelope.operation_id; + let permission = client.permission().await?; + ensure!( + permission.identity == journal.record.publisher, + "resume publisher differs from the original publication identity" + ); + + // Observe an already admitted operation before requiring original local + // object files or applying today's new-publication permission/limits. + if journal.record.submitted + && let Some(database) = journal.record.database + { + match observe_or_replay(client, journal, database, operation).await { + Ok(Some(status)) => { + journal.record.check_status(&request, &status)?; + journal.record.status = Some(status); + journal.save()?; + return wait_and_name(client, journal, wait, &cancel).await; + } + Ok(None) => (), + Err(error) => { + return Err(error).context("publication observation was not confirmed; resume the same directory") + } + } + } + let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; + operation_expiry_ms(operation, now)?; + ensure!( + !cancel.is_cancelled(), + "publication cancelled; resume directory retained" + ); + if journal.record.database.is_none() { + let reservation = client + .reserve( + journal + .record + .reservation + .as_ref() + .context("creation reservation is missing")?, + ) + .await?; + ensure!( + reservation.operation_id == operation && reservation.staging_open, + "reservation was not confirmed for this operation" + ); + let returned = client.artifact_endpoint(&reservation.artifact_endpoint, approved_artifact)?; + ensure!( + returned.as_str() == artifact.as_str(), + "reservation changed the selected artifact endpoint" + ); + journal.record.database = Some(reservation.database_identity); + journal.save()?; + } + journal.verify_objects(cancel.clone()).await?; + for index in 0..journal.record.uploads.len() { + upload(client, &artifact, journal, index, &cancel).await?; + } + ensure!( + !cancel.is_cancelled(), + "publication cancelled; resume directory retained" + ); + // This marker must be durable before sending the admission request. A crash + // after the server commits but before the response can then observe/replay. + journal.record.submitted = true; + journal.save()?; + let status = client + .submit_bytes(journal.record.database.unwrap(), journal.record.request_json.as_bytes()) + .await + .context("publication outcome is not confirmed; resume the same directory")?; + journal.record.check_status(&request, &status)?; + journal.record.status = Some(status); + journal.save()?; + wait_and_name(client, journal, wait, &cancel).await +} + +/// Operation inspection has current database-read authorization. Exact PUT +/// replay instead authenticates the original admitted publisher and immutable +/// request. It must precede local artifact/expiry work after read revocation. +async fn observe_or_replay( + client: &PublisherClient, + journal: &Journal, + database: spacetimedb_lib::Identity, + operation: spacetimedb_lib::Uuid, +) -> Result> { + match client.status(database, operation).await { + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.status == reqwest::StatusCode::FORBIDDEN) => + { + client + .submit_bytes(database, journal.record.request_json.as_bytes()) + .await + .map(Some) + .context("exact publication replay was not confirmed; keep the same resume directory") + } + result => result, + } +} + +async fn upload( + client: &PublisherClient, + artifact: &ArtifactEndpoint, + journal: &mut Journal, + index: usize, + cancel: &CancellationToken, +) -> Result<()> { + let database = journal.record.database.context("reservation missing")?; + let item = journal.record.uploads[index].clone(); + let mut status = if let Some(prior) = item.session { + match client.upload_status(artifact, database, prior.id, item.object).await { + Ok(status) => status, + Err(error) + if error.downcast_ref::().is_some_and(|error| { + matches!(error.status, reqwest::StatusCode::NOT_FOUND | reqwest::StatusCode::GONE) + }) => + { + client.begin_upload(artifact, database, item.kind, item.object).await? + } + Err(error) => return Err(error), + } + } else { + client.begin_upload(artifact, database, item.kind, item.object).await? + }; + store_upload(journal, index, &status)?; + if status.complete { + return Ok(()); + } + let mut file = tokio::fs::File::open(journal.object_path(&journal.record.uploads[index])).await?; + let mut ambiguous_attempts = 0; + while status.offset < status.object.size { + ensure!( + !cancel.is_cancelled(), + "artifact upload cancelled; resume directory retained" + ); + file.seek(std::io::SeekFrom::Start(status.offset)).await?; + let len = usize::try_from((status.object.size - status.offset).min(UPLOAD_CHUNK_BYTES as u64))?; + let mut bytes = vec![0; len]; + file.read_exact(&mut bytes) + .await + .context("retained artifact was truncated")?; + let expected = status.offset + len as u64; + match client.append_upload(artifact, database, &status, bytes).await { + Ok(next) => { + ensure!( + next.offset == expected, + "artifact append acknowledged an unexpected offset" + ); + status = next; + ambiguous_attempts = 0; + } + Err(error) => { + if !retryable(&error) { + return Err(error); + } + ambiguous_attempts += 1; + ensure!( + ambiguous_attempts <= 3, + "artifact append outcome remains unknown; resume the same directory" + ); + let next = client + .upload_status(artifact, database, status.id, status.object) + .await?; + ensure!( + next.offset == status.offset || next.offset == expected, + "artifact offset changed outside this append" + ); + status = next; + } + } + store_upload(journal, index, &status)?; + } + status = match client.complete_upload(artifact, database, &status).await { + Ok(status) => status, + Err(error) if retryable(&error) => { + let observed = client + .upload_status(artifact, database, status.id, status.object) + .await?; + if observed.complete { + observed + } else { + return Err(error).context("artifact completion is not confirmed; resume the same directory"); + } + } + Err(error) => return Err(error), + }; + store_upload(journal, index, &status) +} +fn store_upload(journal: &mut Journal, index: usize, status: &UploadStatus) -> Result<()> { + journal.record.uploads[index].session = Some(status.clone()); + journal.save() +} +fn retryable(error: &anyhow::Error) -> bool { + error + .downcast_ref::() + .is_some_and(|error| error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()) + || error.downcast_ref::().is_some_and(|error| { + error.status.is_server_error() + || matches!( + error.status, + reqwest::StatusCode::CONFLICT | reqwest::StatusCode::TOO_MANY_REQUESTS + ) + }) +} +async fn wait_and_name( + client: &PublisherClient, + journal: &mut Journal, + wait: Duration, + cancel: &CancellationToken, +) -> Result { + let deadline = tokio::time::Instant::now() + wait; + let request = journal.record.request()?; + loop { + let status = journal + .record + .status + .clone() + .context("confirmed publication status missing")?; + match status.phase { + PublicationPhase::AbortedBeforeCommit => return Ok(Outcome::Aborted(status)), + PublicationPhase::Complete => { + if let Some(name) = journal + .record + .requested_name + .clone() + .filter(|_| !journal.record.name_confirmed) + { + // Existing name replacement has no CAS token. Do not retry an + // ambiguous name update and overwrite aliases added later. + if journal.record.naming_attempted { + return Ok(Outcome::NamingUnconfirmed(status)); + } + journal.record.naming_attempted = true; + journal.save()?; + if client.set_name(status.database_identity, &name).await.is_err() { + return Ok(Outcome::NamingUnconfirmed(status)); + } + journal.record.name_confirmed = true; + journal.save()?; + } + return Ok(Outcome::Complete(status)); + } + _ => (), + } + if tokio::time::Instant::now() >= deadline { + return Ok(Outcome::Pending(status)); + } + tokio::select! { + _=cancel.cancelled()=>bail!("publication wait cancelled; its durable operation continues, resume the same directory"), + _=tokio::time::sleep_until((tokio::time::Instant::now()+Duration::from_secs(1)).min(deadline))=>(), + } + let next = tokio::time::timeout_at( + deadline, + observe_or_replay(client, journal, status.database_identity, status.operation_id), + ) + .await; + match next { + Ok(Ok(Some(next))) => { + journal.record.check_status(&request, &next)?; + journal.record.status = Some(next); + journal.save()?; + } + Ok(Ok(None)) => bail!("confirmed publication disappeared; keep the same operation and resume directory"), + Ok(Err(error)) => { + return Err(error).context("publication status is not confirmed; resume the same directory") + } + Err(_) => return Ok(Outcome::Pending(status)), + } + } +} diff --git a/crates/cli/src/container/publish/tests.rs b/crates/cli/src/container/publish/tests.rs new file mode 100644 index 00000000000..6c9d04ce73a --- /dev/null +++ b/crates/cli/src/container/publish/tests.rs @@ -0,0 +1,628 @@ +//! These fixtures own numeric-loopback sockets and in-memory credentials. They +//! never load CLI configuration, environment endpoints, or external builders. +use super::*; +use axum::{ + body::Bytes, + extract::State, + http::{HeaderMap, Method, StatusCode, Uri}, + response::{IntoResponse, Response}, + Json, Router, +}; +use client::{ObjectRef, UploadKind, UploadStatus}; +use journal::{Record, UploadRecord}; +use serde_json::json; +use spacetimedb_lib::{ + deployment::{api::*, manifest::*, *}, + Identity, Uuid, +}; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +const TOKEN: &str = "Bearer isolated-fixture-credential"; +pub(crate) fn publisher() -> Identity { + Identity::from_be_byte_array([42; 32]) +} +pub(crate) fn database() -> Identity { + Identity::from_be_byte_array([43; 32]) +} +#[derive(Default)] +pub(crate) struct Behavior { + pub permission: bool, + pub lose_append: bool, + pub lose_completion: bool, + pub lose_submit_after_commit: bool, + pub lose_submit_before_commit: bool, + pub lose_reservation: bool, + pub stale: bool, + pub bad_receipt: bool, + pub bad_completion: bool, + pub bad_status: bool, + pub wrong_reservation: bool, + pub deny_preflight: bool, + pub deny_upload: bool, + pub deny_status: bool, + pub fail_naming: bool, + pub redirect: Option, + pub prior: Option, + pub uploads: BTreeMap)>, + pub submits: Vec>, + pub reservations: Vec, + pub names: usize, + pub preflights: usize, + pub begin_count: usize, + pub status: Option, + pub authenticated: usize, +} +pub(crate) struct Fixture { + pub endpoint: String, + pub state: Arc>, + stop: CancellationToken, + task: tokio::task::JoinHandle<()>, +} +impl Fixture { + pub async fn new() -> Self { + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("http://{}/", listener.local_addr().unwrap()); + let state = Arc::new(Mutex::new(Behavior { + permission: true, + ..Default::default() + })); + let stop = CancellationToken::new(); + let cancellation = stop.clone(); + let app = Router::new() + .fallback(handler) + .layer(axum::extract::DefaultBodyLimit::max(32 * 1024 * 1024)) + .with_state((state.clone(), endpoint.clone())); + let task = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(cancellation.cancelled_owned()) + .await + .unwrap(); + }); + Self { + endpoint, + state, + stop, + task, + } + } + pub fn client(&self) -> PublisherClient { + PublisherClient::new(&self.endpoint, TOKEN.parse().unwrap()).unwrap() + } + pub fn record(&self, new: bool, name: bool) -> Record { + let bytes = SYSTEM_EMPTY_MODULE_V1_BYTES; + let envelope = PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), + expected_revision: None, + module_action: ModuleAction::Set(UserModule { + kind: UserModuleKind::Wasm, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }), + container_action: Default::default(), + }; + let module_artifact = SYSTEM_EMPTY_MODULE_V1_ARTIFACT; + let request = PublishRequest { + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + deployment: envelope.resolve(None, &Default::default()).unwrap(), + envelope, + module_artifact, + migration_policy: PreparedMigrationPolicy::Compatible, + }), + creation: new.then_some(CreationOptions { + parent: None, + organization: None, + num_replicas: None, + enforce_anti_affinity: true, + }), + image_source: None, + }; + let request_json = serde_json::to_string(&request).unwrap(); + Record { + version: 1, + server: self.endpoint.clone(), + artifact_endpoint: self.endpoint.clone(), + publisher: publisher(), + database: (!new).then_some(database()), + reservation: request.creation.clone().map(|options| ReserveDatabaseRequest { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: request.manifest.current().envelope.operation_id, + options, + }), + requested_name: name.then_some("fixture-name".into()), + request_digest: spacetimedb_oci::sha256(request_json.as_bytes()), + request_json, + uploads: vec![UploadRecord { + kind: UploadKind::Module, + object: ObjectRef { + digest: module_artifact.digest, + size: module_artifact.size_bytes, + }, + session: None, + }], + submitted: false, + status: None, + name_confirmed: false, + naming_attempted: false, + } + } + pub async fn close(self) { + self.stop.cancel(); + self.task.await.unwrap(); + } +} +async fn handler( + State((shared, endpoint)): State<(Arc>, String)>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let mut state = shared.lock().unwrap(); + let path = uri.path(); + if path == "/v1/containers/capabilities" { + assert!(headers.get("authorization").is_none()); + if let Some(location) = &state.redirect { + return (StatusCode::TEMPORARY_REDIRECT, [("location", location.clone())]).into_response(); + } + return Json(PublicationCapabilities { + version: 1, + enabled: true, + artifact_endpoint: Some(endpoint), + }) + .into_response(); + } + if headers.get("authorization").and_then(|v| v.to_str().ok()) != Some(TOKEN) { + return StatusCode::UNAUTHORIZED.into_response(); + } + state.authenticated += 1; + if path == "/v1/containers/publish-permission" { + return Json(PublishPermission { + identity: publisher(), + can_publish: state.permission, + source_revision: None, + }) + .into_response(); + } + if path == "/v1/containers/reservations" { + let request: ReserveDatabaseRequest = serde_json::from_slice(&body).unwrap(); + if let Some(prior) = state.reservations.first() { + assert_eq!(prior, &request); + } + state.reservations.push(request.clone()); + if std::mem::take(&mut state.lose_reservation) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(DatabaseReservation { + database_identity: database(), + operation_id: if state.wrong_reservation { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) + } else { + request.operation_id + }, + expires_at: "fixture-only".into(), + staging_open: true, + artifact_endpoint: endpoint, + }) + .into_response(); + } + if path.ends_with("/pre_publish") { + state.preflights += 1; + if state.deny_preflight { + return StatusCode::FORBIDDEN.into_response(); + } + assert_eq!(body.as_ref(), SYSTEM_EMPTY_MODULE_V1_BYTES); + return Json(spacetimedb_client_api_messages::name::PrePublishResult::AutoMigrate( + spacetimedb_client_api_messages::name::PrePublishAutoMigrateResult { + migrate_plan: "fixture migration".into(), + break_clients: false, + token: spacetimedb_lib::Hash::ZERO, + major_version_upgrade: false, + }, + )) + .into_response(); + } + if path.ends_with("/names") { + state.names += 1; + if state.fail_naming { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(spacetimedb_client_api_messages::name::SetDomainsResult::Success).into_response(); + } + if path.contains("/deployment/operations/") { + if state.deny_status { + return StatusCode::FORBIDDEN.into_response(); + } + return state + .status + .clone() + .map(|status| Json(status).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + if path.ends_with("/deployment") { + if method == Method::GET { + return state + .prior + .clone() + .map(|p| Json(p).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + assert_eq!(method, Method::PUT); + state.submits.push(body.to_vec()); + if let Some(status) = &state.status { + assert_eq!(state.submits.first().unwrap().as_slice(), body.as_ref()); + return Json(status.clone()).into_response(); + } + if state.stale { + return (StatusCode::CONFLICT, "private-server-error-must-not-be-printed").into_response(); + } + if std::mem::take(&mut state.lose_submit_before_commit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let request: PublishRequest = serde_json::from_slice(&body).unwrap(); + request.manifest.validate(&Default::default()).unwrap(); + let status = PublicationStatus { + database_identity: database(), + operation_id: request.manifest.current().envelope.operation_id, + phase: PublicationPhase::Complete, + expected_revision: request.manifest.current().envelope.expected_revision, + proposed_revision: if state.bad_status { + spacetimedb_lib::Hash::ZERO + } else { + request.manifest.current().deployment.revision().unwrap() + }, + error: None, + }; + state.status = Some(status.clone()); + if std::mem::take(&mut state.lose_submit_after_commit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(status).into_response(); + } + if path.ends_with("/uploads") { + state.begin_count += 1; + if state.deny_upload { + return StatusCode::FORBIDDEN.into_response(); + } + let request: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + headers["x-spacetimedb-artifact-kind"], + request["kind"].as_str().unwrap() + ); + let object: ObjectRef = serde_json::from_value(request["object"].clone()).unwrap(); + let status = UploadStatus { + id: uuid::Uuid::new_v4(), + object, + offset: 0, + expires_at: u64::MAX, + complete: false, + }; + state.uploads.insert(status.id, (status.clone(), vec![])); + return Json(status).into_response(); + } + if let Some((_, tail)) = path.split_once("/uploads/") { + let id: uuid::Uuid = tail.split('/').next().unwrap().parse().unwrap(); + let Some((status, bytes)) = state.uploads.get_mut(&id) else { + return StatusCode::NOT_FOUND.into_response(); + }; + if method == Method::PATCH { + assert!(body.len() <= client::UPLOAD_CHUNK_BYTES); + let offset: u64 = uri.query().unwrap().strip_prefix("offset=").unwrap().parse().unwrap(); + assert_eq!(offset, status.offset); + bytes.extend_from_slice(&body); + status.offset += body.len() as u64; + } else if path.ends_with("/complete") { + assert_eq!(bytes.len() as u64, status.object.size); + assert_eq!(spacetimedb_oci::sha256(bytes), status.object.digest); + status.complete = true; + } + let mut response = status.clone(); + if path.ends_with("/complete") { + if std::mem::take(&mut state.lose_completion) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let mut object = response.object; + if state.bad_completion { + object.size += 1; + } + return Json(object).into_response(); + } + if method == Method::PATCH && std::mem::take(&mut state.lose_append) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + if state.bad_receipt { + response.object.size += 1; + } + return Json(response).into_response(); + } + panic!("unexpected fixture request: {method} {uri}"); +} +async fn run_now(client: &PublisherClient, journal: &mut Journal) -> Result { + run(client, journal, None, Duration::ZERO, CancellationToken::new()).await +} +fn journal(base: &Path, record: Record) -> Journal { + Journal::create(base, record, None, Some(SYSTEM_EMPTY_MODULE_V1_BYTES)).unwrap() +} +use std::path::Path; + +#[tokio::test] +async fn lost_append_and_committed_response_resume_without_reupload_or_current_permission() { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.lose_append = true; + s.lose_submit_after_commit = true; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.submitted); + let path = journal.directory().to_owned(); + drop(journal); + std::fs::remove_file(path.join("module.blob")).unwrap(); + fixture.state.lock().unwrap().permission = false; + let mut journal = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.submits, [exact.into_bytes()]); + assert_eq!(state.begin_count, 1); + } + drop(journal); + fixture.close().await; +} +#[tokio::test] +async fn lost_reservation_and_unadmitted_put_replay_exact_request_and_generated_identity() { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.lose_reservation = true; + s.lose_submit_before_commit = true; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(true, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.database.is_none()); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert_eq!(journal.record.database, Some(database())); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.reservations.len(), 2); + assert_eq!(state.submits, [exact.as_bytes(), exact.as_bytes()]); + assert_eq!(state.begin_count, 1); + } + fixture.close().await; +} +#[tokio::test] +async fn stale_revision_preserves_operation_and_redacts_response_without_fallback() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().stale = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); + assert!(!format!("{error:#}").contains("private-server-error")); + assert_eq!(journal.record.request_json, exact); + assert!(journal.record.submitted); + assert_eq!(fixture.state.lock().unwrap().submits.len(), 1); + fixture.close().await; +} +#[tokio::test] +async fn mismatched_reservation_or_upload_receipt_never_reaches_admission() { + for reservation in [true, false] { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.wrong_reservation = reservation; + s.bad_receipt = !reservation; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(reservation, false)); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(!journal.record.submitted); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + fixture.close().await; + } +} +#[tokio::test] +async fn completion_descriptor_must_match_before_admission() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().bad_completion = true; + let directory = tempfile::tempdir().unwrap(); + let mut journal = journal(directory.path(), fixture.record(true, false)); + let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); + assert!(error.to_string().contains("completion descriptor changed")); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + assert!(!journal.record.submitted); + fixture.state.lock().unwrap().bad_completion = false; + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + let status = journal.record.uploads[0].session.as_ref().unwrap(); + assert!(status.complete); + assert_eq!(status.offset, status.object.size); + fixture.close().await; +} + +#[tokio::test] +async fn lost_completion_response_observes_the_same_complete_session() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_completion = true; + let directory = tempfile::tempdir().unwrap(); + let mut journal = journal(directory.path(), fixture.record(true, false)); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + let session = journal.record.uploads[0].session.as_ref().unwrap(); + assert!(session.complete); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.begin_count, 1); + assert_eq!(state.uploads.len(), 1); + assert!(state.uploads[&session.id].0.complete); + } + fixture.close().await; +} + +#[tokio::test] +async fn wrong_publication_scope_and_denied_upload_are_not_accepted() { + for denied in [true, false] { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.deny_upload = denied; + s.bad_status = !denied; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.status.is_none()); + assert_eq!(fixture.state.lock().unwrap().submits.len(), usize::from(!denied)); + fixture.close().await; + } +} +#[tokio::test] +async fn naming_failure_reports_active_identity_without_republishing_or_overwriting_later_names() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().fail_naming = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(true, true)); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::NamingUnconfirmed(_) + )); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::NamingUnconfirmed(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.names, 1); + assert_eq!(state.submits.len(), 1); + } + fixture.close().await; +} +#[tokio::test] +async fn redirect_and_cross_origin_never_forward_credentials_without_exact_approval() { + let first = Fixture::new().await; + let foreign = Fixture::new().await; + first.state.lock().unwrap().redirect = Some(format!("{}v1/containers/publish-permission", foreign.endpoint)); + assert!(first.client().capabilities().await.is_err()); + assert!(first.client().artifact_endpoint(&foreign.endpoint, None).is_err()); + assert!(first + .client() + .artifact_endpoint(&foreign.endpoint, Some(&first.endpoint)) + .is_err()); + assert!(first + .client() + .artifact_endpoint(&foreign.endpoint, Some(&foreign.endpoint)) + .is_ok()); + assert_eq!(foreign.state.lock().unwrap().authenticated, 0); + first.close().await; + foreign.close().await; +} +#[tokio::test] +async fn journal_locks_and_reverifies_local_bytes_without_persisting_credentials() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let path = journal.directory().to_owned(); + assert!(Journal::open(&path).is_err()); + let bytes = std::fs::read_to_string(path.join("publication.json")).unwrap(); + assert!(!bytes.contains("isolated-fixture-credential")); + std::fs::write(path.join("module.blob"), vec![0; SYSTEM_EMPTY_MODULE_V1_BYTES.len()]).unwrap(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert_eq!(fixture.state.lock().unwrap().begin_count, 0); + drop(journal); + let mut value: serde_json::Value = serde_json::from_str(&bytes).unwrap(); + value["request_json"] = json!("{}"); + std::fs::write(path.join("publication.json"), serde_json::to_vec(&value).unwrap()).unwrap(); + assert!(Journal::open(&path).is_err()); + fixture.close().await; +} +#[test] +fn endpoints_and_receipts_are_bounded_and_unambiguous() { + for endpoint in [ + "Maincloud", + "http://10.0.0.1", + "https://user:password@example.invalid", + "https://example.invalid/?token=secret", + ] { + assert!(client::endpoint(endpoint).is_err()); + } + for endpoint in ["http://localhost:3000", "http://[::1]:3000"] { + assert!(client::endpoint(endpoint).is_ok()); + } + let object = ObjectRef { + digest: SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size: 1, + }; + let status = UploadStatus { + id: uuid::Uuid::now_v7(), + object, + offset: 0, + expires_at: 0, + complete: false, + }; + assert!(status.validate(object, None).is_err()); +} + +#[tokio::test] +async fn incomplete_artifact_retention_never_publishes_a_resume_record_or_mutates_server() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let base = temporary.path().join("new").join("nested").join("state"); + let record = fixture.record(false, false); + let id = record.request().unwrap().manifest.current().envelope.operation_id; + // A missing promised module fails the artifact flush before publication.json + // can make this operation available for reservation or admission. + assert!(Journal::create(&base, record, None, None).is_err()); + assert!(!base.join(id.to_string()).exists()); + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} + +#[tokio::test] +async fn revoked_status_access_replays_original_put_without_missing_local_artifacts_or_new_uploads() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_after_commit = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + let path = journal.directory().to_owned(); + drop(journal); + std::fs::remove_file(path.join("module.blob")).unwrap(); + { + let mut state = fixture.state.lock().unwrap(); + state.permission = false; + state.deny_status = true; + state.deny_upload = true; + } + let mut journal = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.submits, [exact.as_bytes(), exact.as_bytes()]); + assert_eq!(state.begin_count, 1); + } + fixture.close().await; +} diff --git a/crates/cli/src/container/tests.rs b/crates/cli/src/container/tests.rs new file mode 100644 index 00000000000..6282db120f9 --- /dev/null +++ b/crates/cli/src/container/tests.rs @@ -0,0 +1,720 @@ +use super::*; +use crate::spacetime_config::SpacetimeConfig; +use serde_json::json; +use std::{collections::BTreeMap, sync::Mutex}; + +static TEST_PREPARATIONS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn platform() -> ImagePlatform { + ImagePlatform { + os: "linux".into(), + architecture: "amd64".into(), + } +} +pub(crate) fn declaration(image: serde_json::Value) -> ContainerConfig { + serde_json::from_value(json!({"image":image,"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}})).unwrap() +} +fn blob(layout: &Path, bytes: &[u8], media: &str) -> Descriptor { + let digest = spacetimedb_oci::sha256(bytes); + fs::create_dir_all(layout.join("blobs/sha256")).unwrap(); + fs::write( + layout + .join("blobs/sha256") + .join(digest.to_string().strip_prefix("sha256:").unwrap()), + bytes, + ) + .unwrap(); + Descriptor { + media_type: media.into(), + digest, + size: bytes.len() as u64, + platform: None, + urls: vec![], + data: None, + artifact_type: None, + } +} +pub(crate) fn fixture(layout: &Path) -> Descriptor { + let mut archive = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(4); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, "app.js", &b"code"[..]).unwrap(); + let layer = archive.into_inner().unwrap(); + let layer = blob(layout, &layer, "application/vnd.oci.image.layer.v1.tar"); + let config = blob(layout, &serde_json::to_vec(&json!({"architecture":"amd64","os":"linux","config":{"Entrypoint":["/usr/bin/env"],"Cmd":["node","app.js"],"User":"1000:1000","WorkingDir":"/app","Env":["BAKED=private-image-value"]},"rootfs":{"type":"layers","diff_ids":[layer.digest]}})).unwrap(), spacetimedb_oci::OCI_CONFIG); + let manifest = blob( + layout, + &serde_json::to_vec( + &json!({"schemaVersion":2,"mediaType":spacetimedb_oci::OCI_MANIFEST,"config":config,"layers":[layer]}), + ) + .unwrap(), + spacetimedb_oci::OCI_MANIFEST, + ); + fs::write(layout.join("oci-layout"), br#"{"imageLayoutVersion":"1.0.0"}"#).unwrap(); + fs::write( + layout.join("index.json"), + serde_json::to_vec(&json!({"schemaVersion":2,"mediaType":spacetimedb_oci::OCI_INDEX,"manifests":[manifest]})) + .unwrap(), + ) + .unwrap(); + manifest +} +fn copy_layout(source: &Path, output: &Path) { + fs::create_dir_all(output.join("blobs/sha256")).unwrap(); + for name in ["oci-layout", "index.json"] { + fs::copy(source.join(name), output.join(name)).unwrap(); + } + for file in fs::read_dir(source.join("blobs/sha256")).unwrap() { + let file = file.unwrap(); + fs::copy(file.path(), output.join("blobs/sha256").join(file.file_name())).unwrap(); + } +} +type RecordedCall = (String, Vec, Vec); + +struct FakeRunner { + layout: PathBuf, + calls: Mutex>, + fail_detection: bool, + version: &'static str, +} +impl FakeRunner { + fn new(layout: &Path) -> Self { + Self { + layout: layout.into(), + calls: Mutex::new(vec![]), + fail_detection: false, + version: "railpack 0.35.0\n", + } + } +} +impl Runner for FakeRunner { + async fn run(&self, invocation: Invocation) -> Result { + self.calls.lock().unwrap().push(( + invocation.label.into(), + invocation.args.clone(), + invocation.env.iter().map(|(name, _)| name.clone()).collect(), + )); + match invocation.label { + "Railpack version check" => { + return Ok(process::Output { + stdout: self.version.as_bytes().to_vec(), + }) + } + "Railpack detection" => { + ensure!(!self.fail_detection, "unsupported source detection"); + fs::write(invocation.workspace.path().join("railpack-plan.json"), b"{}")?; + } + "Skopeo image import" => copy_layout(&self.layout, &invocation.workspace.path().join("input")), + "BuildKit OCI build" => { + let file = fs::File::create(invocation.workspace.path().join("image.tar"))?; + let mut tar = tar::Builder::new(file); + for name in ["oci-layout", "index.json"] { + tar.append_path_with_name(self.layout.join(name), name)?; + } + for blob in fs::read_dir(self.layout.join("blobs/sha256"))? { + let blob = blob?; + tar.append_path_with_name(blob.path(), Path::new("blobs/sha256").join(blob.file_name()))?; + } + tar.finish()?; + } + _ => anyhow::bail!("unexpected fake image tool"), + } + Ok(process::Output { stdout: vec![] }) + } +} + +#[test] +fn image_source_is_exclusive_and_builder_defaults_are_strict() { + for image in [ + json!({}), + json!({"build":{},"oci_ref":"example/image"}), + json!({"build":{"builder":"unknown"}}), + json!({"build":{"builder":"railpack","dockerfile":"Dockerfile"}}), + ] { + let mut config = serde_json::to_value(declaration(json!({"oci_ref":"example/image"}))).unwrap(); + config["image"] = image; + assert!(serde_json::from_value::(config).is_err()); + } + assert!(matches!( + declaration(json!({"build":{}})).image, + ImageSource::Build(config::BuildImage { + build: SourceBuild::Dockerfile { .. } + }) + )); +} + +#[test] +fn container_declarations_never_inherit_through_children_or_overrides() { + let a = serde_json::to_value(declaration(json!({"build":{}}))).unwrap(); + let b = serde_json::to_value(declaration(json!({"oci_ref":"example/child"}))).unwrap(); + let config: SpacetimeConfig = serde_json::from_value(json!({"database":"root","module-path":"shared","container":a,"children":[{"database":"plain","children":[{"database":"grandchild"}]},{"database":"own","container":b,"children":[{"database":"own-grandchild"}]}]})).unwrap(); + let targets = config.collect_all_targets_with_inheritance(); + let declarations: BTreeMap<_, _> = targets + .iter() + .map(|target| (target.fields["database"].as_str().unwrap(), target.container.is_some())) + .collect(); + assert_eq!( + declarations, + BTreeMap::from([ + ("root", true), + ("plain", false), + ("grandchild", false), + ("own", true), + ("own-grandchild", false) + ]) + ); + assert!(targets.iter().all(|target| !target.fields.contains_key("container"))); + assert!(crate::subcommands::container::select(&config, Some("plain")).is_err()); + assert!(crate::subcommands::container::select(&config, Some("not-a-local-target")).is_err()); + assert!(crate::subcommands::container::select(&config, None).is_err()); + assert!(crate::subcommands::container::select(&config, Some("own")).is_ok()); +} + +#[tokio::test] +async fn local_prebuilt_output_is_owned_verified_and_keeps_image_defaults() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("prebuilt image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let token = CancellationToken::new(); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"oci:prebuilt image"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + token.clone(), + ) + .await + .unwrap(); + assert!(!token.is_cancelled()); + assert!(runner.calls.lock().unwrap().is_empty()); + assert_eq!(prepared.metadata.manifest.digest, manifest.digest); + assert_eq!(prepared.metadata.container.argv, ["/usr/bin/env", "node", "app.js"]); + assert_eq!(prepared.metadata.container.user, "1000:1000"); + assert_eq!(prepared.metadata.container.working_directory, "/app"); + assert!(!serde_json::to_string(&prepared.metadata) + .unwrap() + .contains("private-image-value")); + assert_eq!(prepared.metadata.objects.len(), 3); + let temporary = prepared.layout(); + let output = root.path().join("reusable"); + prepared.persist(&output).unwrap(); + assert!(!temporary.exists()); + assert!(output.join("prepared.json").is_file()); + let reread = prepare_container( + &declaration(json!({"oci_ref":"oci:reusable"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + token, + ) + .await + .unwrap(); + assert_eq!(reread.metadata.manifest.digest, manifest.digest); + let temporary = reread.layout(); + drop(reread); + assert!(!temporary.exists()); +} + +#[tokio::test] +async fn command_override_replaces_image_argv_and_invalid_bytes_never_persist() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let mut config = declaration(json!({"oci_ref":"oci:image"})); + config.command = Some(vec!["/bin/custom".into()]); + let prepared = prepare_container( + &config, + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(prepared.metadata.container.argv, ["/bin/custom"]); + drop(prepared); + fs::write( + input + .join("blobs/sha256") + .join(manifest.digest.to_string().strip_prefix("sha256:").unwrap()), + b"tampered", + ) + .unwrap(); + assert!(prepare_container( + &config, + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert!(fs::read_dir(root.path()).unwrap().all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".spacetime-image-"))); +} + +#[tokio::test] +async fn dockerfile_and_railpack_have_explicit_local_endpoint_and_separate_secrets() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("fixture"); + fixture(&input); + let context = root.path().join("source with spaces"); + fs::create_dir(&context).unwrap(); + fs::write(context.join("Dockerfile"), b"FROM scratch").unwrap(); + let secret = root.path().join("secret file"); + fs::write(&secret, b"sensitive-build-value").unwrap(); + for builder in ["dockerfile", "railpack"] { + let runner = FakeRunner::new(&input); + let tools = BuildTools { + buildkit_host: Some("unix:///disposable/fake-buildkit.sock".into()), + secrets: vec![BuildSecret { + name: "BUILD_KEY".into(), + file: secret.clone(), + }], + ..Default::default() + }; + let config = declaration(json!({"build":{"builder":builder,"context":"source with spaces"}})); + let prepared = prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + let calls = runner.calls.lock().unwrap(); + let (_, argv, environment) = calls + .iter() + .find(|(label, _, _)| label == "BuildKit OCI build") + .unwrap(); + assert_eq!(&argv[..2], ["--addr", "unix:///disposable/fake-buildkit.sock"]); + assert!(argv.contains(&format!("context={}", context.canonicalize().unwrap().display()).into())); + assert!(argv.contains(&"--no-cache".into())); + assert!(environment.contains(&"DOCKER_CONFIG".into())); + assert!(!format!("{argv:?}").contains("sensitive-build-value")); + assert!(!serde_json::to_string(&prepared.metadata).unwrap().contains("BUILD_KEY")); + if builder == "railpack" { + assert!(argv.contains(&format!("source={RAILPACK_FRONTEND}").into())); + let (_, args, _) = calls + .iter() + .find(|(label, _, _)| label == "Railpack detection") + .unwrap(); + assert!(args.contains(&"BUILD_KEY=".into())); + } else { + assert_eq!(calls.len(), 1); + } + } +} + +#[tokio::test] +async fn unsupported_railpack_and_remote_builder_never_fall_back() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let config = declaration(json!({"build":{"builder":"railpack"}})); + let tools = BuildTools { + buildkit_host: Some("unix:///disposable/fake-buildkit.sock".into()), + ..Default::default() + }; + let mut runner = FakeRunner::new(&input); + runner.fail_detection = true; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert_eq!(runner.calls.lock().unwrap().len(), 2); + runner.calls.lock().unwrap().clear(); + runner.version = "railpack 99.0.0"; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert_eq!(runner.calls.lock().unwrap().len(), 1); + runner.calls.lock().unwrap().clear(); + let tools = BuildTools { + buildkit_host: Some("tcp://untrusted:1234".into()), + ..Default::default() + }; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert!(runner.calls.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn registry_import_uses_explicit_anonymous_auth_and_records_selected_digest() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"example.invalid/team/image:tag"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(prepared.metadata.manifest.digest, manifest.digest); + let calls = runner.calls.lock().unwrap(); + assert_eq!(calls.len(), 1); + let args = &calls[0].1; + assert!(args.contains(&"--preserve-digests".into())); + assert!(args.contains(&"docker://example.invalid/team/image:tag".into())); + let auth = PathBuf::from(&args[args.iter().position(|arg| arg == "--authfile").unwrap() + 1]); + assert_eq!(fs::read(auth).unwrap(), br#"{"auths":{}}"#); +} + +#[test] +fn archive_rejects_links_and_cancelled_reads() { + let root = tempfile::tempdir().unwrap(); + for kind in [tar::EntryType::Symlink, tar::EntryType::Link] { + let path = root.path().join("input.tar"); + let mut tar = tar::Builder::new(fs::File::create(&path).unwrap()); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(kind); + header.set_size(0); + header.set_link_name("/outside").unwrap(); + header.set_cksum(); + tar.append_data(&mut header, "index.json", std::io::empty()).unwrap(); + tar.finish().unwrap(); + let output = tempfile::tempdir().unwrap(); + assert!(oci::extract_archive( + &path, + output.path(), + &CancellationToken::new(), + Instant::now() + Duration::from_secs(1) + ) + .is_err()); + } + let token = CancellationToken::new(); + token.cancel(); + assert!(oci::check(&token, Instant::now() + Duration::from_secs(1)).is_err()); +} + +#[test] +fn container_build_command_requires_explicit_platform_and_output() { + crate::subcommands::container::cli().debug_assert(); + assert!(crate::subcommands::container::cli() + .try_get_matches_from(["container", "build"]) + .is_err()); + assert!(crate::subcommands::container::cli() + .try_get_matches_from([ + "container", + "build", + "local-target", + "--platform", + "linux/amd64", + "--out-dir", + "output" + ]) + .is_ok()); +} + +#[test] +fn publish_preserves_container_target_metadata_without_affecting_plain_children() { + use crate::subcommands::publish::{build_publish_schema, get_filtered_publish_configs}; + let config: SpacetimeConfig = serde_json::from_value(json!({ + "database":"container-db", "container":declaration(json!({"build":{}})), + "children":[{"database":"ordinary-db"}] + })) + .unwrap(); + let command = crate::subcommands::publish::cli(); + let schema = build_publish_schema(&command).unwrap(); + for selected in ["container-db", "*"] { + let args = command.clone().try_get_matches_from(["publish", selected]).unwrap(); + let targets = get_filtered_publish_configs(&config, &command, &schema, &args).unwrap(); + assert!(targets[0].container().is_some()); + assert!(targets.iter().skip(1).all(|target| target.container().is_none())); + } + let args = command + .clone() + .try_get_matches_from(["publish", "ordinary-db"]) + .unwrap(); + assert_eq!( + get_filtered_publish_configs(&config, &command, &schema, &args) + .unwrap() + .len(), + 1 + ); +} + +#[tokio::test] +async fn local_index_defaults_media_type_but_never_overwrites_existing_output() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let mut index: serde_json::Value = serde_json::from_slice(&fs::read(input.join("index.json")).unwrap()).unwrap(); + index.as_object_mut().unwrap().remove("mediaType"); + fs::write(input.join("index.json"), serde_json::to_vec(&index).unwrap()).unwrap(); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"oci:image"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &FakeRunner::new(&input), + CancellationToken::new(), + ) + .await + .unwrap(); + let output = root.path().join("existing-empty-output"); + fs::create_dir(&output).unwrap(); + assert!(prepared.persist(&output).is_err()); + assert!(fs::read_dir(&output).unwrap().next().is_none()); +} + +#[test] +fn wrong_platform_and_duplicate_archive_object_are_rejected() { + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let wrong = ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }; + assert!(oci::verify_layout( + &input, + &root.path().join("output"), + &wrong, + &CancellationToken::new(), + Instant::now() + Duration::from_secs(2) + ) + .is_err()); + let archive = root.path().join("duplicate.tar"); + let mut tar = tar::Builder::new(fs::File::create(&archive).unwrap()); + for _ in 0..2 { + tar.append_path_with_name(input.join("index.json"), "index.json") + .unwrap(); + } + tar.finish().unwrap(); + assert!(oci::extract_archive( + &archive, + &root.path().join("duplicate"), + &CancellationToken::new(), + Instant::now() + Duration::from_secs(2) + ) + .is_err()); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[tokio::test] +async fn subprocess_exit_cancellation_caller_drop_and_output_overflow_reap_before_workspace_release() { + use process::LocalRunner; + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + // These scripts exercise only owned local subprocesses. No Docker, server, + // network, saved configuration, or user credentials are involved. + for mode in [ + "exit", + "exit_without_descendants", + "cancel", + "drop", + "overflow", + "timeout", + ] { + let workspace = Arc::new( + tempfile::Builder::new() + .prefix("fake-tool-") + .tempdir_in(root.path()) + .unwrap(), + ); + let path = workspace.path().to_path_buf(); + let pid_file = root.path().join(format!("{mode}.pid")); + let script = match mode { + "exit" => "echo $$ > \"$1\"; sleep 60 & exit 0", + "exit_without_descendants" => "echo $$ > \"$1\"; exit 0", + "overflow" => "echo $$ > \"$1\"; yes x", + _ => "echo $$ > \"$1\"; sleep 60 & wait", + }; + let token = CancellationToken::new(); + let task = tokio::spawn(LocalRunner.run(Invocation { + tool: "/bin/sh".into(), + label: "owned fake builder", + args: vec![ + "-c".into(), + script.into(), + "fake-builder".into(), + pid_file.clone().into_os_string(), + ], + env: vec![], + cwd: root.path().into(), + workspace, + timeout: if mode == "timeout" { + Duration::from_millis(250) + } else { + Duration::from_secs(4) + }, + cancel: token.clone(), + })); + tokio::time::timeout(Duration::from_secs(2), async { + while !pid_file.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let pid = fs::read_to_string(&pid_file).unwrap().trim().parse::().unwrap(); + let pid = rustix::process::Pid::from_raw(pid).unwrap(); + if mode == "cancel" { + token.cancel(); + } + if mode == "drop" { + task.abort(); + let _ = task.await; + } else { + let result = tokio::time::timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap(); + assert_eq!(result.is_ok(), matches!(mode, "exit" | "exit_without_descendants")); + } + tokio::time::timeout(Duration::from_secs(3), async { + while path.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!( + matches!( + rustix::process::waitpid(Some(pid), rustix::process::WaitOptions::NOHANG), + Err(rustix::io::Errno::CHILD) + ), + "builder leader must already be reaped" + ); + } +} + +#[test] +fn configuration_overlay_replaces_only_the_selected_container_declaration() { + let root = tempfile::tempdir().unwrap(); + let parent = declaration(json!({"build":{}})); + let child = declaration(json!({"oci_ref":"example.invalid/child:v1"})); + fs::write(root.path().join("spacetime.json"),serde_json::to_vec(&json!({"database":"parent","container":parent,"children":[{"database":"plain"},{"database":"own","container":child}]})).unwrap()).unwrap(); + let replacement = declaration(json!({"build":{"builder":"railpack"}})); + fs::write( + root.path().join("spacetime.dev.json"), + serde_json::to_vec(&json!({"container":replacement})).unwrap(), + ) + .unwrap(); + let loaded = crate::spacetime_config::find_and_load_with_env_from(Some("dev"), root.path().into()) + .unwrap() + .unwrap(); + assert!(matches!( + crate::subcommands::container::select(&loaded.config, Some("parent")) + .unwrap() + .image, + ImageSource::Build(config::BuildImage { + build: SourceBuild::Railpack { .. } + }) + )); + assert!(crate::subcommands::container::select(&loaded.config, Some("plain")).is_err()); + assert!(matches!( + crate::subcommands::container::select(&loaded.config, Some("own")) + .unwrap() + .image, + ImageSource::Prebuilt(_) + )); +} + +#[tokio::test] +async fn build_command_executes_local_import_with_only_project_configuration() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + fixture(&root.path().join("input")); + fs::write( + root.path().join("spacetime.json"), + serde_json::to_vec(&json!({ + "database":"local-selection", "server":"https://must-not-be-contacted.invalid", + "container":declaration(json!({"oci_ref":"oci:input"})) + })) + .unwrap(), + ) + .unwrap(); + let output = root.path().join("output"); + let args = crate::subcommands::container::cli() + .try_get_matches_from([ + OsString::from("container"), + "build".into(), + "local-selection".into(), + "--project-path".into(), + root.path().into(), + "--out-dir".into(), + output.clone().into_os_string(), + "--platform".into(), + "linux/amd64".into(), + ]) + .unwrap(); + crate::exec_local_subcommand("container", &args).await.unwrap().unwrap(); + let metadata: PreparedMetadata = serde_json::from_slice(&fs::read(output.join("prepared.json")).unwrap()).unwrap(); + assert_eq!(metadata.container.argv, ["/usr/bin/env", "node", "app.js"]); + assert_eq!(metadata.objects.len(), 3); +} + +#[test] +fn omitted_image_working_directory_normalizes_to_linux_root() { + let declaration = declaration(json!({"oci_ref":"example.invalid/no-working-dir"})); + let image = spacetimedb_oci::ContainerConfig { + cmd: Some(vec!["/app".into()]), + ..Default::default() + }; + let normalized = declaration + .normalize(spacetimedb_oci::sha256(b"fixture"), platform(), &image) + .unwrap(); + assert_eq!(normalized.working_directory, "/"); + let mut explicit = declaration; + explicit.working_directory = Some(String::new()); + assert!(explicit + .normalize(spacetimedb_oci::sha256(b"fixture"), platform(), &image) + .is_err()); +} diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 9411e6f5e5e..4e9fa358c35 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,6 +1,7 @@ pub mod api; mod common_args; mod config; +pub mod container; pub(crate) mod detect; mod edit_distance; mod errors; @@ -36,6 +37,7 @@ pub fn get_subcommands() -> Vec { logout::cli(), init::cli(), build::cli(), + subcommands::container::cli(), server::cli(), subscribe::cli(), start::cli(), @@ -45,6 +47,23 @@ pub fn get_subcommands() -> Vec { ] } +/// Dispatch commands that need only project files before opening saved CLI +/// server settings or credentials. Future container network commands use the +/// ordinary authenticated dispatcher below. +pub async fn exec_local_subcommand(cmd: &str, args: &ArgMatches) -> Option> { + if cmd == "container" + && let Some(("build", args)) = args.subcommand() + { + Some( + subcommands::container::exec_build(args) + .await + .map(|()| ExitCode::SUCCESS), + ) + } else { + None + } +} + pub async fn exec_subcommand( config: Config, paths: &SpacetimePaths, @@ -66,6 +85,7 @@ pub async fn exec_subcommand( "list" => list::exec(config, args).await, "init" => init::exec(config, args).await.map(|_| ()), "build" => build::exec(config, args).await.map(drop), + "container" => subcommands::container::exec(config, args).await, "server" => server::exec(config, paths, args).await, "subscribe" => subscribe::exec(config, args).await, "start" => return start::exec(config, paths, args).await, diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index d6f58757eea..e5bf606175f 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -32,6 +32,10 @@ async fn main() -> anyhow::Result { let matches = get_command().get_matches(); let (cmd, subcommand_args) = matches.subcommand().unwrap(); + if let Some(result) = exec_local_subcommand(cmd, subcommand_args).await { + return result; + } + let root_dir = matches.get_one::("root_dir"); let paths = match root_dir { Some(dir) => SpacetimePaths::from_root_dir(dir), @@ -135,3 +139,14 @@ Commands: "#, ) } + +#[cfg(test)] +mod tests { + #[test] + fn managed_publish_help_passes_entrypoint_validation() { + let help = super::get_command() + .try_get_matches_from(["spacetime", "publish", "--help"]) + .unwrap_err(); + assert_eq!(help.kind(), clap::error::ErrorKind::DisplayHelp); + } +} diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index b3316f2c318..dc493abba8f 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -108,6 +108,9 @@ pub enum CommandConfigError { #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "kebab-case")] pub struct SpacetimeConfig { + /// Container declaration belongs only to this database, never its children. + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, /// Configuration for the dev command. Root-level only, not inherited. #[serde(skip_serializing_if = "Option::is_none")] pub dev: Option, @@ -144,6 +147,7 @@ pub struct DevConfig { /// Contains all fields needed for both publish and generate operations. #[derive(Debug, Clone)] pub struct FlatTarget { + pub container: Option, /// All entity-level fields (database, module-path, server, etc.) pub fields: HashMap, /// Name of the config file from which this target's `database` value was merged. @@ -201,6 +205,7 @@ impl SpacetimeConfig { let effective_generate = self.generate.clone(); let target = FlatTarget { + container: self.container.clone(), fields: fields.clone(), source_config: self.source_config.clone(), generate: effective_generate, @@ -253,6 +258,8 @@ pub struct CommandConfig<'a> { config_values: HashMap, /// CLI arguments matches: &'a ArgMatches, + /// A declaration belongs to this exact target and never inherits. + container: Option, } /// Schema that defines the contract between CLI arguments and config file keys. @@ -699,9 +706,23 @@ impl<'a> CommandConfig<'a> { schema, config_values: normalized_values, matches, + container: None, }) } + pub fn with_container(mut self, container: Option) -> Self { + self.container = container; + self + } + + pub fn container(&self) -> Option<&crate::container::config::ContainerConfig> { + self.container.as_ref() + } + + pub(crate) fn matches(&self) -> &ArgMatches { + self.matches + } + /// Get a single value from the config as a specific type. /// First checks clap args (via schema), then falls back to config values. /// diff --git a/crates/cli/src/subcommands/container.rs b/crates/cli/src/subcommands/container.rs new file mode 100644 index 00000000000..5e66ef2eeed --- /dev/null +++ b/crates/cli/src/subcommands/container.rs @@ -0,0 +1,186 @@ +//! Container build and operation commands. Local builds do not read saved +//! server credentials; network commands use the explicitly selected server. +mod operations; +mod url; + +use crate::{ + container::{config::ContainerConfig, prepare_container, process::LocalRunner, BuildSecret, BuildTools}, + spacetime_config::{find_and_load_with_env_from, SpacetimeConfig}, +}; +use anyhow::{ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use std::path::{Path, PathBuf}; +use tokio_util::sync::CancellationToken; + +pub fn cli() -> Command { + let command = Command::new("container") + .about("Build and manage a database's container") + .subcommand_required(true) + .subcommand(url::cli()) + .subcommand( + Command::new("build") + .about("Prepare verified OCI artifacts locally without publishing") + .arg(Arg::new("database").help("Database target in local spacetime.json; no server lookup")) + .arg( + Arg::new("project_path") + .long("project-path") + .default_value(".") + .value_parser(clap::value_parser!(PathBuf)) + .help("Directory in which to find spacetime.json"), + ) + .arg( + Arg::new("out_dir") + .long("out-dir") + .required(true) + .value_parser(clap::value_parser!(PathBuf)) + .help("New directory for verified OCI artifacts and prepared.json"), + ) + .arg( + Arg::new("platform") + .long("platform") + .required(true) + .value_parser(["linux/amd64", "linux/arm64"]) + .help("Target Linux platform, independent of this computer's architecture"), + ) + .arg(Arg::new("env").long("env").help("Local configuration overlay name")) + .arg( + Arg::new("buildkit_host") + .long("buildkit-host") + .help("Explicit local BuildKit Unix socket, required for source builds"), + ) + .arg( + Arg::new("buildctl") + .long("buildctl") + .default_value("buildctl") + .value_parser(clap::value_parser!(PathBuf)) + .help("BuildKit client executable"), + ) + .arg( + Arg::new("railpack") + .long("railpack") + .default_value("railpack") + .value_parser(clap::value_parser!(PathBuf)) + .help("Pinned Railpack executable for explicitly selected Railpack builds"), + ) + .arg( + Arg::new("skopeo") + .long("skopeo") + .default_value("skopeo") + .value_parser(clap::value_parser!(PathBuf)) + .help("Skopeo executable for prebuilt registry images"), + ) + .arg( + Arg::new("registry_auth_file") + .long("registry-auth-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Explicit registry auth JSON; omitted means anonymous, never saved Docker credentials"), + ) + .arg( + Arg::new("build_secret") + .long("build-secret") + .action(ArgAction::Append) + .value_name("NAME=FILE") + .help("Explicit build secret file; separate from runtime env_keys"), + ), + ); + operations::commands(command) +} + +pub(crate) fn select(config: &SpacetimeConfig, database: Option<&str>) -> Result { + let targets = config.collect_all_targets_with_inheritance(); + let selected = if let Some(database) = database { + let mut matches = targets + .iter() + .filter(|target| target.fields.get("database").and_then(|v| v.as_str()) == Some(database)); + let target = matches + .next() + .context("database target is not in local spacetime.json")?; + ensure!( + matches.next().is_none(), + "database target is ambiguous in local spacetime.json" + ); + target + } else { + let mut matches = targets.iter().filter(|target| target.container.is_some()); + let target = matches + .next() + .context("no container declaration in local spacetime.json")?; + ensure!( + matches.next().is_none(), + "several container targets exist; select a DATABASE from local spacetime.json" + ); + target + }; + selected + .container + .clone() + .context("selected database has no container declaration; containers are not inherited") +} + +pub async fn exec(mut config: crate::Config, args: &ArgMatches) -> Result<()> { + match args.subcommand().context("missing container command")? { + ("build", args) => exec_build(args).await, + ("url", args) => url::exec(&config, args).await, + (name @ ("status" | "start" | "stop" | "restart"), args) => operations::exec(&mut config, name, args).await, + _ => anyhow::bail!("unsupported container command"), + } +} + +pub async fn exec_build(args: &ArgMatches) -> Result<()> { + let project = args.get_one::("project_path").unwrap().canonicalize()?; + let loaded = find_and_load_with_env_from(args.get_one::("env").map(String::as_str), project)? + .context("spacetime.json not found")?; + let declaration = select(&loaded.config, args.get_one::("database").map(String::as_str))?; + let output = std::env::current_dir()?.join(args.get_one::("out_dir").unwrap()); + ensure!(!output.exists(), "output already exists: {}", output.display()); + let parent = output.parent().unwrap_or(Path::new(".")); + ensure!(parent.is_dir(), "output parent directory does not exist"); + let mut tools = BuildTools { + buildctl: args.get_one::("buildctl").unwrap().clone(), + railpack: args.get_one::("railpack").unwrap().clone(), + skopeo: args.get_one::("skopeo").unwrap().clone(), + buildkit_host: args.get_one::("buildkit_host").cloned(), + registry_auth_file: args.get_one::("registry_auth_file").cloned(), + secrets: vec![], + }; + for secret in args.get_many::("build_secret").into_iter().flatten() { + let (name, file) = secret + .split_once('=') + .context("build secret must be NAME=FILE, not a value")?; + ensure!(!file.is_empty(), "build secret file is missing"); + tools.secrets.push(BuildSecret { + name: name.into(), + file: file.into(), + }); + } + let (os, architecture) = args.get_one::("platform").unwrap().split_once('/').unwrap(); + let cancel = CancellationToken::new(); + let prepared = { + let prepare = prepare_container( + &declaration, + &loaded.config_dir, + spacetimedb_lib::container::ImagePlatform { + os: os.into(), + architecture: architecture.into(), + }, + &tools, + parent, + &LocalRunner, + cancel.clone(), + ); + tokio::pin!(prepare); + tokio::select! { + result = &mut prepare => result?, + signal = tokio::signal::ctrl_c() => { + signal?; + cancel.cancel(); + let _ = prepare.await; + anyhow::bail!("container build cancelled; no output was published"); + } + } + }; + let digest = prepared.metadata.manifest.digest; + prepared.persist(&output)?; + println!("Prepared {digest} at {}", output.display()); + Ok(()) +} diff --git a/crates/cli/src/subcommands/container/operations.rs b/crates/cli/src/subcommands/container/operations.rs new file mode 100644 index 00000000000..3236baca42d --- /dev/null +++ b/crates/cli/src/subcommands/container/operations.rs @@ -0,0 +1,315 @@ +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use reqwest::{ + header::{HeaderValue, AUTHORIZATION}, + Client, Method, StatusCode, Url, +}; +use serde::de::DeserializeOwned; +use spacetimedb_lib::{ + container::{endpoints::ContainerEndpoints, operations::*}, + Identity, Uuid, +}; +use std::time::Duration; + +const MAX_RESPONSE: usize = 64 * 1024; +pub(super) fn commands(command: Command) -> Command { + let base = |name: &'static str, about: &'static str| { + Command::new(name) + .about(about) + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg(crate::common_args::server()) + .arg(crate::common_args::yes()) + .arg( + Arg::new("json") + .long("json") + .action(ArgAction::SetTrue) + .help("Print the typed response as JSON"), + ) + }; + let action = |name, about| { + base(name, about) + .arg(Arg::new("request_id").long("request-id") + .help("Retry an original UUIDv7 request; DATABASE must be its recorded Identity")) + .after_help("Acceptance records the desired action; physical stop and readiness are asynchronous. A timeout must be retried with the original Identity and request ID printed on stderr.") + }; + command + .subcommand(base( + "status", + "Inspect container control state without opening its database", + )) + .subcommand(action("start", "Request container execution")) + .subcommand(action("stop", "Request container stop")) + .subcommand(action( + "restart", + "Request a new container instance and environment snapshot", + )) +} + +pub(super) async fn exec(config: &mut crate::Config, name: &str, args: &ArgMatches) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let database = args.get_one::("database").context("database is required")?; + validate_database(database)?; + let server = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; + let supplied_request = args + .try_get_one::("request_id") + .ok() + .flatten() + .map(|id| Uuid::parse_str(id).context("request-id must be a UUIDv7")) + .transpose()?; + if supplied_request.is_some() { + ensure!( + database.parse::().is_ok(), + "retry with the original database Identity printed by the first command" + ); + } + let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; + let client = ContainerClient::new( + server, + auth.to_header().context("container operations require a login")?, + )?; + if name == "status" { + let status = client.status(database).await?; + if args.get_flag("json") { + println!("{}", serde_json::to_string(&status)?); + } else { + print_status(&status); + } + return Ok(()); + } + let action = match name { + "start" => ContainerAction::Start, + "stop" => ContainerAction::Stop, + "restart" => ContainerAction::Restart, + _ => bail!("unsupported container operation"), + }; + // Names are resolved once before mutation. The replay target is immutable. + let identity = match database.parse::() { + Ok(identity) => identity, + Err(_) => client.status(database).await?.database_identity, + }; + let request_id = supplied_request.unwrap_or_else(|| Uuid::from_u128(uuid::Uuid::now_v7().as_u128())); + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), + )?; + spacetimedb_lib::deployment::operation_expiry_ms(request_id, now) + .context("request ID is not within its seven-day retry window")?; + // Structured retry parameters preserve the selected origin without + // presenting unescaped server text as an executable shell command. + let retry = serde_json::json!({ + "command": format!("container {}", action.path()), + "database": identity.to_hex().to_string(), + "request_id": request_id.to_string(), + "server": client.server.as_str(), + }); + eprintln!("Container request retry parameters: {retry}"); + let receipt = client.operate(identity, request_id, action).await.with_context(|| { + format!("request did not return a verified receipt; retain these exact retry parameters: {retry}") + })?; + if args.get_flag("json") { + println!("{}", serde_json::to_string(&receipt)?); + } else { + println!( + "Accepted {} for {} at generation {}", + action.path(), + identity.to_hex(), + receipt.generation + ); + } + Ok(()) +} + +struct ContainerClient { + http: Client, + server: Url, + authorization: HeaderValue, +} +impl ContainerClient { + fn new(server: Url, mut authorization: HeaderValue) -> Result { + ensure!( + authorization.as_bytes().starts_with(b"Bearer "), + "container operations require ordinary Bearer authentication" + ); + authorization.set_sensitive(true); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { + http, + server, + authorization, + }) + } + fn url(&self, database: &str, operation: &str) -> Result { + validate_database(database)?; + let mut url = self.server.clone(); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid server origin"))? + .pop_if_empty() + .extend(["v1", "database", database, "container", operation]); + Ok(url) + } + async fn status(&self, database: &str) -> Result { + let response = self + .http + .get(self.url(database, "status")?) + .header(AUTHORIZATION, self.authorization.clone()) + .send() + .await + .map_err(|_| anyhow::anyhow!("container status could not be reached on the selected server"))?; + let status: ContainerStatus = read_response(response, StatusCode::OK).await?; + if let Ok(identity) = database.parse::() { + ensure!( + status.database_identity == identity, + "status returned another database Identity" + ); + } + validate_status(&status)?; + Ok(status) + } + async fn operate( + &self, + identity: Identity, + request_id: Uuid, + action: ContainerAction, + ) -> Result { + let response = self + .http + .request(Method::POST, self.url(identity.to_hex().as_ref(), action.path())?) + .header(AUTHORIZATION, self.authorization.clone()) + .json(&ContainerOperationRequest { request_id }) + .send() + .await + .map_err(|_| anyhow::anyhow!("container operation response was not received"))?; + let receipt: ContainerOperationReceipt = read_response(response, StatusCode::ACCEPTED).await?; + ensure!( + receipt.database_identity == identity && receipt.request_id == request_id && receipt.action == action, + "container operation receipt does not match this request" + ); + Ok(receipt) + } +} +fn validate_database(database: &str) -> Result<()> { + ensure!( + !database.is_empty() + && database.len() <= 1024 + && !matches!(database, "." | "..") + && !database.chars().any(char::is_control), + "invalid database name or Identity" + ); + Ok(()) +} +async fn read_response(mut response: reqwest::Response, expected: StatusCode) -> Result { + let status = response.status(); + ensure!( + response.content_length().is_none_or(|size| size <= MAX_RESPONSE as u64), + "container response exceeds its size limit" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_RESPONSE.saturating_sub(bytes.len()), + "container response exceeds its size limit" + ); + bytes.extend_from_slice(&chunk); + } + if status != expected { + let error = serde_json::from_slice::(&bytes) + .ok() + .map(|value| value.error); + bail!( + "{} (HTTP {})", + match error { + Some(ContainerErrorCode::AccessDenied) => "database role does not permit this operation", + Some(ContainerErrorCode::NotFound) => "database was not found", + Some(ContainerErrorCode::InvalidRequest) => "container request is invalid or expired", + Some(ContainerErrorCode::Conflict) => "container request conflicts with current state", + Some(ContainerErrorCode::OutcomeUnknown) => + "container request outcome is unknown; retry the same request ID", + _ => "container service is unavailable on the selected server", + }, + status.as_u16() + ); + } + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid container response")) +} +fn validate_status(status: &ContainerStatus) -> Result<()> { + if let Some(state) = &status.operational + && let Some(instance) = &state.current_instance + { + ensure!( + instance.generation == state.generation, + "status instance is not the current generation" + ); + if let Some(id) = &instance.applied_env_generation { + ensure!(Uuid::parse_str(id).is_ok(), "invalid applied environment generation"); + } + } + if let EndpointStatus::Available { endpoints } = &status.endpoints { + super::url::validate(&ContainerEndpoints { + database_identity: status.database_identity, + endpoints: endpoints.clone(), + })?; + } + Ok(()) +} +fn print_status(status: &ContainerStatus) { + println!("Database: {}", status.database_identity.to_hex()); + if !status.published { + println!("Container: none published"); + } + if let Some(state) = &status.operational { + println!("Desired: {:?} (generation {})", state.desired_state, state.generation); + println!("Condition: {:?}", state.condition); + if let Some(instance) = &state.current_instance { + println!("Observed: {:?}", instance.state); + if let Some(environment) = &instance.applied_env_generation { + println!("Environment generation: {environment}"); + } + if let Some(code) = instance.exit_code { + println!("Exit code: {code}"); + } + if instance.oom_killed { + println!("Out of memory: yes"); + } + } else { + println!("Observed: no instance admitted for this generation"); + } + } + match &status.endpoints { + EndpointStatus::Available { endpoints } => { + for endpoint in endpoints { + println!("{}: {}", endpoint.name, endpoint.url); + } + } + EndpointStatus::Pending => println!("Endpoints: pending"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn target_path_encodes_one_database_and_rejects_dot_segments() { + let client = ContainerClient::new( + Url::parse("http://127.0.0.1:43123/").unwrap(), + HeaderValue::from_static("Bearer unusable"), + ) + .unwrap(); + assert_eq!( + client.url("name/child?x#y", "status").unwrap().path(), + "/v1/database/name%2Fchild%3Fx%23y/container/status" + ); + for invalid in ["", ".", "..", "name\n"] { + assert!(client.url(invalid, "status").is_err()); + } + } +} diff --git a/crates/cli/src/subcommands/container/url.rs b/crates/cli/src/subcommands/container/url.rs new file mode 100644 index 00000000000..b66eee410aa --- /dev/null +++ b/crates/cli/src/subcommands/container/url.rs @@ -0,0 +1,172 @@ +//! Anonymous discovery prints one validated public address, never a constructed +//! hostname, authentication token or private runtime address. + +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgMatches, Command}; +use reqwest::{redirect::Policy, Client, StatusCode}; +use spacetimedb_lib::{ + container::{endpoints::ContainerEndpoints, MAX_PORTS}, + Identity, +}; +use std::{collections::BTreeSet, time::Duration}; +use url::Url; + +const MAX_RESPONSE_BYTES: usize = 64 * 1024; + +pub(super) fn cli() -> Command { + Command::new("url") + .about("Print a container's published HTTPS URL") + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg( + Arg::new("port") + .long("port") + .help("Declared port name; required when several ports are published"), + ) + .arg(crate::common_args::server()) + .after_help("Discovery does not start the container or wait for readiness. No login is required.") +} + +pub(super) async fn exec(config: &crate::Config, args: &ArgMatches) -> Result<()> { + let server = args.get_one::("server").map(String::as_str); + let database = args.get_one::("database").context("database is required")?; + let port = args.get_one::("port").map(String::as_str); + let endpoints = fetch(&config.get_host_url(server)?, database).await?; + println!("{}", select(&endpoints, port)?); + Ok(()) +} + +async fn fetch(server: &str, database: &str) -> Result { + ensure!( + !database.is_empty() + && !matches!(database, "." | "..") + && database.len() <= 1024 + && !database.chars().any(char::is_control), + "invalid database name or Identity" + ); + let mut url = Url::parse(server).context("invalid server URL")?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && url.path() == "/", + "server must be an HTTP or HTTPS origin without credentials or a path" + ); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid server URL"))? + .clear() + .extend(["v1", "database", database, "container", "endpoints"]); + // This endpoint is public. Do not load a login, resolve an Identity through + // another request, follow redirects, or forward saved credentials. + let client = Client::builder() + .no_proxy() + .redirect(Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(15)) + .build()?; + let mut response = client + .get(url) + .send() + .await + .map_err(|_| anyhow::anyhow!("could not reach container endpoint discovery on the selected server"))?; + let status = response.status(); + if status == StatusCode::NOT_FOUND { + bail!("database or container endpoint discovery was not found on the selected server"); + } + if status == StatusCode::SERVICE_UNAVAILABLE { + bail!("container endpoints are not available yet; retry shortly"); + } + if status != StatusCode::OK { + bail!("container endpoint discovery failed (HTTP {})", status.as_u16()); + } + ensure!( + response + .content_length() + .is_none_or(|length| length <= MAX_RESPONSE_BYTES as u64), + "container endpoint discovery response exceeds its size limit" + ); + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container endpoint discovery response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_RESPONSE_BYTES.saturating_sub(body.len()), + "container endpoint discovery response exceeds its size limit" + ); + body.extend_from_slice(&chunk); + } + let endpoints: ContainerEndpoints = + serde_json::from_slice(&body).map_err(|_| anyhow::anyhow!("invalid container endpoint discovery response"))?; + if let Ok(identity) = database.parse::() { + ensure!( + identity == endpoints.database_identity, + "container endpoint discovery returned another database Identity" + ); + } + validate(&endpoints)?; + Ok(endpoints) +} + +pub(super) fn validate(endpoints: &ContainerEndpoints) -> Result<()> { + ensure!( + endpoints.endpoints.len() <= MAX_PORTS, + "too many container endpoints in discovery response" + ); + let mut names = BTreeSet::new(); + for endpoint in &endpoints.endpoints { + let name = endpoint.name.as_bytes(); + ensure!( + (1..=32).contains(&name.len()) + && name[0].is_ascii_lowercase() + && name + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') + && names.insert(&endpoint.name), + "invalid or duplicate port name in discovery response" + ); + let address = Url::parse(&endpoint.url).context("invalid container endpoint URL")?; + ensure!( + endpoint.url.len() <= 2048 + && endpoint.url.is_ascii() + && !endpoint.url.bytes().any(|byte| byte.is_ascii_control()) + && address.as_str() == endpoint.url + && address.scheme() == "https" + && matches!(address.host(), Some(url::Host::Domain(_))) + && address.username().is_empty() + && address.password().is_none() + && address.port().is_none() + && address.query().is_none() + && address.fragment().is_none() + && address.path() == "/", + "discovery returned an invalid public HTTPS endpoint" + ); + } + Ok(()) +} + +fn select<'a>(endpoints: &'a ContainerEndpoints, port: Option<&str>) -> Result<&'a str> { + match (port, endpoints.endpoints.as_slice()) { + (_, []) => bail!("this database has no declared public HTTP ports"), + (None, [endpoint]) => Ok(&endpoint.url), + (Some(port), entries) => entries + .iter() + .find(|endpoint| endpoint.name == port) + .map(|endpoint| endpoint.url.as_str()) + .context("the requested port name is not published by this database"), + (None, entries) => bail!( + "several ports are published; select one with --port: {}", + entries + .iter() + .map(|entry| entry.name.as_str()) + .collect::>() + .join(", ") + ), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/container/url/tests.rs b/crates/cli/src/subcommands/container/url/tests.rs new file mode 100644 index 00000000000..37a544d8b75 --- /dev/null +++ b/crates/cli/src/subcommands/container/url/tests.rs @@ -0,0 +1,198 @@ +use super::*; +use axum::{ + body::{Body, Bytes}, + extract::Path, + http::{header, HeaderMap}, + response::{IntoResponse, Response}, + routing::get, + Json, Router, +}; +use spacetimedb_lib::container::{endpoints::ContainerEndpoint, PortProtocol}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn response() -> ContainerEndpoints { + ContainerEndpoints { + database_identity: Identity::ZERO, + endpoints: vec![ContainerEndpoint { + name: "http".into(), + protocol: PortProtocol::Http, + url: "https://aaaqeayeaudaocajbifqydiob4.container.example.net/".into(), + }], + } +} + +struct Server { + origin: String, + stop: Option>, + task: tokio::task::JoinHandle>, +} + +impl Server { + async fn start(router: Router) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let origin = format!("http://{}", listener.local_addr()?); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + Ok(Self { + origin, + stop: Some(stop), + task, + }) + } + + async fn shutdown(mut self) -> Result<()> { + self.stop.take().unwrap().send(()).ok(); + tokio::time::timeout(Duration::from_secs(5), &mut self.task).await???; + Ok(()) + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[tokio::test] +async fn discovery_is_anonymous_and_encodes_the_entire_database_path_segment() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let count = requests.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/endpoints", + get(move |Path(database): Path, headers: HeaderMap| { + let count = count.clone(); + async move { + assert_eq!(database, "project/child?query=value#fragment"); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + count.fetch_add(1, Ordering::SeqCst); + Json(response()) + } + }), + )) + .await?; + for invalid in [".", ".."] { + assert!(fetch(&server.origin, invalid).await.is_err()); + } + assert_eq!(requests.load(Ordering::SeqCst), 0); + let discovered = fetch(&server.origin, "project/child?query=value#fragment").await?; + assert_eq!(select(&discovered, None)?, response().endpoints[0].url); + assert_eq!(requests.load(Ordering::SeqCst), 1); + server.shutdown().await +} + +#[test] +fn port_selection_never_picks_an_arbitrary_endpoint() -> Result<()> { + let mut endpoints = response(); + assert_eq!(select(&endpoints, Some("http"))?, endpoints.endpoints[0].url); + assert!(select(&endpoints, Some("absent")).is_err()); + let mut second = endpoints.endpoints[0].clone(); + second.name = "metrics".into(); + second.url = "https://bbbbbbbbbbbbbbbbbbbbbbbbaa.container.example.net/".into(); + endpoints.endpoints.push(second); + assert!(select(&endpoints, None) + .unwrap_err() + .to_string() + .contains("--port: http, metrics")); + assert_eq!(select(&endpoints, Some("metrics"))?, endpoints.endpoints[1].url); + endpoints.endpoints.clear(); + assert!(select(&endpoints, None) + .unwrap_err() + .to_string() + .contains("no declared public HTTP ports")); + Ok(()) +} + +#[test] +fn endpoint_output_rejects_terminal_controls_credentials_and_ambiguous_names() { + let mut endpoints = response(); + for address in [ + "javascript:alert(1)", + "http://application.example/", + "https://user:secret@application.example/", + "https://127.0.0.1/", + "https://application.example/path", + "https://application.example/?secret=value", + "https://application.example/#fragment", + "https://application.example:8443/", + "https://application.example/\n", + "\u{1b}[2Jhttps://application.example/", + ] { + endpoints.endpoints[0].url = address.into(); + assert!(validate(&endpoints).is_err()); + } + endpoints = response(); + for name in ["", "HTTP", "1http", "http\n", "http/metrics"] { + endpoints.endpoints[0].name = name.into(); + assert!(validate(&endpoints).is_err()); + } + endpoints = response(); + endpoints.endpoints.push(endpoints.endpoints[0].clone()); + assert!(validate(&endpoints).is_err()); +} + +#[tokio::test] +async fn discovery_bounds_streams_rejects_redirects_and_preserves_pending_errors() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let count = requests.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/endpoints", + get(move |Path(database): Path| { + let count = count.clone(); + async move { + count.fetch_add(1, Ordering::SeqCst); + match database.as_str() { + "pending" => (StatusCode::SERVICE_UNAVAILABLE, "private diagnostic").into_response(), + "missing" => StatusCode::NOT_FOUND.into_response(), + "redirect" => (StatusCode::FOUND, [(header::LOCATION, "/unexpected")]).into_response(), + "invalid" => Json(serde_json::json!({"private diagnostic": "do not print"})).into_response(), + "oversize" => Response::new(Body::from_stream(futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from(vec![b' '; MAX_RESPONSE_BYTES])), + Ok(Bytes::from_static(b"x")), + ]))), + _ => Json(response()).into_response(), + } + } + }), + )) + .await?; + for (database, expected) in [ + ("pending", "not available yet"), + ("missing", "not found"), + ("redirect", "HTTP 302"), + ("invalid", "invalid container endpoint discovery response"), + ("oversize", "size limit"), + ] { + let error = format!("{:#}", fetch(&server.origin, database).await.unwrap_err()); + assert!(error.contains(expected), "{error}"); + assert!(!error.contains("private diagnostic")); + } + let other_identity = Identity::from_u256(1_u64.into()).to_string(); + assert!(fetch(&server.origin, &other_identity) + .await + .unwrap_err() + .to_string() + .contains("another database Identity")); + assert_eq!(requests.load(Ordering::SeqCst), 6); + server.shutdown().await +} + +#[test] +fn url_command_requires_database_and_parses_explicit_server_and_port() { + assert!(cli().try_get_matches_from(["url"]).is_err()); + let args = cli() + .try_get_matches_from(["url", "demo", "--server", "http://127.0.0.1:3000", "--port", "http"]) + .unwrap(); + assert_eq!(args.get_one::("database").unwrap(), "demo"); + assert_eq!(args.get_one::("server").unwrap(), "http://127.0.0.1:3000"); + assert_eq!(args.get_one::("port").unwrap(), "http"); +} diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs index 95cc579182f..1e91ac43c22 100644 --- a/crates/cli/src/subcommands/dev.rs +++ b/crates/cli/src/subcommands/dev.rs @@ -958,6 +958,10 @@ fn determine_publish_configs<'a>( } if !publish_configs.is_empty() { + anyhow::ensure!( + publish_configs.iter().all(|target| target.container().is_none()), + "spacetime dev does not yet run container targets; use spacetime publish for managed publication" + ); return Ok(publish_configs); } diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index af0d2e364c3..cefe1a8e4c7 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -1,5 +1,6 @@ pub mod build; pub mod call; +pub mod container; pub mod db_arg_resolution; pub mod delete; pub mod describe; diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 745664880f0..e6c411ad312 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -20,6 +20,8 @@ use crate::util::{add_auth_header_opt, get_auth_header, strip_verbatim_prefix, A use crate::util::{decode_identity, y_or_n}; use crate::{build, common_args}; +mod managed; + /// Individual prompts that `--yes` can suppress. `All` is a shorthand for every category below. #[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] #[clap(rename_all = "kebab-case")] @@ -84,7 +86,7 @@ fn yes_flags_from_args(args: &ArgMatches) -> YesFlags { /// Build the CommandSchema for publish command pub fn build_publish_schema(command: &clap::Command) -> Result { - CommandSchemaBuilder::new() + managed::exclude_args(CommandSchemaBuilder::new()) .key(Key::new("database").from_clap("name|identity").required()) .key(Key::new("server")) .key(Key::new("module_path").module_specific()) @@ -173,7 +175,7 @@ pub fn get_filtered_publish_configs<'a>( let configs: Vec = filtered_targets .into_iter() .map(|target| { - let config = CommandConfig::new(schema, target.fields, args)?; + let config = CommandConfig::new(schema, target.fields, args)?.with_container(target.container); config.validate()?; Ok(config) }) @@ -191,7 +193,7 @@ pub fn get_filtered_publish_configs<'a>( } pub fn cli() -> clap::Command { - clap::Command::new("publish") + managed::add_args(clap::Command::new("publish") .about("Create and update a SpacetimeDB database") .arg(common_args::clear_database()) .arg( @@ -319,7 +321,7 @@ i.e. only lowercase ASCII letters and numbers, separated by dashes."), .help("Use NativeAOT-LLVM compilation for C# modules (experimental, Windows only)") ) .arg(common_args::dotnet_version()) - .after_help("Run `spacetime help publish` for more detailed information.") + .after_help("Run `spacetime help publish` for more detailed information.")) } fn confirm_and_clear( @@ -381,6 +383,9 @@ pub async fn exec_with_options( quiet_config: bool, pre_loaded_config: Option<&LoadedConfig>, ) -> Result<(), anyhow::Error> { + if args.get_one::("resume_publication").is_some() { + return managed::resume(&mut config, args, yes_flags_from_args(args)).await; + } // Build schema let cmd = cli(); let schema = build_publish_schema(&cmd)?; @@ -468,6 +473,15 @@ pub async fn exec_from_entry( execute_publish_configs(&mut config, vec![command_config], true, config_dir, clear_database, yes).await } +fn dotnet_version_from_config(command_config: &CommandConfig<'_>) -> anyhow::Result> { + if command_config.is_from_cli("dotnet_version") { + Ok(command_config.get_one::("dotnet_version")?) + } else { + let dotnet_version = command_config.get_one::("dotnet_version")?; + parse_optional_dotnet_version(dotnet_version.as_deref()) + } +} + async fn execute_publish_configs<'a>( config: &mut Config, publish_configs: Vec>, @@ -496,20 +510,7 @@ async fn execute_publish_configs<'a>( }) }; - if using_config { - if let Some(path_to_project) = path_to_project.as_ref() { - println!( - "Publishing module {} to database '{}'", - strip_verbatim_prefix(path_to_project).display(), - name_or_identity.unwrap() - ); - } else { - println!( - "Publishing precompiled module to database '{}'", - name_or_identity.unwrap() - ); - } - } + managed::validate_target_options(&command_config)?; let database_host = config.get_host_url(server)?; let build_options = command_config .get_one::("build_options")? @@ -521,12 +522,7 @@ async fn execute_publish_configs<'a>( let org_opt = command_config.get_one::("organization")?; let org = org_opt.as_deref(); let native_aot = command_config.get_one::("native_aot")?.unwrap_or(false); - let dotnet_version = if command_config.is_from_cli("dotnet_version") { - command_config.get_one::("dotnet_version")? - } else { - let dotnet_version = command_config.get_one::("dotnet_version")?; - parse_optional_dotnet_version(dotnet_version.as_deref())? - }; + let dotnet_version = dotnet_version_from_config(&command_config)?; // If the user didn't specify an identity and we didn't specify an anonymous identity, then // we want to use the default identity @@ -536,6 +532,36 @@ async fn execute_publish_configs<'a>( let (name_or_identity, parent) = validate_name_and_parent(name_or_identity, parent)?; + if managed::try_execute( + &command_config, + config_dir, + &database_host, + &auth_header, + name_or_identity, + parent, + clear_database, + yes, + ) + .await? + { + continue; + } + + if using_config { + if let Some(path_to_project) = path_to_project.as_ref() { + println!( + "Publishing module {} to database '{}'", + strip_verbatim_prefix(path_to_project).display(), + name_or_identity.unwrap() + ); + } else { + println!( + "Publishing precompiled module to database '{}'", + name_or_identity.unwrap() + ); + } + } + if let Some(path_to_project) = path_to_project.as_ref() && !path_to_project.exists() { @@ -1284,6 +1310,40 @@ mod tests { assert_eq!(matches.get_one::("dotnet_version").copied(), Some(10)); } + #[test] + fn managed_publish_preserves_dotnet_selection_and_selective_yes() { + let command = cli(); + let schema = build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "--managed", + "--dotnet-version", + "10", + "--yes=remote,skip-login", + "--server", + "http://127.0.0.1:1", + "test-db", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + target.validate().unwrap(); + assert!(args.get_flag("managed")); + assert_eq!(dotnet_version_from_config(&target).unwrap(), Some(10)); + let yes = yes_flags_from_args(&args); + assert!(yes.publish_to_remote && yes.skip_login); + assert!(!yes.migrate_major_version && !yes.break_clients && !yes.delete_data); + + let args = cli().get_matches_from(["publish", "--managed", "test-db"]); + let target = CommandConfig::new( + &schema, + HashMap::from([("dotnet_version".into(), serde_json::json!("10"))]), + &args, + ) + .unwrap(); + assert_eq!(dotnet_version_from_config(&target).unwrap(), Some(10)); + } + #[test] fn test_publish_cli_rejects_unsupported_dotnet_version() { assert!(cli() diff --git a/crates/cli/src/subcommands/publish/managed.rs b/crates/cli/src/subcommands/publish/managed.rs new file mode 100644 index 00000000000..994c5541080 --- /dev/null +++ b/crates/cli/src/subcommands/publish/managed.rs @@ -0,0 +1,975 @@ +//! Managed publication frontend. The resume journal owns every byte needed to +//! retry; project configuration and mutable image tags are only read initially. +use super::{confirm_major_version_upgrade, YesFlags}; +use crate::{ + common_args::ClearMode, + config::Config, + container::{ + self, + oci::ArtifactKind, + process::LocalRunner, + publish::{ + self, + client::{ObjectRef, PublisherClient, UploadKind}, + journal::{Journal, Record, UploadRecord}, + Outcome, + }, + BuildSecret, BuildTools, + }, + spacetime_config::{CommandConfig, CommandSchemaBuilder}, + util::{get_auth_header, y_or_n, AuthHeader}, +}; +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_client_api_messages::name::{is_identity, DomainName, PrePublishResult}; +use spacetimedb_lib::{ + container::{ContainerAction, ImagePlatform}, + deployment::{self, api::*, manifest::*, ModuleAction, PublishEnvelope, UserModule, UserModuleKind}, + Identity, Uuid, +}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; +use tokio_util::sync::CancellationToken; + +const ARGUMENTS: &[&str] = &[ + "managed", + "container_platform", + "artifact_endpoint", + "remove_container", + "remove_module", + "publication_state_dir", + "resume_publication", + "publication_wait", + "buildkit_host", + "buildctl", + "railpack", + "skopeo", + "registry_auth_file", + "build_secret", +]; +pub(super) fn exclude_args(mut schema: CommandSchemaBuilder) -> CommandSchemaBuilder { + for arg in ARGUMENTS { + schema = schema.exclude(*arg); + } + schema +} +pub(super) fn add_args(mut command: Command) -> Command { + command = command + .arg( + Arg::new("managed") + .long("managed") + .action(ArgAction::SetTrue) + .help("Use managed deployment publication, including for a module-only database"), + ) + .arg( + Arg::new("container_platform") + .long("container-platform") + .value_parser(["linux/amd64", "linux/arm64"]) + .help("Required platform when publishing a container declaration"), + ) + .arg( + Arg::new("artifact_endpoint") + .long("artifact-endpoint") + .help("Explicitly authorize this exact artifact URL to receive the publisher credential"), + ) + .arg( + Arg::new("remove_container") + .long("remove-container") + .action(ArgAction::SetTrue) + .help("Explicitly remove the container while preserving the module unless separately changed"), + ) + .arg( + Arg::new("remove_module") + .long("remove-module") + .action(ArgAction::SetTrue) + .conflicts_with_all(["module_path", "wasm_file", "js_file"]) + .help("Replace the module with the versioned empty module after migration preflight"), + ) + .arg( + Arg::new("publication_state_dir") + .long("publication-state-dir") + .value_parser(clap::value_parser!(PathBuf)) + .help("Private local directory retaining managed publication bytes and progress"), + ) + .arg( + Arg::new("resume_publication") + .long("resume-publication") + .value_parser(clap::value_parser!(PathBuf)) + .conflicts_with_all([ + "managed", + "remove_container", + "remove_module", + "module_path", + "wasm_file", + "js_file", + "container_platform", + "name|identity", + "parent", + "organization", + "clear-database", + ]) + .help("Resume this operation directory without rebuilding or reading spacetime.json"), + ) + .arg( + Arg::new("publication_wait") + .long("publication-wait") + .default_value("60") + .value_parser(clap::value_parser!(u64).range(0..=3600)) + .help("Seconds to wait for managed activation; pending operations retain their resume directory"), + ) + .arg( + Arg::new("buildkit_host") + .long("buildkit-host") + .help("Explicit local BuildKit Unix socket for container source builds"), + ) + .arg( + Arg::new("registry_auth_file") + .long("registry-auth-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Explicit registry auth JSON; otherwise image preparation is anonymous"), + ) + .arg( + Arg::new("build_secret") + .long("build-secret") + .action(ArgAction::Append) + .value_name("NAME=FILE") + .help("Build secret file, separate from runtime env_keys"), + ); + for (tool, help) in [ + ( + "buildctl", + "Path to the buildctl executable for Dockerfile or Railpack builds", + ), + ( + "railpack", + "Path to the Railpack executable for explicitly selected Railpack builds", + ), + ( + "skopeo", + "Path to the skopeo executable for copying prebuilt OCI images", + ), + ] { + command = command.arg( + Arg::new(tool) + .help(help) + .long(tool) + .default_value(tool) + .value_parser(clap::value_parser!(PathBuf)), + ); + } + command +} +fn approved(args: &ArgMatches) -> Option<&str> { + args.get_one::("artifact_endpoint").map(String::as_str) +} +fn wait(args: &ArgMatches) -> Duration { + Duration::from_secs(*args.get_one::("publication_wait").unwrap_or(&60)) +} +pub(super) fn validate_target_options(target: &CommandConfig<'_>) -> Result<()> { + let args = target.matches(); + if target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module") + { + ensure!( + !target.get_one::("anon_identity")?.unwrap_or(false), + "managed publication requires an authenticated publisher" + ); + } + Ok(()) +} +fn authenticated(server: &str, auth: &AuthHeader) -> Result { + PublisherClient::new( + server, + auth.to_header() + .context("managed publication requires an authenticated publisher; anonymous publication is unsupported")?, + ) +} +pub(super) async fn resume(config: &mut Config, args: &ArgMatches, yes: YesFlags) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let server = config.get_host_url(selection)?; + let mut journal = Journal::open(args.get_one::("resume_publication").unwrap())?; + ensure!( + container::publish::client::endpoint(&server)?.as_str() == journal.record.server, + "resume server differs from the original publication endpoint" + ); + let auth = get_auth_header(config, false, selection, !yes.skip_login).await?; + let client = authenticated(&server, &auth)?; + execute(&client, &mut journal, args).await +} + +/// False is only returned for an ordinary, un-managed module publication. +/// Once managed intent or a managed revision is known, errors cannot fall back. +#[allow(clippy::too_many_arguments)] +pub(super) async fn try_execute( + target: &CommandConfig<'_>, + config_dir: Option<&Path>, + server: &str, + auth: &AuthHeader, + name: Option<&str>, + parent: Option<&str>, + clear: ClearMode, + yes: YesFlags, +) -> Result { + let args = target.matches(); + let explicit = target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module"); + if !explicit && name.is_none() { + return Ok(false); + } + let client = match authenticated(server, auth) { + Ok(client) => client, + Err(error) if explicit => return Err(error), + Err(_) => return Ok(false), + }; + try_execute_with_client(target, config_dir, client, name, parent, clear, yes).await +} + +async fn try_execute_with_client( + target: &CommandConfig<'_>, + config_dir: Option<&Path>, + client: PublisherClient, + name: Option<&str>, + parent: Option<&str>, + clear: ClearMode, + yes: YesFlags, +) -> Result { + let args = target.matches(); + let explicit = target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module"); + let caps = client.capabilities().await?; + let Some(caps) = caps else { + ensure!( + !explicit, + "this server does not support managed publication; no module-only fallback was attempted" + ); + return Ok(false); + }; + let prior = if let Some(name) = name { + client.deployment(name).await? + } else { + None + }; + if !explicit && prior.as_ref().and_then(|p| p.revision).is_none() { + return Ok(false); + } + ensure!( + caps.enabled && caps.version == deployment::PUBLISH_PROTOCOL_VERSION, + "managed publication is disabled or incompatible on this server" + ); + ensure!( + clear == ClearMode::Never, + "managed publication does not support --delete-data; resolve migrations without replacing database storage" + ); + ensure!( + !(target.container().is_some() && args.get_flag("remove_container")), + "a container declaration and --remove-container cannot both select this target" + ); + if prior.is_none() { + ensure!( + !name.is_some_and(is_identity), + "a new database Identity must be generated by the reservation service; select a name or omit DATABASE" + ); + } + let artifact = client.artifact_endpoint( + caps.artifact_endpoint + .as_deref() + .context("server did not advertise an artifact endpoint")?, + approved(args), + )?; + let permission = client.permission().await?; + if target.container().is_some() || prior.is_none() { + ensure!( + permission.can_publish, + "the current publisher does not have container publication permission" + ); + } + if !container::publish::client::is_loopback(client.server()) { + ensure!( + y_or_n( + yes.publish_to_remote, + "Publish this managed deployment to the selected remote server?" + )?, + "publication cancelled" + ); + } + let cwd = std::env::current_dir()?; + let base = args + .get_one::("publication_state_dir") + .cloned() + .unwrap_or_else(|| config_dir.unwrap_or(&cwd).join(".spacetime/publications")); + std::fs::create_dir_all(&base)?; + let cancel = CancellationToken::new(); + let prepare = prepare_request( + &client, + target, + config_dir.unwrap_or(&cwd), + &base, + prior.as_ref(), + name, + parent, + permission.identity, + artifact.as_str(), + yes, + cancel.clone(), + ); + tokio::pin!(prepare); + let mut journal = tokio::select! { + result = &mut prepare => result?, + signal = tokio::signal::ctrl_c() => { + signal?; cancel.cancel(); let _ = prepare.await; + bail!("managed preparation cancelled before admission"); + } + }; + execute(&client, &mut journal, args).await?; + Ok(true) +} + +#[allow(clippy::too_many_arguments)] +async fn prepare_request( + client: &PublisherClient, + target: &CommandConfig<'_>, + config_dir: &Path, + base: &Path, + prior: Option<&DeploymentStatus>, + name: Option<&str>, + parent: Option<&str>, + publisher: Identity, + artifact_endpoint: &str, + yes: YesFlags, + cancel: CancellationToken, +) -> Result { + let args = target.matches(); + let image = if let Some(declaration) = target.container() { + let (os, architecture) = args + .get_one::("container_platform") + .context("publishing a container requires --container-platform linux/amd64 or linux/arm64")? + .split_once('/') + .unwrap(); + Some( + container::prepare_container( + declaration, + config_dir, + ImagePlatform { + os: os.into(), + architecture: architecture.into(), + }, + &tools(args)?, + base, + &LocalRunner, + cancel.clone(), + ) + .await?, + ) + } else { + None + }; + let has_module = ["module_path", "wasm_file", "js_file"] + .iter() + .any(|key| target.is_from_cli(key) || target.get_config_value(key).is_some()); + ensure!( + !(has_module && args.get_flag("remove_module")), + "module configuration and --remove-module cannot both select this target" + ); + let module = if !args.get_flag("remove_module") + && (has_module || (target.container().is_none() && !args.get_flag("remove_container"))) + { + Some(load_module(target, config_dir).await?) + } else { + None + }; + let module_action = if args.get_flag("remove_module") { + ModuleAction::Remove + } else if let Some((kind, bytes)) = &module { + ModuleAction::Set(UserModule { + kind: *kind, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }) + } else { + ModuleAction::Keep + }; + let container_action = if let Some(image) = &image { + ContainerAction::Set(image.metadata.container.clone()) + } else if args.get_flag("remove_container") { + ContainerAction::Remove + } else { + ContainerAction::Keep + }; + let envelope = PublishEnvelope { + version: deployment::PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), + expected_revision: prior.and_then(|p| p.revision), + module_action, + container_action, + }; + let deployment = envelope.resolve(prior.map(|p| &p.deployment), &Default::default())?; + let module_artifact = if let Some((_, bytes)) = &module { + ModuleArtifact { + digest: spacetimedb_oci::sha256(bytes), + size_bytes: bytes.len() as u64, + } + } else if let Some(prior) = prior.filter(|_| !matches!(envelope.module_action, ModuleAction::Remove)) { + let artifact = prior.module_artifact; + ModuleArtifact { + digest: artifact.digest, + size_bytes: artifact.size_bytes, + } + } else { + deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT + }; + let migration_policy = if let Some(prior) = prior { + if let Some((kind, bytes)) = &module { + migration(client, prior.database_identity, *kind, bytes, target, yes).await? + } else if matches!(envelope.module_action, ModuleAction::Remove) { + migration( + client, + prior.database_identity, + UserModuleKind::Wasm, + deployment::SYSTEM_EMPTY_MODULE_V1_BYTES, + target, + yes, + ) + .await? + } else { + PreparedMigrationPolicy::Compatible + } + } else { + PreparedMigrationPolicy::Compatible + }; + let creation = if prior.is_none() { + let parent = if let Some(parent) = parent { + Some( + client + .deployment(parent) + .await? + .context("parent database does not exist or is not accessible")? + .database_identity, + ) + } else { + None + }; + let organization = target + .get_one::("organization")? + .map(|value| { + value + .parse::() + .context("managed publication currently requires --organization IDENTITY, not an organization name") + }) + .transpose()?; + Some(CreationOptions { + parent, + organization, + num_replicas: target.get_one::("num_replicas")?.map(u32::from), + enforce_anti_affinity: true, + }) + } else { + None + }; + let request = PublishRequest { + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope, + deployment, + module_artifact, + migration_policy, + }), + creation, + image_source: image.as_ref().map(|image| ArtifactReference { + digest: image.metadata.manifest.digest, + size_bytes: image.metadata.manifest.size, + }), + }; + request.manifest.validate(&Default::default())?; + let request_json = serde_json::to_string(&request)?; + let mut uploads = image + .as_ref() + .map(|image| { + image + .metadata + .objects + .iter() + .map(|object| UploadRecord { + kind: match object.kind { + ArtifactKind::Manifest => UploadKind::Manifest, + ArtifactKind::Config => UploadKind::Config, + ArtifactKind::Layer => UploadKind::Layer, + }, + object: ObjectRef { + digest: object.descriptor.digest, + size: object.descriptor.size, + }, + session: None, + }) + .collect::>() + }) + .unwrap_or_default(); + if module.is_some() { + uploads.push(UploadRecord { + kind: UploadKind::Module, + object: ObjectRef { + digest: module_artifact.digest, + size: module_artifact.size_bytes, + }, + session: None, + }); + } + let requested_name = if prior.is_none() { + name.map(|name| name.parse::().map(|name| name.to_string())) + .transpose()? + } else { + None + }; + let record = Record { + version: 1, + server: client.server().to_string(), + artifact_endpoint: artifact_endpoint.into(), + publisher, + database: prior.map(|p| p.database_identity), + reservation: request.creation.clone().map(|options| ReserveDatabaseRequest { + version: deployment::PUBLISH_PROTOCOL_VERSION, + operation_id: request.manifest.current().envelope.operation_id, + options, + }), + requested_name, + request_digest: spacetimedb_oci::sha256(request_json.as_bytes()), + request_json, + uploads, + submitted: false, + status: None, + name_confirmed: false, + naming_attempted: false, + }; + Journal::create(base, record, image, module.as_ref().map(|(_, bytes)| bytes.as_slice())) +} +fn tools(args: &ArgMatches) -> Result { + let mut tools = BuildTools { + buildctl: args.get_one::("buildctl").unwrap().clone(), + railpack: args.get_one::("railpack").unwrap().clone(), + skopeo: args.get_one::("skopeo").unwrap().clone(), + buildkit_host: args.get_one::("buildkit_host").cloned(), + registry_auth_file: args.get_one::("registry_auth_file").cloned(), + secrets: vec![], + }; + for value in args.get_many::("build_secret").into_iter().flatten() { + let (name, file) = value.split_once('=').context("build secret must be NAME=FILE")?; + ensure!(!file.is_empty(), "build secret file is missing"); + tools.secrets.push(BuildSecret { + name: name.into(), + file: file.into(), + }); + } + Ok(tools) +} +async fn load_module(target: &CommandConfig<'_>, config_dir: &Path) -> Result<(UserModuleKind, Vec)> { + let native_aot = target.get_one::("native_aot")?.unwrap_or(false); + ensure!( + !native_aot, + "managed NativeAOT builds are not yet supported; build separately and pass --bin-path" + ); + let (path, kind) = if let Some(path) = target.get_resolved_path("wasm_file", Some(config_dir))? { + (path, "Wasm") + } else if let Some(path) = target.get_resolved_path("js_file", Some(config_dir))? { + (path, "Js") + } else { + let path = target + .get_resolved_path("module_path", Some(config_dir))? + .unwrap_or_else(|| super::default_publish_module_path(config_dir)); + crate::build::exec_with_argstring( + &path, + &target.get_one::("build_options")?.unwrap_or_default(), + native_aot, + super::dotnet_version_from_config(target)?, + ) + .await? + }; + let kind = match kind { + "Wasm" => UserModuleKind::Wasm, + "Js" => UserModuleKind::Js, + _ => bail!("unsupported managed module kind"), + }; + let metadata = tokio::fs::metadata(&path).await?; + ensure!( + metadata.is_file() && metadata.len() > 0 && metadata.len() <= MAX_MODULE_ARTIFACT_BYTES, + "module must be a regular file of at most 32 MiB" + ); + use tokio::io::AsyncReadExt as _; + let mut bytes = Vec::new(); + tokio::fs::File::open(&path) + .await? + .take(MAX_MODULE_ARTIFACT_BYTES + 1) + .read_to_end(&mut bytes) + .await?; + ensure!( + !bytes.is_empty() && bytes.len() as u64 <= MAX_MODULE_ARTIFACT_BYTES, + "module exceeds 32 MiB" + ); + Ok((kind, bytes)) +} +async fn migration( + client: &PublisherClient, + database: Identity, + kind: UserModuleKind, + bytes: &[u8], + target: &CommandConfig<'_>, + yes: YesFlags, +) -> Result { + let pre = client + .preflight( + database, + match kind { + UserModuleKind::Wasm => "Wasm", + UserModuleKind::Js => "Js", + }, + bytes, + ) + .await?; + match pre { + PrePublishResult::ManualMigrate(_) => { + bail!("managed publication requires manual migration; no storage was cleared") + } + PrePublishResult::AutoMigrate(auto) => { + if auto.major_version_upgrade { + confirm_major_version_upgrade(yes.migrate_major_version)?; + } + println!("{}", auto.migrate_plan); + if auto.break_clients { + ensure!( + y_or_n( + yes.break_clients || target.get_one::("break_clients")?.unwrap_or(false), + "These changes will BREAK existing clients. Proceed?" + )?, + "publication cancelled" + ); + } + Ok(PreparedMigrationPolicy::BreakClients(auto.token)) + } + } +} +async fn execute(client: &PublisherClient, journal: &mut Journal, args: &ArgMatches) -> Result<()> { + println!( + "Publication {}. Resume directory: {}", + journal.operation()?, + journal.directory().display() + ); + let cancel = CancellationToken::new(); + let operation = publish::run(client, journal, approved(args), wait(args), cancel.clone()); + tokio::pin!(operation); + let outcome = tokio::select! { + result = &mut operation => result?, + signal = tokio::signal::ctrl_c() => { signal?; cancel.cancel(); bail!("publication interrupted; resume the same directory to determine its outcome"); } + }; + match outcome { + Outcome::Complete(status) => println!( + "Deployment {} is active on {}", + status.proposed_revision, status.database_identity + ), + Outcome::Pending(status) => println!( + "Publication {} is {:?} on {}; resume to observe activation", + status.operation_id, status.phase, status.database_identity + ), + Outcome::Aborted(status) => bail!( + "publication {} was aborted before commit on {}; resume state retained", + status.operation_id, + status.database_identity + ), + Outcome::NamingUnconfirmed(status) => bail!( + "deployment is active on {}, but naming was not confirmed; publication must not be repeated", + status.database_identity + ), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::container::publish::tests::{database, Fixture}; + use crate::spacetime_config::SpacetimeConfig; + use serde_json::json; + use spacetimedb_paths::FromPathUnchecked as _; + use std::collections::HashMap; + + #[tokio::test] + async fn container_only_frontend_uploads_verified_closure_with_server_reserved_identity() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let layout = temporary.path().join("layout"); + let selected = crate::container::tests::fixture(&layout); + let declaration = crate::container::tests::declaration(json!({"oci_ref":"oci:layout"})); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let state = temporary.path().join("state"); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--container-platform", + "linux/amd64", + "--publication-state-dir", + state.to_str().unwrap(), + "--publication-wait", + "0", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args) + .unwrap() + .with_container(Some(declaration)); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + { + let snapshot = fixture.state.lock().unwrap(); + assert_eq!(snapshot.reservations.len(), 1); + assert_eq!(snapshot.submits.len(), 1); + assert_eq!(snapshot.begin_count, 3); + let request: PublishRequest = serde_json::from_slice(&snapshot.submits[0]).unwrap(); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Keep + )); + assert!(matches!( + request.manifest.current().deployment.current().module, + deployment::ModuleComponent::SystemEmpty(1) + )); + assert_eq!( + request.manifest.current().module_artifact, + deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT + ); + assert_eq!(request.image_source.unwrap().digest, selected.digest); + let dir = state.join(request.manifest.current().envelope.operation_id.to_string()); + let journal = Journal::open(&dir).unwrap(); + assert_eq!(journal.record.database, Some(database())); + let bytes = std::fs::read_to_string(dir.join("publication.json")).unwrap(); + assert!(!bytes.contains("private-image-value")); + } + fixture.close().await; + } + #[tokio::test] + async fn existing_managed_module_update_keeps_container_and_preflights_without_pro() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let wasm = temporary.path().join("module.wasm"); + std::fs::write(&wasm, deployment::SYSTEM_EMPTY_MODULE_V1_BYTES).unwrap(); + let prior_request = fixture.record(false, false).request().unwrap(); + let prior = DeploymentStatus { + database_identity: database(), + revision: Some(prior_request.manifest.current().deployment.revision().unwrap()), + deployment: prior_request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size_bytes: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.size_bytes, + }, + }; + { + let mut state = fixture.state.lock().unwrap(); + state.permission = false; + state.prior = Some(prior.clone()); + } + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--bin-path", + wasm.to_str().unwrap(), + "--publication-state-dir", + temporary.path().join("state").to_str().unwrap(), + "--publication-wait", + "0", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.preflights, 1); + assert_eq!(state.reservations.len(), 0); + let request: PublishRequest = serde_json::from_slice(&state.submits[0]).unwrap(); + assert_eq!(request.manifest.current().envelope.expected_revision, prior.revision); + assert!(matches!( + request.manifest.current().envelope.container_action, + ContainerAction::Keep + )); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Set(_) + )); + } + fixture.close().await; + } + #[tokio::test] + async fn remove_module_requires_authorized_preflight_before_any_upload_or_submission() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let request = fixture.record(false, false).request().unwrap(); + { + let mut state = fixture.state.lock().unwrap(); + state.deny_preflight = true; + state.prior = Some(DeploymentStatus { + database_identity: database(), + revision: Some(request.manifest.current().deployment.revision().unwrap()), + deployment: request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size_bytes: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.size_bytes, + }, + }); + } + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--remove-module", + "--publication-state-dir", + temporary.path().to_str().unwrap(), + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .is_err()); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.preflights, 1); + assert_eq!(state.begin_count, 0); + assert!(state.submits.is_empty()); + } + fixture.close().await; + } + #[tokio::test] + async fn ordinary_module_target_returns_to_legacy_only_when_no_managed_revision_exists() { + let fixture = Fixture::new().await; + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from(["publish", "fixture-name", "--server", fixture.endpoint.as_str()]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(!try_execute_with_client( + &target, + None, + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + fixture.close().await; + } + #[tokio::test] + async fn resume_entrypoint_ignores_changed_project_and_reuses_original_bytes() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_before_commit = true; + let temporary = tempfile::tempdir().unwrap(); + let record = fixture.record(false, false); + let exact = record.request_json.clone(); + let mut journal = Journal::create( + temporary.path(), + record, + None, + Some(deployment::SYSTEM_EMPTY_MODULE_V1_BYTES), + ) + .unwrap(); + assert!(publish::run( + &fixture.client(), + &mut journal, + None, + Duration::ZERO, + CancellationToken::new() + ) + .await + .is_err()); + let resume = journal.directory().to_owned(); + drop(journal); + let cli_config = temporary.path().join("isolated-cli.toml"); + std::fs::write(&cli_config, "spacetimedb_token = 'isolated-fixture-credential'\n").unwrap(); + let config = Config::load(spacetimedb_paths::cli::CliTomlPath::from_path_unchecked(cli_config)).unwrap(); + let changed_project = crate::spacetime_config::LoadedConfig { + config: serde_json::from_value(json!({"database":"different-target", "module_path":"missing-build-source", "server":"must-never-resolve-this-alias"})).unwrap(), + config_dir: temporary.path().into(), loaded_files: vec![], has_dev_file: false, + }; + let args = super::super::cli() + .try_get_matches_from([ + "publish", + "--resume-publication", + resume.to_str().unwrap(), + "--server", + fixture.endpoint.as_str(), + "--publication-wait", + "0", + ]) + .unwrap(); + super::super::exec_with_options(config, &args, true, Some(&changed_project)) + .await + .unwrap(); + assert_eq!( + fixture.state.lock().unwrap().submits, + [exact.as_bytes(), exact.as_bytes()] + ); + fixture.close().await; + } + #[test] + fn nested_targets_keep_only_their_own_container_and_resume_rejects_new_inputs() { + let config: SpacetimeConfig = serde_json::from_value(json!({"database":"parent", "container":{"image":{"oci_ref":"oci:layout"},"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}}, "children":[{"database":"child"}]})).unwrap(); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command.clone().try_get_matches_from(["publish"]).unwrap(); + let targets = super::super::get_filtered_publish_configs(&config, &command, &schema, &args).unwrap(); + assert_eq!(targets.len(), 2); + assert!(targets[0].container().is_some()); + assert!(targets[1].container().is_none()); + for flags in [ + ["--resume-publication", "operation", "--remove-module"], + ["--resume-publication", "operation", "--managed"], + ] { + assert!(command + .clone() + .try_get_matches_from(std::iter::once("publish").chain(flags)) + .is_err()); + } + } +} diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index 8b26700167e..ce542a2a43f 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -324,14 +324,14 @@ enum Error { #[error("error sending subscription queries")] Subscribe { #[source] - source: WsError, + source: Box, }, #[error("protocol error: {details}")] Protocol { details: &'static str }, #[error("websocket error: {source}")] Websocket { #[source] - source: WsError, + source: Box, }, #[error("encountered failed transaction: {reason}")] TransactionFailure { reason: Box }, @@ -384,12 +384,7 @@ impl UpdateCounter { impl Error { fn is_server_closed_connection(&self) -> bool { - matches!( - self, - Self::Websocket { - source: WsError::ConnectionClosed - } - ) + matches!(self, Self::Websocket { source } if matches!(source.as_ref(), WsError::ConnectionClosed)) } } @@ -403,7 +398,7 @@ fn connection_closed_error(num: Option, num_received: u32) -> Error { None => { eprintln!("disconnected by server"); Error::Websocket { - source: WsError::ConnectionClosed, + source: Box::new(WsError::ConnectionClosed), } } } @@ -421,7 +416,9 @@ where }, ))) .unwrap(); - ws.send(msg.into()).await.map_err(|source| Error::Subscribe { source }) + ws.send(msg.into()) + .await + .map_err(|source| Error::Subscribe { source: source.into() }) } /// Send a v3 BSATN subscribe message. @@ -437,7 +434,7 @@ where let msg = bsatn::to_vec(&msg).map_err(|source| Error::BsatnEncode { source })?; ws.send(WsMessage::Binary(msg.into())) .await - .map_err(|source| Error::Subscribe { source }) + .map_err(|source| Error::Subscribe { source: source.into() }) } /// Parse a v1 text websocket message as JSON. @@ -457,7 +454,11 @@ where { const RECV_TX_UPDATE: &str = "protocol error: received transaction update before initial subscription update"; - while let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? { + while let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + { let Some(msg) = parse_msg_json(&msg) else { continue }; match msg { ws_v1::ServerMessage::InitialSubscription(sub) => { @@ -542,7 +543,11 @@ where if num.is_some_and(|n| num_received.get() >= n) { return Ok(()); } - let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + else { return Err(connection_closed_error(num, num_received.get())); }; @@ -627,7 +632,11 @@ where return Ok(Some(msg)); } - let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + else { return Ok(None); }; let WsMessage::Binary(msg) = msg else { continue }; diff --git a/crates/cli/tests/container_build_acceptance/README.md b/crates/cli/tests/container_build_acceptance/README.md new file mode 100644 index 00000000000..fd8954c0f07 --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/README.md @@ -0,0 +1,77 @@ +# Real local container builder acceptance + +This opt-in fixture exercises the CLI with actual BuildKit and explicitly +selected Railpack. It does not connect to a SpacetimeDB server. The resulting +verified OCI layouts can be consumed by the separate managed publication +acceptance test. + +The checked-in tool lock currently supports macOS on arm64. It records official +release download URLs, SHA-256 checksums, and image digests. The fixture downloads +the two tools into its own workspace and extracts only the expected regular +binary. Nothing is installed globally. Release checksum provenance is recorded +in `tool-lock.json`; image pins were verified against the primary registries. + +Build the CLI and deadline harness from the public workspace, without selecting +or connecting to a server: + +```sh +cargo build --locked --offline -p spacetimedb-cli --bin spacetimedb-cli +cargo test --locked --offline -p spacetimedb-cli --test real_container_builder --no-run +``` + +Use the absolute test executable path printed by the second command. Invoke the +script with absolute paths and a new workspace directory: + +```sh +python3 crates/cli/tests/container_build_acceptance/acceptance.py \ + --docker-socket /absolute/path/to/verified/docker-desktop.sock \ + --cli /absolute/path/to/public/target/debug/spacetimedb-cli \ + --deadline-test-binary /absolute/path/to/public/target/debug/deps/real_container_builder-HASH \ + --workspace /private/tmp/new-owned-builder-workspace +``` + +The socket must already be verified as an owned disposable Docker Desktop +endpoint. The script checks its reported host identity again. Every Docker +command specifies that socket and an empty fixture configuration; saved Docker +contexts and registry credentials are not used. The fixture pulls pinned public +images, starts a uniquely named BuildKit container with a dedicated cache volume, +and binds its port to numeric loopback. A private Unix socket forwards only to +that port. The container is limited to two CPUs, 2 GiB of memory and 256 processes. +Container logs rotate at 4 MiB with at most two files. BuildKit requires +privileged execution in this local fixture; the Docker socket +is not mounted into it. This is a trusted-tool test, not a sandbox for arbitrary +build programs. + +The checks cover: + +- A Dockerfile using a build-secret mount and an explicitly selected Railpack + shell-script build. Neither successful nor failed build logs may reveal the + secret, and retained image objects must not contain it. +- Exact digest, size, platform and executable manifest/config/layer closure of + both prepared outputs. A modified retained object is rejected on import. +- Failed builds and failed Railpack detection, with no automatic builder fallback. +- A destination created while a build is running, which must remain untouched. +- SIGINT cancellation of real `buildctl`, followed by positive PID absence and + workspace release. +- The same cleanup on a two-second deadline through the production local process + runner. Only the test runner shortens the normal build deadline. + +Commands have finite deadlines. Teardown stops the proxy, validates the exact +owned container's name and label, waits for its successful synchronous removal, +then removes its cache volume. If the run reply is lost, cleanup looks up only +the original generated name and requires the same ownership label. An ambiguous +daemon answer fails cleanup; an arbitrary inspection error is not treated as +proof of absence. A teardown failure fails the fixture. Downloaded binaries, +diagnostic inputs and the two OCI layouts remain in the private workspace; public image cache entries +are not pruned. `acceptance.json` records the completed checks and the output +paths without credentials or build-secret values. + +The retained layouts establish build and artifact preparation behavior. They do +not establish container execution, readiness, or isolation in the production +runtime. Those are separate Linux supervisor and Kata acceptance boundaries. + +The cleanup failure tests need no Docker daemon or network: + +```sh +python3 -B -m unittest discover -s crates/cli/tests/container_build_acceptance -p test_fixture.py -v +``` diff --git a/crates/cli/tests/container_build_acceptance/acceptance.py b/crates/cli/tests/container_build_acceptance/acceptance.py new file mode 100644 index 00000000000..ebbc575dd6c --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/acceptance.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Explicitly invoked real local-builder acceptance, never part of cargo test. + +Only an explicitly verified Docker Desktop Unix socket is accepted. The CLI +uses a fixture-owned BuildKit Unix proxy and never opens Spacetime credentials. +Downloaded tools and retained OCI output live under a new private workspace. +""" +import argparse +import concurrent.futures +import gzip +import hashlib +import json +import os +from pathlib import Path +import platform +import selectors +import shutil +import shlex +import signal +import socket +import subprocess +import tarfile +import threading +import time +import urllib.request +import uuid + + +LOCK = json.loads(Path(__file__).with_name("tool-lock.json").read_text()) +LIMIT = 32 * 1024 * 1024 + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def environment(root): + return { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(root / "home"), + "TMPDIR": str(root / "tmp"), + } + + +def run(args, env, timeout=120, check=True): + # No shell expansion, inherited proxy/daemon/server variables or credentials. + child = subprocess.Popen(args, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + out, err = child.communicate(timeout=timeout) + except BaseException: + child.kill() + child.communicate(timeout=10) + raise + require(len(out) <= LIMIT and len(err) <= LIMIT, "fixture command output exceeded bound") + if check: + require(child.returncode == 0, + f"fixture command failed: {Path(args[0]).name} (status {child.returncode})") + return child.returncode, out, err + + +def tools(root): + require(platform.system() == "Darwin" and platform.machine() == "arm64", + "the checked-in tool lock supports Darwin arm64") + directory = root / "tools" + directory.mkdir() + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + binaries = {} + for name, pin in LOCK["tools"].items(): + print(f"Downloading and verifying pinned {name} {pin['version']}", flush=True) + with opener.open(pin["url"], timeout=60) as response: + data = response.read(LIMIT + 1) + require(len(data) <= LIMIT, "tool archive exceeded bound") + require(hashlib.sha256(data).hexdigest() == pin["sha256"], "tool archive checksum mismatch") + archive = directory / (name + ".tar.gz") + archive.write_bytes(data) + with tarfile.open(archive) as contents: + matches = [item for item in contents if item.name.removeprefix("./") == pin["member"]] + require(len(matches) == 1 and matches[0].isreg(), "unexpected tool archive member") + require(matches[0].size <= 128 * 1024 * 1024, "tool binary exceeded bound") + binary = directory / name + with contents.extractfile(matches[0]) as source: + binary.write_bytes(source.read()) + binary.chmod(0o700) + binaries[name] = binary + return binaries + + +class UnixProxy: + """Forward only to this fixture's Docker-published numeric-loopback port.""" + def __init__(self, path, port): + self.path, self.port = path, port + self.stop = threading.Event() + self.slots = threading.BoundedSemaphore(32) + self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=32) + self.listener = socket.socket(socket.AF_UNIX) + self.listener.bind(str(path)) + path.chmod(0o600) + self.listener.listen(32) + self.listener.settimeout(0.2) + self.thread = threading.Thread(target=self.accept) + self.thread.start() + + def accept(self): + while not self.stop.is_set(): + try: + client, _ = self.listener.accept() + except TimeoutError: + continue + except OSError: + break + if self.slots.acquire(blocking=False): + self.pool.submit(self.forward, client) + else: + client.close() + + def forward(self, client): + try: + with client, socket.create_connection(("127.0.0.1", self.port), timeout=5) as upstream: + # Bounded socket writes and periodic stop checks; no filesystem + # path or request header can select another destination. + client.settimeout(1) + upstream.settimeout(1) + with selectors.DefaultSelector() as select: + select.register(client, selectors.EVENT_READ, upstream) + select.register(upstream, selectors.EVENT_READ, client) + while not self.stop.is_set(): + for key, _ in select.select(0.2): + data = key.fileobj.recv(64 * 1024) + if not data: + return + key.data.sendall(data) + except (OSError, TimeoutError): + pass + finally: + self.slots.release() + + def close(self): + self.stop.set() + self.listener.close() + self.thread.join(timeout=3) + require(not self.thread.is_alive(), "BuildKit proxy did not stop accepting") + self.pool.shutdown(wait=True) + self.path.unlink() + + +class Builder: + def __init__(self, root, docker_socket, binaries): + self.root, self.binaries = root, binaries + self.env = environment(root) + docker = shutil.which("docker") + require(docker is not None, "Docker executable is required") + self.docker = [docker, "--host", "unix://" + str(docker_socket), + "--config", str(root / "docker-config")] + self.name = "stdb-cli-build-" + uuid.uuid4().hex + self.volume = self.name + "-cache" + self.container = None + self.run_attempted = False + self.proxy = None + self.volume_created = False + + def command(self, *args, timeout=120, check=True): + return run(self.docker + list(args), self.env, timeout, check) + + def start(self): + _, info, _ = self.command("info", "--format", "{{.Name}}|{{.OperatingSystem}}|{{.OSType}}") + require(info.decode().strip() == "docker-desktop|Docker Desktop|linux", + "explicit socket is not the expected local Docker Desktop fixture host") + print("Verified explicit Docker Desktop socket; starting isolated BuildKit", flush=True) + self.command("pull", "--platform", "linux/arm64", LOCK["images"]["buildkit"], timeout=300) + _, raw, _ = self.command("image", "inspect", LOCK["images"]["buildkit"]) + inspected = json.loads(raw)[0] + require(LOCK["images"]["buildkit"] in inspected["RepoDigests"], "pulled BuildKit digest mismatch") + require(inspected["Architecture"] == "arm64", "BuildKit architecture mismatch") + self.command("volume", "create", "--label", "spacetimedb.fixture=" + self.name, self.volume) + self.volume_created = True + # A timed-out CLI may have created the container before losing its + # response. Retain the exact name/label cleanup key before dispatch. + self.run_attempted = True + _, identifier, _ = self.command( + "run", "--detach", "--pull", "never", "--platform", "linux/arm64", + "--name", self.name, "--label", "spacetimedb.fixture=" + self.name, + "--privileged", "--cpus", "2", "--memory", "2g", "--memory-swap", "2g", + "--log-driver", "json-file", "--log-opt", "max-size=4m", "--log-opt", "max-file=2", + "--pids-limit", "256", "--mount", f"type=volume,src={self.volume},dst=/var/lib/buildkit", + "--publish", "127.0.0.1::1234", LOCK["images"]["buildkit"], + "--addr", "tcp://0.0.0.0:1234", "--oci-worker-snapshotter=native") + self.container = identifier.decode().strip() + _, raw, _ = self.command("inspect", self.container) + owned = json.loads(raw)[0] + require(owned["Config"]["Labels"]["spacetimedb.fixture"] == self.name, "container ownership mismatch") + ports = owned["NetworkSettings"]["Ports"]["1234/tcp"] + require(len(ports) == 1 and ports[0]["HostIp"] == "127.0.0.1", "BuildKit is not loopback bound") + self.proxy = UnixProxy(self.root / "buildkit.sock", int(ports[0]["HostPort"])) + deadline = time.monotonic() + 30 + while True: + status, _, _ = run([str(self.binaries["buildctl"]), "--addr", self.endpoint, + "debug", "workers"], self.env, timeout=5, check=False) + if status == 0: + break + require(time.monotonic() < deadline, "BuildKit fixture did not become ready") + time.sleep(0.2) + + @property + def endpoint(self): + return "unix://" + str(self.root / "buildkit.sock") + + def close(self): + errors = [] + if self.proxy: + try: + self.proxy.close() + except Exception as error: + errors.append(str(error)) + if self.run_attempted: + try: + # Inspect only the exact returned ID or generated name. A + # failed/ambiguous inspect is not proof of absence. The label + # must match before deletion, including after a lost run reply. + _, raw, _ = self.command("container", "inspect", self.container or self.name) + objects = json.loads(raw) + require(len(objects) == 1, "ambiguous owned container lookup") + owned = objects[0] + require(owned["Name"] == "/" + self.name and + owned["Config"]["Labels"]["spacetimedb.fixture"] == self.name, + "container cleanup ownership mismatch") + if self.container: + require(owned["Id"] == self.container, "container cleanup ID mismatch") + # A successful synchronous removal is positive completion. + self.command("rm", "--force", "--volumes", owned["Id"]) + except Exception as error: + errors.append(str(error)) + if self.volume_created: + try: + self.command("volume", "rm", self.volume) + except Exception as error: + errors.append(str(error)) + require(not errors, "positive builder teardown failed: " + "; ".join(errors)) + + +def config(project, image): + project.mkdir() + document = { + "database": "local-builder-fixture", + "container": { + "image": image, + "env_keys": ["RUNTIME_ONLY"], + "resources": {"cpu_millicores": 1000, "memory_bytes": 536870912, + "scratch_bytes": 1073741824, "pids_max": 128}, + }, + } + (project / "spacetime.json").write_text(json.dumps(document)) + + +def verify_layout(layout, secret): + metadata = json.loads((layout / "prepared.json").read_text()) + require(secret not in (layout / "prepared.json").read_bytes(), "build secret entered prepared metadata") + objects = {} + for item in metadata["objects"]: + descriptor = item["descriptor"] + path = Path(item["path"]) + require(not path.is_absolute() and ".." not in path.parts, "artifact path escaped layout") + data = (layout / path).read_bytes() + require(len(data) == descriptor["size"], "artifact size mismatch") + require("sha256:" + hashlib.sha256(data).hexdigest() == descriptor["digest"], "artifact hash mismatch") + require(secret not in data, "build secret entered retained image object") + if item["kind"] == "layer" and data[:2] == b"\x1f\x8b": + require(secret not in gzip.decompress(data), "build secret entered retained image layer") + objects[descriptor["digest"]] = (item["kind"], data) + manifest = json.loads(objects[metadata["manifest"]["digest"]][1]) + require(set(objects) == {metadata["manifest"]["digest"], manifest["config"]["digest"], + *(entry["digest"] for entry in manifest["layers"])}, + "prepared closure is not the exact executable manifest/config/layers") + image = json.loads(objects[manifest["config"]["digest"]][1]) + require(image["os"] == "linux" and image["architecture"] == "arm64", "image platform mismatch") + return metadata, image + + +def cases(root, cli, binaries, builder, deadline_test): + secret = ("build-secret-" + uuid.uuid4().hex).encode() + secret_file = root / "build-secret" + secret_file.write_bytes(secret) + secret_file.chmod(0o600) + base = [str(cli), "--root-dir", str(root / "cli-root"), "--config-path", str(root / "unused-config"), + "container", "build", "--platform", "linux/arm64", "--buildctl", str(binaries["buildctl"]), + "--railpack", str(binaries["railpack"]), "--buildkit-host", builder.endpoint] + + def build(project, output, success=True, secrets=False): + args = base + ["--project-path", str(project), "--out-dir", str(output)] + if secrets: + args += ["--build-secret", "BUILD_SENTINEL=" + str(secret_file)] + status, out, err = run(args, environment(root), timeout=1200, check=False) + require(secret not in out + err, "CLI leaked builder secret diagnostics") + if (status == 0) != success: + # Error output has been checked for the generated sentinel and the + # fixture never supplies ordinary or registry credentials. + raise RuntimeError("CLI build outcome mismatch: " + (out + err).decode(errors="replace")[:2048]) + return out + err + + project = root / "dockerfile-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\n" + "RUN --mount=type=secret,id=BUILD_SENTINEL test -s /run/secrets/BUILD_SENTINEL && cat /run/secrets/BUILD_SENTINEL >&2\n" + "WORKDIR /app\nUSER 1001:1001\nENV IMAGE_DEFAULT=retained\n" + 'ENTRYPOINT ["/bin/sh"]\nCMD ["-c", "echo dockerfile-ready"]\n') + output = root / "dockerfile-output" + print("Actual Dockerfile build with explicit secret mount", flush=True) + build(project, output, secrets=True) + metadata, image = verify_layout(output, secret) + require(metadata["container"]["argv"] == ["/bin/sh", "-c", "echo dockerfile-ready"], "argv was not normalized") + require(metadata["container"]["user"] == "1001:1001", "image user was lost") + require(metadata["container"]["working_directory"] == "/app", "image working directory was lost") + require("IMAGE_DEFAULT=retained" in image["config"]["Env"], "image environment defaults were lost") + + project = root / "railpack-project" + config(project, {"build": {"builder": "railpack", "context": "."}}) + (project / "start.sh").write_text("#!/bin/sh\nprintf 'railpack-ready\\n'\n") + (project / "start.sh").chmod(0o700) + output = root / "railpack-output" + print("Actual explicitly selected Railpack build", flush=True) + build(project, output) + verify_layout(output, secret) + + print("Failed secret-using build must not expose logs or publish output", flush=True) + project = root / "failed-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\n" + "RUN --mount=type=secret,id=BUILD_SENTINEL cat /run/secrets/BUILD_SENTINEL >&2; exit 23\n") + output = root / "failed-output" + build(project, output, success=False, secrets=True) + require(not output.exists(), "failed build published output") + + print("Explicit unsupported Railpack detection cannot fall back to Dockerfile", flush=True) + project = root / "unsupported-project" + config(project, {"build": {"builder": "railpack", "context": "."}}) + (project / "Dockerfile").write_text('FROM scratch\nCMD ["would-be-wrong-builder"]\n') + output = root / "unsupported-output" + build(project, output, success=False) + require(not output.exists(), "unsupported Railpack silently used another builder") + + print("Tampering with a retained OCI object must fail before output publication", flush=True) + tampered = root / "tampered-layout" + shutil.copytree(root / "dockerfile-output", tampered) + metadata = json.loads((tampered / "prepared.json").read_text()) + target = next(item for item in metadata["objects"] if item["kind"] == "config") + with (tampered / target["path"]).open("ab") as changed: + changed.write(b" ") + project = root / "tampered-project" + config(project, {"oci_ref": "oci:" + str(tampered)}) + output = root / "tampered-output" + build(project, output, success=False) + require(not output.exists(), "tampered OCI image was accepted") + + print("A path created during the actual build must never be replaced", flush=True) + project = root / "replacement-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\nRUN sleep 3\nCMD [\"/bin/true\"]\n") + output = root / "replacement-output" + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task = pool.submit(build, project, output, False) + deadline = time.monotonic() + 30 + while not list(root.glob(".spacetime-image-*")): + require(not task.done(), "build finished before concurrent replacement could be tested") + require(time.monotonic() < deadline, "build never created its private workspace") + time.sleep(0.01) + output.mkdir() + (output / "sentinel").write_text("must survive") + task.result(timeout=60) + require(list(output.iterdir()) == [output / "sentinel"], "existing output was replaced") + require((output / "sentinel").read_text() == "must survive", "existing output content changed") + + print("SIGINT must cancel the real buildctl and positively reap it", flush=True) + project = root / "cancel-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\nRUN sleep 120\nCMD [\"/bin/true\"]\n") + output = root / "cancel-output" + pid_file = root / "actual-buildctl.pid" + wrapper = root / "record-buildctl" + wrapper.write_text("#!/bin/sh\nprintf '%s\\n' \"$$\" > " + shlex.quote(str(pid_file)) + + "\nexec " + shlex.quote(str(binaries["buildctl"])) + ' "$@"\n') + wrapper.chmod(0o700) + args = list(base) + args[args.index("--buildctl") + 1] = str(wrapper) + args += ["--project-path", str(project), "--out-dir", str(output)] + child = subprocess.Popen(args, env=environment(root), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 30 + while not pid_file.exists(): + require(child.poll() is None, "CLI exited before invoking real buildctl") + require(time.monotonic() < deadline, "real buildctl did not start") + time.sleep(0.01) + tool_pid = int(pid_file.read_text()) + child.send_signal(signal.SIGINT) + out, err = child.communicate(timeout=15) + require(child.returncode != 0 and b"cancelled" in out + err, "CLI did not report cancellation") + require(secret not in out + err, "cancelled CLI leaked a secret") + try: + os.kill(tool_pid, 0) + except ProcessLookupError: + pass + else: + raise RuntimeError("real buildctl PID still exists after acknowledged CLI cancellation") + require(not output.exists(), "cancelled build published output") + finally: + if child.poll() is None: + child.kill() + child.communicate(timeout=10) + require(not list(root.glob(".spacetime-image-*")), "completed CLI left builder workspaces behind") + + print("A short harness deadline must reap real buildctl before releasing its workspace", flush=True) + pid_file.unlink() + deadline_env = environment(root) + deadline_env.update({ + "STDB_BUILDER_CONTEXT": str(project), + "STDB_BUILDER_BUILDCTL": str(wrapper), + "STDB_BUILDER_PID_FILE": str(pid_file), + "STDB_BUILDER_WORKSPACE": str(root), + "STDB_BUILDER_SOCKET": builder.endpoint, + }) + run([str(deadline_test), "--ignored", "--exact", + "actual_buildctl_deadline_reaps_before_workspace_release", "--test-threads=1"], + deadline_env, timeout=30) + return { + "version": 1, + "dockerfile": str(root / "dockerfile-output"), + "railpack": str(root / "railpack-output"), + "tool_lock": LOCK, + "checks": ["actual_dockerfile", "actual_railpack", "secret_logs", "failed_build", + "no_fallback", "tampered_closure", "concurrent_output", "cancel_and_reap", + "deadline_and_reap"], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--docker-socket", type=Path, required=True) + parser.add_argument("--cli", type=Path, required=True) + parser.add_argument("--deadline-test-binary", type=Path, required=True) + parser.add_argument("--workspace", type=Path, required=True) + args = parser.parse_args() + require(args.docker_socket.is_absolute() and args.docker_socket.is_socket(), "explicit Docker Unix socket required") + require(args.cli.is_absolute() and args.cli.is_file(), "absolute prebuilt CLI executable required") + require(args.deadline_test_binary.is_absolute() and args.deadline_test_binary.is_file(), + "absolute prebuilt deadline test executable required") + require(args.workspace.is_absolute() and not args.workspace.exists(), "workspace must be a new absolute directory") + args.workspace.mkdir(mode=0o700) + for child in ["home", "tmp", "docker-config"]: + (args.workspace / child).mkdir(mode=0o700) + # A missing named config proves local build dispatch does not read global + # credentials or open the supplied ordinary configuration path. + binaries = tools(args.workspace) + builder = Builder(args.workspace, args.docker_socket, binaries) + try: + builder.start() + receipt = cases(args.workspace, args.cli, binaries, builder, args.deadline_test_binary) + finally: + builder.close() + (args.workspace / "acceptance.json").write_text(json.dumps(receipt, indent=2)) + print("Builder acceptance passed; retained OCI layouts are ready for managed publication", flush=True) + + +if __name__ == "__main__": + main() diff --git a/crates/cli/tests/container_build_acceptance/test_fixture.py b/crates/cli/tests/container_build_acceptance/test_fixture.py new file mode 100644 index 00000000000..40ed63ea94d --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/test_fixture.py @@ -0,0 +1,64 @@ +"""Owned-resource failure paths, without contacting Docker or any server.""" +import json +import unittest + +from acceptance import Builder + + +class CleanupTests(unittest.TestCase): + def builder(self, reply): + builder = Builder.__new__(Builder) + builder.name = "fixture-owned-name" + builder.container = None # docker run reply was lost + builder.run_attempted = True + builder.proxy = None + builder.volume_created = False + calls = [] + + def command(*args): + calls.append(args) + if args[:2] == ("container", "inspect"): + if isinstance(reply, Exception): + raise reply + return 0, json.dumps(reply).encode(), b"" + return 0, b"", b"" + + builder.command = command + return builder, calls + + def owned(self): + return [{"Name": "/fixture-owned-name", "Id": "owned-id", "Config": { + "Labels": {"spacetimedb.fixture": "fixture-owned-name"}}}] + + def test_lost_run_reply_removes_only_exact_owned_container(self): + builder, calls = self.builder(self.owned()) + builder.close() + self.assertEqual(calls, [ + ("container", "inspect", "fixture-owned-name"), + ("rm", "--force", "--volumes", "owned-id"), + ]) + + def test_wrong_label_never_authorizes_removal(self): + reply = self.owned() + reply[0]["Config"]["Labels"]["spacetimedb.fixture"] = "another-fixture" + builder, calls = self.builder(reply) + with self.assertRaisesRegex(RuntimeError, "ownership mismatch"): + builder.close() + self.assertEqual(len(calls), 1) + + def test_inspect_error_is_incomplete_cleanup(self): + builder, calls = self.builder(RuntimeError("daemon unavailable")) + with self.assertRaisesRegex(RuntimeError, "positive builder teardown failed"): + builder.close() + self.assertEqual(len(calls), 1) + + def test_returned_identifier_must_match_inspected_object(self): + builder, calls = self.builder(self.owned()) + builder.container = "different-id" + with self.assertRaisesRegex(RuntimeError, "ID mismatch"): + builder.close() + self.assertEqual(len(calls), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/crates/cli/tests/container_build_acceptance/tool-lock.json b/crates/cli/tests/container_build_acceptance/tool-lock.json new file mode 100644 index 00000000000..e62b8e64ec2 --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/tool-lock.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "platform": "darwin-arm64", + "tools": { + "buildctl": { + "version": "0.33.0", + "url": "https://github.com/moby/buildkit/releases/download/v0.33.0/buildkit-v0.33.0.darwin-arm64.tar.gz", + "sha256": "730ac4ffd6f4a88dc404fc675aeaf4cfee414915036042847f0a60861cc8790c", + "member": "bin/buildctl", + "source": "https://api.github.com/repos/moby/buildkit/releases/tags/v0.33.0" + }, + "railpack": { + "version": "0.35.0", + "url": "https://github.com/railwayapp/railpack/releases/download/v0.35.0/railpack-v0.35.0-arm64-apple-darwin.tar.gz", + "sha256": "fb4c16d57458eb7868d48ed8a454014ef40716a6c939929d7b7d5986563a0c65", + "member": "railpack", + "source": "https://api.github.com/repos/railwayapp/railpack/releases/tags/v0.35.0" + } + }, + "images": { + "buildkit": "moby/buildkit@sha256:6c2fa84a6b61ccd72899dde4239f8d5717f05f9a8ca6f3cad185fb1a95a94de3", + "buildkit_arm64": "sha256:e8efce994e456acb94944bcc0530b3188478572914f413d4476568d8b63515c6", + "alpine": "alpine@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1", + "railpack_frontend": "ghcr.io/railwayapp/railpack-frontend@sha256:bc73534934e7929ab3dc41765fb7e25c8c69d9be98c43ef8792fea51f65317bd" + }, + "image_sources": [ + "https://registry-1.docker.io/v2/moby/buildkit/manifests/v0.33.0", + "https://registry-1.docker.io/v2/library/alpine/manifests/3.22.1", + "https://ghcr.io/v2/railwayapp/railpack-frontend/manifests/v0.35.0" + ] +} diff --git a/crates/cli/tests/real_container_builder.rs b/crates/cli/tests/real_container_builder.rs new file mode 100644 index 00000000000..cad32c9c63b --- /dev/null +++ b/crates/cli/tests/real_container_builder.rs @@ -0,0 +1,80 @@ +//! Run only from container_build_acceptance/acceptance.py, which owns and +//! verifies the Docker/BuildKit endpoint and all paths below. No saved defaults. +#![cfg(any(target_os = "macos", target_os = "linux"))] + +use anyhow::{ensure, Context, Result}; +use spacetimedb_cli::container::{ + config::ContainerConfig, + prepare_container, + process::{Invocation, LocalRunner, Output, Runner}, + BuildTools, +}; +use spacetimedb_lib::container::ImagePlatform; +use std::{path::PathBuf, time::Duration}; +use tokio_util::sync::CancellationToken; + +struct ShortDeadline; +impl Runner for ShortDeadline { + async fn run(&self, mut invocation: Invocation) -> Result { + // Exercise the production local owner/kill/reap path without making + // acceptance wait for the ordinary thirty-minute build deadline. + invocation.timeout = Duration::from_secs(2); + LocalRunner.run(invocation).await + } +} + +fn input(name: &str) -> Result { + let path = PathBuf::from(std::env::var_os(name).with_context(|| format!("explicit {name} is required"))?); + ensure!(path.is_absolute(), "fixture input must be absolute"); + Ok(path) +} + +#[tokio::test] +#[ignore = "requires the explicitly owned local BuildKit acceptance fixture"] +async fn actual_buildctl_deadline_reaps_before_workspace_release() -> Result<()> { + let context = input("STDB_BUILDER_CONTEXT")?.canonicalize()?; + let tool = input("STDB_BUILDER_BUILDCTL")?.canonicalize()?; + let pid_file = input("STDB_BUILDER_PID_FILE")?; + let workspace = input("STDB_BUILDER_WORKSPACE")?.canonicalize()?; + let endpoint = std::env::var("STDB_BUILDER_SOCKET")?; + ensure!( + endpoint.starts_with("unix:///"), + "explicit local BuildKit Unix socket is required" + ); + let document: serde_json::Value = serde_json::from_slice(&std::fs::read(context.join("spacetime.json"))?)?; + let configuration: ContainerConfig = serde_json::from_value(document["container"].clone())?; + let result = prepare_container( + &configuration, + &context, + ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + &BuildTools { + buildctl: tool, + buildkit_host: Some(endpoint), + ..Default::default() + }, + &workspace, + &ShortDeadline, + CancellationToken::new(), + ) + .await; + let error = result.err().context("long-running real build unexpectedly succeeded")?; + ensure!( + error.to_string().contains("build deadline"), + "unexpected build failure: {error:#}" + ); + let pid = std::fs::read_to_string(pid_file)?.trim().parse()?; + let pid = rustix::process::Pid::from_raw(pid).context("invalid recorded builder PID")?; + ensure!( + rustix::process::test_kill_process(pid) == Err(rustix::io::Errno::SRCH), + "real buildctl PID still exists after the deadline returned" + ); + ensure!( + std::fs::read_dir(workspace)?.all(|entry| entry + .is_ok_and(|entry| !entry.file_name().to_string_lossy().starts_with(".spacetime-image-"))), + "deadline returned before its owned workspace was released" + ); + Ok(()) +} diff --git a/crates/client-api/src/auth.rs b/crates/client-api/src/auth.rs index 729d001e2c3..38d91489c7e 100644 --- a/crates/client-api/src/auth.rs +++ b/crates/client-api/src/auth.rs @@ -88,6 +88,7 @@ pub struct SpacetimeAuth { pub claims: SpacetimeIdentityClaims, /// The JWT payload as a json string (after base64 decoding). pub jwt_payload: Box, + pub hosted: Option, } impl SpacetimeAuth { @@ -100,6 +101,7 @@ impl SpacetimeAuth { creds, claims, jwt_payload: payload, + hosted: None, }) } } @@ -109,6 +111,7 @@ impl From for ConnectionAuthCtx { ConnectionAuthCtx { claims: auth.claims, jwt_payload: auth.jwt_payload.clone(), + hosted: auth.hosted, } } } @@ -214,6 +217,9 @@ impl SpacetimeAuth { signer: &impl TokenSigner, expiry: Duration, ) -> Result<(SpacetimeIdentityClaims, String), JwtError> { + if self.hosted.is_some() { + return Err(JwtErrorKind::InvalidToken.into()); + } TokenClaims::from(self.clone()).encode_and_sign_with_expiry(signer, Some(expiry)) } } @@ -402,13 +408,47 @@ pub struct SpacetimeAuthHeader { } #[async_trait::async_trait] -impl axum::extract::FromRequestParts for SpacetimeAuthHeader { +impl axum::extract::FromRequestParts for SpacetimeAuthHeader { type Rejection = AuthorizationRejection; async fn from_request_parts(parts: &mut request::Parts, state: &S) -> Result { let Some(creds) = SpacetimeCreds::from_request_parts(parts)? else { return Ok(Self { auth: None }); }; + if spacetimedb::auth::hosted_tokens::has_reserved_hosted_token_kind(&creds.token) + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))? + { + // Use Axum's matched route parameters, never a client header or a + // hand-parsed URL suffix. Tokens are ineligible for root publishing, + // identity allocation, and generic token exchange routes. + #[derive(Deserialize)] + struct HostedTargetPath { + name_or_identity: crate::util::NameOrIdentity, + } + let axum::extract::Path(params) = axum::extract::Path::::from_request_parts(parts, state) + .await + .map_err(|_| AuthorizationRejection::Required)?; + let target = params + .name_or_identity + .resolve(state) + .await + .map_err(|_| AuthorizationRejection::Required)?; + let verified = state + .authenticate_hosted_token(&creds.token, target) + .await + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))?; + let connection = verified + .into_connection_auth() + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))?; + let auth = SpacetimeAuth { + creds, + claims: connection.claims, + jwt_payload: connection.jwt_payload, + hosted: connection.hosted, + }; + return Ok(Self { auth: Some(auth) }); + } + let claims = validate_token(state, &creds.token) .await .map_err(AuthorizationRejection::Custom)?; @@ -420,6 +460,7 @@ impl axum::extract::FromRequestParts for Space creds, claims, jwt_payload: payload.into(), + hosted: None, }; Ok(Self { auth: Some(auth) }) } @@ -478,7 +519,9 @@ impl SpacetimeAuthHeader { pub struct SpacetimeAuthRequired(pub SpacetimeAuth); #[async_trait::async_trait] -impl axum::extract::FromRequestParts for SpacetimeAuthRequired { +impl axum::extract::FromRequestParts + for SpacetimeAuthRequired +{ type Rejection = AuthorizationRejection; async fn from_request_parts(parts: &mut request::Parts, state: &S) -> Result { let auth = SpacetimeAuthHeader::from_request_parts(parts, state).await?; diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 68fd0a61bb5..264ed505421 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -55,6 +55,15 @@ pub trait NodeDelegate: Send + Sync { type JwtAuthProviderT: auth::JwtAuthProvider; fn jwt_auth_provider(&self) -> &Self::JwtAuthProviderT; + /// Authenticate a platform-issued hosted credential for the exact resolved + /// database in this request. Editions without container hosting fail closed. + async fn authenticate_hosted_token( + &self, + _token: &str, + _target: Identity, + ) -> anyhow::Result { + anyhow::bail!("Hosted database credentials are not enabled on this server") + } /// Return the leader [`Host`] of `database_id`. /// /// The [`Host`] is spawned implicitly if not already running. @@ -136,7 +145,7 @@ impl Host { pub async fn exec_sql( &self, - auth: AuthCtx, + auth: impl Into, _database: Database, confirmed_read: bool, body: String, @@ -206,6 +215,28 @@ impl Host { .update_module_host(database, host_type, self.replica_id, program_bytes, policy) .await } + + /// Used only by an authenticated publication coordinator after control + /// admission and quiescing. This does not authorize or start a container. + pub async fn update_with_deployment( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + deployment: spacetimedb::db::deployment::DeploymentCommit, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_deployment( + database, + host_type, + self.replica_id, + program_bytes, + policy, + Some(deployment), + ) + .await + } } /// Parameters for publishing a database. /// @@ -494,6 +525,14 @@ impl NodeDelegate for Arc { (**self).jwt_auth_provider() } + async fn authenticate_hosted_token( + &self, + token: &str, + target: Identity, + ) -> anyhow::Result { + (**self).authenticate_hosted_token(token, target).await + } + async fn leader(&self, database_id: u64) -> Result { (**self).leader(database_id).await } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 9d3561b4542..654852c3203 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -165,13 +165,14 @@ pub async fn call( let caller_auth: ConnectionAuthCtx = auth.into(); + let caller = spacetimedb::auth::invocation::InvocationCaller::from(&caller_auth); let owner_identity = database.owner_identity; let module = find_database_module(&worker_ctx, &database).await?; - let fut = async move |module: ModuleHost, caller_identity: Identity, connection_id: ConnectionId| { + let fut = async move |module: ModuleHost, _caller_identity: Identity, connection_id: ConnectionId| { let result = match module .call_reducer( - caller_identity, + caller.clone(), Some(connection_id), None, None, @@ -185,7 +186,7 @@ pub async fn call( Err(ReducerCallError::NoSuchReducer | ReducerCallError::ScheduleReducerNotFound) => { // Not a reducer — try procedure instead match module - .call_procedure(caller_identity, Some(connection_id), None, &reducer, args) + .call_procedure(caller, Some(connection_id), None, &reducer, args) .await .result { @@ -741,11 +742,14 @@ where let host = find_database_leader(&worker_ctx, &database).await?; let module = host.module().await.map_err(log_and_500)?; + let sql_caller_auth = caller_auth.clone(); let fut = async move |_module: ModuleHost, caller_identity: Identity, _connection_id: ConnectionId| { let sql_auth = worker_ctx .authorize_sql(caller_identity, database.database_identity) .await?; + let sql_auth = spacetimedb::auth::invocation::SqlCallAuth::authenticated(sql_auth, &sql_caller_auth) + .map_err(log_and_500)?; host.exec_sql( sql_auth, database, @@ -1067,6 +1071,11 @@ pub async fn publish( | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { tx_offset, durable_offset, + } + | UpdateDatabaseResult::DeploymentAlreadyCommitted { + tx_offset, + durable_offset, + .. }, ) => { timeout(confirmation_timeout.min(MAX_UPDATE_CONFIRMATION_TIMEOUT), async { diff --git a/crates/codegen/src/util.rs b/crates/codegen/src/util.rs index fdb0e7fcce9..80348ea0bef 100644 --- a/crates/codegen/src/util.rs +++ b/crates/codegen/src/util.rs @@ -86,7 +86,10 @@ pub(super) fn type_ref_name(module: &ModuleDef, typeref: AlgebraicTypeRef) -> St pub(super) fn is_type_filterable(typespace: &TypespaceForGenerate, ty: &AlgebraicTypeUse) -> bool { match ty { AlgebraicTypeUse::Primitive(prim) => !matches!(prim, PrimitiveType::F32 | PrimitiveType::F64), - AlgebraicTypeUse::String | AlgebraicTypeUse::Identity | AlgebraicTypeUse::ConnectionId => true, + AlgebraicTypeUse::String + | AlgebraicTypeUse::Identity + | AlgebraicTypeUse::ConnectionId + | AlgebraicTypeUse::Uuid => true, // Sum types with all unit variants: AlgebraicTypeUse::Never => true, AlgebraicTypeUse::Option(inner) => matches!(&**inner, AlgebraicTypeUse::Unit), diff --git a/crates/codegen/tests/codegen.rs b/crates/codegen/tests/codegen.rs index b2a04309478..f4a1ff33b03 100644 --- a/crates/codegen/tests/codegen.rs +++ b/crates/codegen/tests/codegen.rs @@ -86,3 +86,45 @@ fn submodule_reducer_wire_name_is_qualified_once() { "namespace was applied twice somewhere in the generated bindings" ); } + +#[test] +fn rust_uuid_primary_and_unique_keys_generate_registered_lookup_accessors() { + use spacetimedb_lib::{ + db::raw_def::{v10::RawModuleDefV10Builder, v9::btree}, + AlgebraicType, + }; + + let mut builder = RawModuleDefV10Builder::new(); + builder + .build_table_with_new_type( + "uuid_rows", + [ + ("id", AlgebraicType::uuid()), + ("alias", AlgebraicType::uuid()), + ("sequence", AlgebraicType::U64), + ], + true, + ) + .with_primary_key(0) + .with_unique_constraint(0) + .with_index(btree(0), "uuid_rows_id_idx", "id") + .with_unique_constraint(1) + .with_index(btree(1), "uuid_rows_alias_idx", "alias") + .with_unique_constraint(2) + .with_index(btree(2), "uuid_rows_sequence_idx", "sequence") + .finish(); + let module = ModuleDef::try_from(builder.finish()).unwrap(); + let table = generate(&module, &Rust, &CodegenOptions::default()) + .into_iter() + .find(|file| file.filename == "uuid_rows_table.rs") + .unwrap() + .code; + for column in ["id", "alias"] { + assert!(table.contains(&format!("pub fn {column}(&self)"))); + assert!(table.contains(&format!( + "add_unique_constraint::<__sdk::Uuid>({column:?}, |row| &row.{column})" + ))); + } + assert!(table.contains("pub fn find(&self, col_val: &__sdk::Uuid) -> Option")); + assert!(table.contains("add_unique_constraint::(\"sequence\", |row| &row.sequence)")); +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0cd5fcef773..a2df624dacc 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -98,6 +98,7 @@ tracing-subscriber.workspace = true tracing-tracy.workspace = true tracing.workspace = true url.workspace = true +uuid = { workspace = true, features = ["v4", "v7"] } v8.workspace = true wasmtime.workspace = true wasmtime-internal-fiber.workspace = true diff --git a/crates/core/src/auth/hosted_tokens.rs b/crates/core/src/auth/hosted_tokens.rs new file mode 100644 index 00000000000..77622b2706a --- /dev/null +++ b/crates/core/src/auth/hosted_tokens.rs @@ -0,0 +1,332 @@ +//! Explicit platform trust for hosted database credentials. No OIDC discovery or fallback. + +use anyhow::{ensure, Context}; +use jsonwebtoken::DecodingKey; +pub use spacetimedb_auth::hosted::{ + has_reserved_hosted_token_kind, sign_hosted_token, HostedTokenBinding, HostedTokenClaims, VerifiedHostedAuth, + HOSTED_TOKEN_KIND, HOSTED_TOKEN_TYPE, MAX_HOSTED_TOKEN_LIFETIME, +}; +use spacetimedb_auth::hosted::{unverified_hosted_token_claims, verify_hosted_token}; +use spacetimedb_lib::Identity; +use std::collections::HashMap; +use std::time::SystemTime; + +/// Only configured platform signers can attest registered source databases. +pub struct HostedTokenValidator { + trusted_issuers: HashMap, DecodingKey>, +} + +impl HostedTokenValidator { + pub fn new(issuers: impl IntoIterator, DecodingKey)>) -> anyhow::Result { + let mut trusted_issuers = HashMap::new(); + for (issuer, key) in issuers { + ensure!( + !issuer.is_empty() && issuer.len() <= 128, + "invalid trusted hosted issuer" + ); + ensure!( + trusted_issuers.insert(issuer, key).is_none(), + "duplicate trusted hosted issuer" + ); + } + Ok(Self { trusted_issuers }) + } + + /// `resolve_binding` reads authoritative state, including this issuer's source + /// registration, open admission, current placement/incarnation and target grant. + /// Return None if any requirement is absent. Its inputs are untrusted routing hints; + /// the callback must never copy them into a fabricated binding or mutate state. + /// The returned proof still requires target-fence checks at every later admission. + pub fn validate_token( + &self, + token: &str, + target: Identity, + now: SystemTime, + resolve_binding: impl FnOnce(&str, Identity, Identity) -> Option, + ) -> anyhow::Result { + let hints = unverified_hosted_token_claims(token)?; + ensure!(hints.target_database == target, "hosted credential target mismatch"); + let key = self + .trusted_issuers + .get(&hints.issuer) + .context("untrusted hosted credential issuer")?; + let binding = resolve_binding(&hints.issuer, hints.source_database, target) + .context("hosted source registration, instance, or target grant is unavailable")?; + ensure!( + binding.target_database == target, + "authoritative hosted binding target mismatch" + ); + verify_hosted_token(token, key, &hints.issuer, &binding, now) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::{ + token_validation::{FullTokenValidator, TokenValidator, UnimplementedTokenValidator}, + JwtKeys, + }; + use jsonwebtoken::{Algorithm, Header}; + use serde_json::{json, Value}; + use spacetimedb_auth::hosted::{has_reserved_hosted_token_kind, MAX_HOSTED_TOKEN_BYTES}; + use std::time::{Duration, UNIX_EPOCH}; + + fn fixture() -> (JwtKeys, HostedTokenValidator, HostedTokenBinding, SystemTime) { + let keys = JwtKeys::generate().unwrap(); + let validator = HostedTokenValidator::new([("platform.test".into(), keys.public.clone())]).unwrap(); + let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let binding = HostedTokenBinding { + source_database: Identity::from_claims("source", "database"), + target_database: Identity::from_claims("target", "database"), + generation: 9_007_199_254_740_993, + grant_revision: 9_007_199_254_740_995, + lease_expires_at: now + Duration::from_secs(30), + }; + (keys, validator, binding, now) + } + + fn mint(keys: &JwtKeys, binding: &HostedTokenBinding, now: SystemTime) -> String { + sign_hosted_token( + &keys.private, + "platform.test", + binding, + now, + now + Duration::from_secs(20), + "token-private-id", + ) + .unwrap() + } + + fn change(token: &str, keys: &JwtKeys, mutate: impl FnOnce(&mut Value, &mut Header)) -> String { + let mut claims = serde_json::to_value(unverified_hosted_token_claims(token).unwrap()).unwrap(); + let mut header = Header::new(Algorithm::ES256); + header.typ = Some(HOSTED_TOKEN_TYPE.into()); + mutate(&mut claims, &mut header); + jsonwebtoken::encode(&header, &claims, &keys.private).unwrap() + } + + #[test] + fn hosted_sender_target_authority_and_claims_are_preserved() { + let (keys, validator, mut binding, now) = fixture(); + for self_call in [false, true] { + if self_call { + binding.target_database = binding.source_database; + } + let token = mint(&keys, &binding, now); + assert!(has_reserved_hosted_token_kind(&token).unwrap()); + let verified = validator + .validate_token(&token, binding.target_database, now, |issuer, source, target| { + assert_eq!(issuer, "platform.test"); + assert_eq!(source, binding.source_database); + assert_eq!(target, binding.target_database); + Some(binding) + }) + .unwrap(); + assert_eq!(verified.is_internal(), self_call); + assert_eq!(verified.generation(), binding.generation); + assert_eq!(verified.grant_revision(), binding.grant_revision); + assert!(verified.check_at(now + Duration::from_secs(20)).is_err()); + let ctx = verified.into_connection_auth().unwrap(); + assert_eq!(ctx.claims.identity, binding.source_database); + assert_ne!( + ctx.claims.identity, + Identity::from_claims(&ctx.claims.issuer, &ctx.claims.subject) + ); + assert!(ctx.hosted.is_some()); + let payload: Value = serde_json::from_str(&ctx.jwt_payload).unwrap(); + assert_eq!(payload["generation"].as_u64(), Some(binding.generation)); + assert_eq!(payload["grant_revision"].as_u64(), Some(binding.grant_revision)); + assert_eq!(payload["aud"], binding.target_database.to_hex().as_str()); + assert_eq!(payload["iss"], "platform.test"); + let debug = format!("{ctx:?}"); + assert!(!debug.contains("token-private-id")); + assert!(!debug.contains(&token)); + assert!(!debug.contains("jwt_payload")); + } + } + + #[test] + fn hosted_validation_rejects_wrong_authority_binding_and_wire_shape() { + let (keys, validator, binding, now) = fixture(); + let token = mint(&keys, &binding, now); + let other_keys = JwtKeys::generate().unwrap(); + assert!(validator + .validate_token( + &mint(&other_keys, &binding, now), + binding.target_database, + now, + |_, _, _| Some(binding) + ) + .is_err()); + assert!(validator + .validate_token(&token, binding.source_database, now, |_, _, _| Some(binding)) + .is_err()); + assert!(validator + .validate_token(&token, binding.target_database, now, |_, _, _| None) + .is_err()); + let invalid_fields = [ + ("kind", json!("spacetimedb_hosted_v2")), + ("iss", json!("unknown.test")), + ("source_database", json!(binding.target_database.to_hex().as_str())), + ("sub", json!("other")), + ("aud", json!(binding.source_database.to_hex().as_str())), + ("aud", json!([binding.target_database.to_hex().as_str()])), + ("generation", json!(binding.generation - 1)), + ("grant_revision", json!(binding.grant_revision - 1)), + ("iat", json!(1_700_000_001_u64)), + ("exp", json!(1_700_000_000_u64)), + ("exp", json!(1_700_000_031_u64)), + ("exp", json!(u64::MAX)), + ("jti", json!("")), + ("hex_identity", json!(binding.source_database.to_hex().as_str())), + ]; + for (field, value) in invalid_fields { + let changed = change(&token, &keys, |claims, _| claims[field] = value); + assert!( + validator + .validate_token(&changed, binding.target_database, now, |_, _, _| Some(binding)) + .is_err(), + "accepted changed {field}" + ); + } + let wrong_type = change(&token, &keys, |_, header| header.typ = Some("JWT".into())); + assert!(validator + .validate_token(&wrong_type, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let missing_exp = change(&token, &keys, |claims, _| { + claims.as_object_mut().unwrap().remove("exp"); + }); + assert!(validator + .validate_token(&missing_exp, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let overflowing_time = change(&token, &keys, |claims, _| { + claims["iat"] = json!(u64::MAX - 20); + claims["exp"] = json!(u64::MAX); + }); + assert!(validator + .validate_token(&overflowing_time, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let short_lease = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(19), + ..binding + }; + assert!(validator + .validate_token(&token, binding.target_database, now, |_, _, _| Some(short_lease)) + .is_err()); + assert!(validator + .validate_token( + &"x".repeat(MAX_HOSTED_TOKEN_BYTES + 1), + binding.target_database, + now, + |_, _, _| Some(binding) + ) + .is_err()); + let claims = unverified_hosted_token_claims(&token).unwrap(); + let mut hs_header = Header::new(Algorithm::HS256); + hs_header.typ = Some(HOSTED_TOKEN_TYPE.into()); + let hs_token = jsonwebtoken::encode( + &hs_header, + &claims, + &jsonwebtoken::EncodingKey::from_secret(b"not-a-platform-key"), + ) + .unwrap(); + assert!(validator + .validate_token(&hs_token, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + } + + #[test] + fn broker_signing_obeys_confirmed_lease_and_lifetime() { + let (keys, _, binding, now) = fixture(); + for expiry in [now, now + Duration::from_secs(31)] { + assert!(sign_hosted_token(&keys.private, "platform.test", &binding, now, expiry, "id").is_err()); + } + let short_lease = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(10), + ..binding + }; + assert!(sign_hosted_token( + &keys.private, + "platform.test", + &short_lease, + now, + now + Duration::from_secs(11), + "id" + ) + .is_err()); + } + + #[tokio::test] + async fn ordinary_validation_rejects_reserved_platform_kinds_and_types() { + let (keys, _, binding, _) = fixture(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(30), + ..binding + }; + let token = mint(&keys, &binding, now); + let ordinary = FullTokenValidator { + local_key: keys.public.clone(), + local_issuer: "platform.test".into(), + oidc_validator: UnimplementedTokenValidator, + }; + for reserved in [ + token.clone(), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_hosted_future"); + header.typ = Some("JWT".into()); + }), + change(&token, &keys, |claims, header| { + claims.as_object_mut().unwrap().remove("kind"); + header.typ = Some("spacetimedb-hosted-v2+jwt".into()); + }), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_container_lease_v1"); + header.typ = Some("JWT".into()); + }), + change(&token, &keys, |claims, header| { + claims.as_object_mut().unwrap().remove("kind"); + header.typ = Some("spacetimedb-container-lease+jwt".into()); + }), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_container_registry_future"); + header.typ = Some("JWT".into()); + }), + ] { + assert!(spacetimedb_auth::hosted::has_reserved_platform_token_kind(&reserved).unwrap()); + assert!(keys.public.validate_token(&reserved).await.is_err()); + assert!(ordinary.validate_token(&reserved).await.is_err()); + } + } + + #[tokio::test] + async fn reserved_classification_preserves_ordinary_token_algorithms() { + let rsa = openssl::rsa::Rsa::generate(2048).unwrap(); + let rsa = openssl::pkey::PKey::from_rsa(rsa).unwrap(); + let ec = JwtKeys::generate().unwrap(); + let keys = [ + (Algorithm::ES256, ec.private, ec.public), + ( + Algorithm::RS256, + jsonwebtoken::EncodingKey::from_rsa_pem(&rsa.private_key_to_pem_pkcs8().unwrap()).unwrap(), + DecodingKey::from_rsa_pem(&rsa.public_key_to_pem().unwrap()).unwrap(), + ), + ( + Algorithm::HS256, + jsonwebtoken::EncodingKey::from_secret(b"ordinary-oidc-test-secret"), + DecodingKey::from_secret(b"ordinary-oidc-test-secret"), + ), + ]; + for (algorithm, private, public) in keys { + let claims = json!({ "iss": "ordinary.test", "sub": "a-user", "iat": 1_700_000_000_u64, "kind": "ordinary_application_kind" }); + let token = jsonwebtoken::encode(&Header::new(algorithm), &claims, &private).unwrap(); + assert!( + !has_reserved_hosted_token_kind(&token).unwrap(), + "misclassified {algorithm:?}" + ); + let validated = public.validate_token(&token).await.unwrap(); + assert_eq!(validated.identity, Identity::from_claims("ordinary.test", "a-user")); + } + } +} diff --git a/crates/core/src/auth/invocation.rs b/crates/core/src/auth/invocation.rs new file mode 100644 index 00000000000..dc9004effaa --- /dev/null +++ b/crates/core/src/auth/invocation.rs @@ -0,0 +1,127 @@ +//! Authority carried from authenticated admission to module execution. +//! +//! Identity equality never establishes internal authority. A hosted proof can +//! only originate in signature verification against trusted platform state. + +use super::hosted_tokens::VerifiedHostedAuth; +use spacetimedb_auth::identity::ConnectionAuthCtx; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_lib::Identity; +use spacetimedb_schema::def::ModuleDef; +use std::time::SystemTime; + +#[derive(Clone, Debug)] +pub struct InvocationCaller { + pub(crate) identity: Identity, + pub(crate) hosted: Option>, +} + +impl From for InvocationCaller { + fn from(identity: Identity) -> Self { + Self { identity, hosted: None } + } +} + +impl From<&ConnectionAuthCtx> for InvocationCaller { + fn from(auth: &ConnectionAuthCtx) -> Self { + Self { + identity: auth.claims.identity, + hosted: auth.hosted.clone().map(std::sync::Arc::new), + } + } +} + +/// SQL permissions and the authenticated container restrictions are independent. +/// Internal status never impersonates the database owner or grants SQL rights. +#[derive(Clone)] +pub struct SqlCallAuth { + permissions: spacetimedb_lib::identity::AuthCtx, + pub(crate) hosted: Option>, +} + +impl From for SqlCallAuth { + fn from(permissions: spacetimedb_lib::identity::AuthCtx) -> Self { + Self { + permissions, + hosted: None, + } + } +} + +impl std::ops::Deref for SqlCallAuth { + type Target = spacetimedb_lib::identity::AuthCtx; + fn deref(&self) -> &Self::Target { + &self.permissions + } +} + +impl SqlCallAuth { + pub fn authenticated( + permissions: spacetimedb_lib::identity::AuthCtx, + auth: &ConnectionAuthCtx, + ) -> anyhow::Result { + anyhow::ensure!( + permissions.caller() == auth.claims.identity, + "SQL caller does not match authenticated sender" + ); + if let Some(proof) = &auth.hosted { + anyhow::ensure!( + proof.source_database() == auth.claims.identity, + "SQL hosted proof does not match authenticated sender" + ); + } + Ok(Self { + permissions, + hosted: auth.hosted.clone().map(std::sync::Arc::new), + }) + } +} + +impl InvocationCaller { + pub(crate) fn flags_for(&self, target: Identity, module: &ModuleDef) -> anyhow::Result { + let Some(proof) = &self.hosted else { return Ok(0) }; + anyhow::ensure!( + proof.source_database() == self.identity, + "hosted caller does not match its proof" + ); + anyhow::ensure!( + proof.target_database() == target, + "hosted credential targets another database" + ); + anyhow::ensure!( + module.supports_hosted_auth_v1(), + "module does not support hosted authentication" + ); + proof.check_at(SystemTime::now())?; + Ok(u32::from(proof.is_internal())) + } +} + +/// Must run while holding the transaction that admits the database operation. +/// A check before queueing does not serialize with generation revocation. +pub(crate) fn check_hosted_admission( + state: &S, + database: &crate::db::relational_db::RelationalDB, + proof: Option<&VerifiedHostedAuth>, +) -> anyhow::Result<()> { + let Some(proof) = proof else { return Ok(()) }; + anyhow::ensure!( + database.hosted_admission().is_open(), + "receiving database has not reconciled hosted admission" + ); + anyhow::ensure!( + proof.target_database() == database.database_identity(), + "hosted credential targets another database" + ); + proof.check_at(SystemTime::now())?; + crate::db::deployment::check_container_fence( + state, + proof.source_database(), + proof.generation(), + proof.grant_revision(), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/auth/invocation/tests.rs b/crates/core/src/auth/invocation/tests.rs new file mode 100644 index 00000000000..0507ca32bdb --- /dev/null +++ b/crates/core/src/auth/invocation/tests.rs @@ -0,0 +1,199 @@ +use super::*; +use crate::auth::hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}; +use crate::auth::JwtKeys; +use crate::db::deployment::install_container_fence; +use crate::db::relational_db::tests_utils::TestDB; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::StContainerFenceRow; +use spacetimedb_lib::db::auth::StAccess; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; +use spacetimedb_lib::identity::AuthCtx; +use std::time::Duration; + +fn module(hosted_auth: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + if hosted_auth { + builder.add_capability("hosted_auth_v1"); + } + builder.finish().try_into().unwrap() +} + +/// Obtain every proof through the production signer and target-bound verifier. +fn authenticate(source: Identity, target: Identity, now: SystemTime) -> ConnectionAuthCtx { + let keys = JwtKeys::generate().unwrap(); + let binding = HostedTokenBinding { + source_database: source, + target_database: target, + generation: 3, + grant_revision: 7, + lease_expires_at: now + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "test.platform", + &binding, + now, + now + Duration::from_secs(20), + "invocation-test", + ) + .unwrap(); + HostedTokenValidator::new([("test.platform".into(), keys.public)]) + .unwrap() + .validate_token(&token, target, now, |issuer, requested_source, requested_target| { + (issuer == "test.platform" && requested_source == source && requested_target == target).then_some(binding) + }) + .unwrap() + .into_connection_auth() + .unwrap() +} + +fn fence(source: Identity, generation: u64, grant_revision: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: source.into(), + generation, + target_grant_revision: grant_revision, + target_set_hash: spacetimedb_lib::hash_bytes(b"configured targets"), + allowed, + } +} + +#[test] +fn internal_requires_verified_self_call_and_updated_bindings() { + let target = Identity::ONE; + let foreign = Identity::from_u256(2u64.into()); + let updated = module(true); + let old_bindings = module(false); + // This also covers an ordinary connection whose sender equals the database. + assert_eq!(InvocationCaller::from(target).flags_for(target, &updated).unwrap(), 0); + for (source, expected_flags) in [(target, 1), (foreign, 0)] { + let auth = authenticate(source, target, SystemTime::now()); + let caller = InvocationCaller::from(&auth); + assert_eq!(caller.flags_for(target, &updated).unwrap(), expected_flags); + assert!(caller.flags_for(target, &old_bindings).is_err()); + assert!(caller.flags_for(Identity::ZERO, &updated).is_err()); + } +} + +#[test] +fn authenticated_proof_cannot_be_paired_with_another_sender_or_sql_caller() { + let source = Identity::ONE; + let target = Identity::from_u256(2u64.into()); + let owner = Identity::from_u256(3u64.into()); + let mut auth = authenticate(source, target, SystemTime::now()); + assert!(SqlCallAuth::authenticated(AuthCtx::for_current(owner), &auth).is_err()); + // ConnectionAuthCtx has public fields for trusted host code. Defend the + // boundary against accidentally mixing separately authenticated contexts. + auth.claims.identity = owner; + assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); + assert!(SqlCallAuth::authenticated(AuthCtx::for_current(owner), &auth).is_err()); +} + +#[test] +fn internal_authentication_does_not_grant_owner_sql_permissions() { + let source = Identity::ONE; + let owner = Identity::from_u256(3u64.into()); + let auth = authenticate(source, source, SystemTime::now()); + assert_eq!( + InvocationCaller::from(&auth).flags_for(source, &module(true)).unwrap(), + 1 + ); + let sql = SqlCallAuth::authenticated(AuthCtx::new(owner, source), &auth).unwrap(); + assert_eq!(sql.caller(), source); + assert!(sql.has_read_access(StAccess::Public)); + assert!(!sql.has_read_access(StAccess::Private)); + assert!(!sql.has_write_access()); + assert!(!sql.bypass_rls()); +} + +#[test] +fn transaction_admission_rechecks_persisted_fences_after_initial_authentication() { + let db = TestDB::in_memory().unwrap(); + let target = db.database_identity(); + let source = Identity::ONE; + let auth = authenticate(source, target, SystemTime::now()); + let proof = auth.hosted.as_ref(); + let wrong_target = authenticate(source, Identity::from_u256(99u64.into()), SystemTime::now()); + db.hosted_admission().begin().unwrap().complete().unwrap(); + let caller = InvocationCaller::from(&auth); + assert_eq!(caller.flags_for(target, &module(true)).unwrap(), 0); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + assert!(check_hosted_admission(tx, &db, proof).is_err()); + install_container_fence(&db, tx, &fence(source, 3, 7, true))?; + assert!(!db.hosted_admission().is_open()); + db.hosted_admission().begin()?.complete()?; + check_hosted_admission(tx, &db, proof)?; + assert!(check_hosted_admission(tx, &db, wrong_target.hosted.as_ref()).is_err()); + Ok(()) + }) + .unwrap(); + // Revocation commits after token verification and before the queued call. + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + install_container_fence(&db, tx, &fence(source, 4, 8, false))?; + Ok(()) + }) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + assert!(check_hosted_admission(tx, &db, proof).is_err()); + check_hosted_admission(tx, &db, None)?; + // Another generation does not reactivate a copied credential. + install_container_fence(&db, tx, &fence(source, 5, 9, true))?; + assert!(check_hosted_admission(tx, &db, proof).is_err()); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn reopened_database_requires_a_fresh_sweep_despite_a_replayed_allowed_fence() { + let db = TestDB::durable().unwrap(); + let source = Identity::ONE; + let target = db.database_identity(); + let auth = authenticate(source, target, SystemTime::now()); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &fence(source, 3, 7, true)) + }) + .unwrap(); + let old_sweep = db.hosted_admission().begin().unwrap(); + let db = db.reopen().unwrap(); + // Shutdown seals the old object, including any late coordinator ticket. + assert!(old_sweep.complete().is_err()); + assert!(!db.hosted_admission().is_open()); + db.with_read_only(Workload::ForTests, |tx| { + crate::db::deployment::check_container_fence(tx, source, 3, 7).unwrap(); + let error = check_hosted_admission(tx, &db, auth.hosted.as_ref()).unwrap_err(); + assert!(error.to_string().contains("has not reconciled")); + check_hosted_admission(tx, &db, None).unwrap(); + }); + db.hosted_admission().begin().unwrap().complete().unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + check_hosted_admission(tx, &db, auth.hosted.as_ref()).unwrap(); + }); +} + +#[test] +fn expired_verified_proof_is_rejected_at_both_call_and_transaction_admission() { + let db = TestDB::in_memory().unwrap(); + let target = db.database_identity(); + let source = Identity::ONE; + db.hosted_admission().begin().unwrap().complete().unwrap(); + // Valid when received, expired before execution, without sleeps or forged proofs. + let auth = authenticate(source, target, SystemTime::now() - Duration::from_secs(60)); + assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + install_container_fence(&db, tx, &fence(source, 3, 7, true))?; + assert!(check_hosted_admission(tx, &db, auth.hosted.as_ref()).is_err()); + Ok(()) + }) + .unwrap(); +} + +#[tokio::test] +async fn shutdown_seals_hosted_admission_even_for_retained_memory_database_handles() { + let db = TestDB::in_memory().unwrap(); + let pending = db.hosted_admission().begin().unwrap(); + assert_eq!(db.shutdown().await, None); + assert!(pending.complete().is_err()); + assert!(!db.hosted_admission().is_open()); + assert!(db.hosted_admission().begin().is_err()); + assert_eq!(db.shutdown().await, None); +} diff --git a/crates/core/src/auth/mod.rs b/crates/core/src/auth/mod.rs index e1e38a667c4..49f2686f062 100644 --- a/crates/core/src/auth/mod.rs +++ b/crates/core/src/auth/mod.rs @@ -7,6 +7,8 @@ use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use crate::config::CertificateAuthority; pub use spacetimedb_auth::identity; +pub mod hosted_tokens; +pub mod invocation; pub mod token_validation; /// JWT verification and signing keys. diff --git a/crates/core/src/auth/token_validation.rs b/crates/core/src/auth/token_validation.rs index 22ebcef743e..0aacfeb0839 100644 --- a/crates/core/src/auth/token_validation.rs +++ b/crates/core/src/auth/token_validation.rs @@ -96,6 +96,7 @@ where T: TokenValidator + Send + Sync, { async fn validate_token(&self, token: &str) -> Result { + reject_reserved_hosted_credentials(token)?; let local_key_error = { let first_validator = BasicTokenValidator { public_key: self.local_key.clone(), @@ -144,6 +145,7 @@ lazy_static! { #[async_trait] impl TokenValidator for DecodingKey { async fn validate_token(&self, token: &str) -> Result { + reject_reserved_hosted_credentials(token)?; let mut validation = Validation::new(jsonwebtoken::Algorithm::ES256); validation.algorithms = match self.family() { AlgorithmFamily::Ec => vec![jsonwebtoken::Algorithm::ES256], @@ -269,6 +271,7 @@ pub struct OidcTokenValidator; // Get the issuer out of a token without validating the signature. fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { + reject_reserved_hosted_credentials(token)?; // We need the issuer before we know which key to use. // This intentionally does not validate the token. // Callers must only use it for key discovery and must verify the token afterwards. @@ -276,6 +279,16 @@ fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { Ok(data.claims.issuer) } +fn reject_reserved_hosted_credentials(token: &str) -> Result<(), TokenValidationError> { + if spacetimedb_auth::hosted::has_reserved_platform_token_kind(token)? { + return Err(anyhow::anyhow!( + "platform container credentials require their dedicated validator and cannot be exchanged" + ) + .into()); + } + Ok(()) +} + #[async_trait] impl TokenValidator for OidcTokenValidator { async fn validate_token(&self, token: &str) -> Result { diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 23e801e4b30..3aa66d91572 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -8,6 +8,7 @@ use std::task::{Context, Poll}; use std::time::{Instant, SystemTime}; use super::{message_handlers, ClientActorId, MessageHandleError, OutboundMessage}; +use crate::auth::{hosted_tokens::VerifiedHostedAuth, invocation::check_hosted_admission}; use crate::db::relational_db::RelationalDB; use crate::error::DBError; use crate::host::module_host::{ClientConnectedError, ProcedureResultTarget}; @@ -25,6 +26,7 @@ use prometheus::{Histogram, IntCounter, IntGauge}; use scopeguard::ScopeGuard; use spacetimedb_auth::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2}; +use spacetimedb_datastore::execution_context::Workload; use spacetimedb_durability::{DurableOffset, TxOffset}; use spacetimedb_lib::identity::{AuthCtx, RequestId}; use spacetimedb_lib::metrics::ExecutionMetrics; @@ -128,9 +130,29 @@ pub trait DurableOffsetSupply: Send { /// - `Ok(Some(DurableOffset))` otherwise /// fn durable_offset(&mut self) -> Result, NoSuchModule>; + + /// Recheck the authoritative target state, never a cached generation. + fn check_hosted_auth( + &mut self, + _proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + Box::pin(async { anyhow::bail!("hosted connection has no authoritative database state") }) + } } impl DurableOffsetSupply for watch::Receiver { + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + if self.has_changed().is_err() { + return Box::pin(async { Err(NoSuchModule.into()) }); + } + let module = self.borrow().clone(); + let mut db = module.relational_db().clone(); + db.check_hosted_auth(proof) + } + fn durable_offset(&mut self) -> Result, NoSuchModule> { let module = if self.has_changed().map_err(|_| NoSuchModule)? { self.borrow_and_update() @@ -143,6 +165,20 @@ impl DurableOffsetSupply for watch::Receiver { } impl DurableOffsetSupply for Arc { + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + let db = self.clone(); + let proof = proof.clone(); + Box::pin(async move { + tokio::task::spawn_blocking(move || { + db.with_read_only(Workload::Internal, |tx| check_hosted_admission(tx, &db, Some(&proof))) + }) + .await? + }) + } + fn durable_offset(&mut self) -> Result, NoSuchModule> { Ok(self.durable_tx_offset()) } @@ -160,6 +196,7 @@ pub struct ClientConnectionReceiver { channel: MeteredReceiver, pending: Vec, offset_supply: Box, + hosted_sender: Option>, } impl ClientConnectionReceiver { @@ -175,6 +212,7 @@ impl ClientConnectionReceiver { channel, pending: Vec::new(), offset_supply: Box::new(offset_supply), + hosted_sender: None, } } @@ -217,6 +255,9 @@ impl ClientConnectionReceiver { /// These values are stored internally, so calling `recv_many` again will /// not lose data. pub async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + if !self.hosted_connection_is_valid().await { + return 0; + } // If there are no pending updates and the input channel has been closed, // no more messages can be received from this receiver. if max == 0 || (self.pending.is_empty() && self.channel.recv_many(&mut self.pending, max).await == 0) { @@ -226,14 +267,14 @@ impl ClientConnectionReceiver { // If we don't have to wait for txns to be made durable, // drain the pending updates. if !self.confirmed_reads { - return self.drain_pending(buf, max); + return self.drain_pending(buf, max).await; } // If we do have to wait for txns to be made durable, // but the next client update doesn't have a tx offset, // there's no reason to wait - just send it. if !self.pending_update_has_offset() { - return self.drain_pending(buf, 1); + return self.drain_pending(buf, 1).await; } // Otherwise, grab the next offset that we should wait for. @@ -246,12 +287,12 @@ impl ClientConnectionReceiver { warn!("database went away while waiting for durable offset"); return 0; } - self.drain_pending(buf, n) + self.drain_pending(buf, n).await } // Database shut down or crashed. Err(NoSuchModule) => 0, // In-memory database. - Ok(None) => self.drain_pending(buf, max), + Ok(None) => self.drain_pending(buf, max).await, } } @@ -268,13 +309,40 @@ impl ClientConnectionReceiver { } /// Drain the pending [`ClientUpdate`]s, up to `max, into `buf`. - fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + async fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + // A queued update may predate revocation, and a confirmed-read wait may + // outlast the credential. Check again immediately before delivery. + if !self.hosted_connection_is_valid().await { + return 0; + } let n = self.pending.len().min(max); buf.reserve(n); buf.extend(self.pending.drain(..n).map(|u| u.message)); n } + async fn hosted_connection_is_valid(&mut self) -> bool { + let Some(sender) = &self.hosted_sender else { return true }; + let valid = match sender.upgrade() { + Some(sender) => { + let valid = match &sender.auth.hosted { + Some(proof) if !sender.is_cancelled() => self.offset_supply.check_hosted_auth(proof).await.is_ok(), + _ => false, + }; + if !valid { + sender.cancel_hosted_connection(); + } + valid + } + None => false, + }; + if !valid { + self.pending.clear(); + self.close(); + } + valid + } + /// Does the next pending update have a tx offset? /// /// Assumes that [`Self::pending`] is not empty. @@ -365,6 +433,13 @@ pub enum ClientSendError { } impl ClientConnectionSender { + /// The fence installer awaits the returned task's completion before ack. + pub(crate) fn cancel_hosted_connection(&self) -> AbortHandle { + self.cancelled.store(true, Ordering::Release); + self.abort_handle.abort(); + self.abort_handle.clone() + } + pub fn dummy_with_channel( id: ClientActorId, config: ClientConfig, @@ -434,6 +509,16 @@ impl ClientConnectionSender { } fn send(&self, message: ClientUpdate) -> Result<(), ClientSendError> { + // Do not acquire a database transaction here: broadcasts can already + // hold one. Durable fencing is checked at admission and delivery. + if self + .auth + .hosted + .as_ref() + .is_some_and(|proof| proof.check_at(SystemTime::now()).is_err()) + { + self.cancel_hosted_connection(); + } if self.cancelled.load(Relaxed) { return Err(ClientSendError::Cancelled); } @@ -486,6 +571,65 @@ impl ClientConnectionSender { } } +/// Runs independently of the socket actor, so blocked writes and idle sockets +/// cannot keep credentials alive. Expiry uses a monotonic deadline captured once. +fn spawn_hosted_connection_watchdog( + sender: std::sync::Weak, + mut supply: impl DurableOffsetSupply + 'static, + subscriptions: Option, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let actor = sender.upgrade().map(|connection| connection.abort_handle.clone()); + async { + let Some(connection) = sender.upgrade() else { return }; + let Some(proof) = connection.auth.hosted.clone() else { + return; + }; + let deadline = tokio::time::Instant::now() + proof.remaining_lifetime(SystemTime::now()); + drop(connection); + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + if let Some(connection) = sender.upgrade() { connection.cancel_hosted_connection(); } + return; + } + _ = interval.tick() => {} + } + let Some(connection) = sender.upgrade() else { return }; + if connection.abort_handle.is_finished() || connection.is_cancelled() { + return; + } + let checked = tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + connection.cancel_hosted_connection(); + return; + } + checked = supply.check_hosted_auth(&proof) => checked, + }; + if checked.is_err() { + connection.cancel_hosted_connection(); + return; + } + } + } + .await; + // Keep the registry entry until socket I/O has actually stopped. A + // concurrent target barrier must still find and await an aborted actor. + if let Some(actor) = actor { + while !actor.is_finished() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + if let Some(subscriptions) = subscriptions { + subscriptions.unregister_hosted_connection(&sender); + } + }) +} + #[derive(Clone)] #[non_exhaustive] pub struct ClientConnection { @@ -900,7 +1044,7 @@ impl ClientConnection { }) .abort_handle(); - let receiver = ClientConnectionReceiver::new( + let mut receiver = ClientConnectionReceiver::new( config.confirmed_reads, MeteredReceiver::with_gauge(sendrx, metrics.sendtx_queue_size.clone()), module_rx.clone(), @@ -915,6 +1059,18 @@ impl ClientConnection { cancelled: AtomicBool::new(false), metrics: Some(metrics), }); + if sender.auth.hosted.is_some() { + receiver.hosted_sender = Some(Arc::downgrade(&sender)); + if module.subscriptions().register_hosted_connection(&sender).is_err() { + sender.cancel_hosted_connection(); + } else { + spawn_hosted_connection_watchdog( + Arc::downgrade(&sender), + module_rx.clone(), + Some(module.subscriptions().clone()), + ); + } + } let this = Self { sender, replica_id, @@ -1031,7 +1187,7 @@ impl ClientConnection { self.module() .call_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), caller, Some(request_id), @@ -1052,7 +1208,7 @@ impl ClientConnection { ) -> Result { self.module() .call_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(self.sender()), Some(request_id), @@ -1078,7 +1234,7 @@ impl ClientConnection { self.module() .enqueue_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), caller, Some(request_id), @@ -1099,7 +1255,7 @@ impl ClientConnection { ) -> Result<(), ReducerCallError> { self.module() .enqueue_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(self.sender()), Some(request_id), @@ -1119,7 +1275,7 @@ impl ClientConnection { ) -> Result<(), BroadcastError> { self.module() .enqueue_procedure( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(timer), procedure, @@ -1139,7 +1295,7 @@ impl ClientConnection { ) -> Result<(), BroadcastError> { self.module() .enqueue_procedure( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(timer), procedure, @@ -1368,6 +1524,354 @@ mod tests { assert_matches!(futures::poll!(f), Poll::Pending); } + fn hosted_auth(db: &RelationalDB, lifetime: std::time::Duration) -> ConnectionAuthCtx { + // These fixtures model an already reconciled receiving host. Tests of + // startup closure explicitly close the gate after constructing it. + if !db.hosted_admission().is_open() { + db.hosted_admission().begin().unwrap().complete().unwrap(); + } + use crate::auth::{ + hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, + JwtKeys, + }; + let keys = JwtKeys::generate().unwrap(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + source_database: db.database_identity(), + target_database: db.database_identity(), + generation: 1, + grant_revision: 1, + lease_expires_at: now + std::time::Duration::from_secs(30), + }; + let token = sign_hosted_token(&keys.private, "platform.test", &binding, now, now + lifetime, "test").unwrap(); + HostedTokenValidator::new([("platform.test".into(), keys.public)]) + .unwrap() + .validate_token(&token, db.database_identity(), now, |_, _, _| Some(binding)) + .unwrap() + .into_connection_auth() + .unwrap() + } + + fn set_fence(db: &RelationalDB, generation: u64, allowed: bool) { + use crate::db::deployment::install_container_fence; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + db, + tx, + &StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: spacetimedb_lib::hash_bytes(b"targets"), + allowed, + }, + ) + }) + .unwrap(); + } + + fn hosted_client( + db: &RelationalDB, + supply: impl DurableOffsetSupply + 'static, + confirmed_reads: bool, + lifetime: std::time::Duration, + ) -> ( + Arc, + ClientConnectionReceiver, + tokio::task::JoinHandle<()>, + ) { + let (mut sender, mut receiver) = ClientConnectionSender::dummy_with_channel( + ClientActorId::for_test(db.database_identity()), + ClientConfig { + confirmed_reads, + ..ClientConfig::for_test() + }, + supply, + ); + sender.auth = hosted_auth(db, lifetime); + let actor = tokio::spawn(std::future::pending()); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + receiver.hosted_sender = Some(Arc::downgrade(&sender)); + (sender, receiver, actor) + } + + struct HostedConfirmedSupply { + db: Arc, + durable: FakeDurableOffset, + durability_requested: Arc, + } + impl DurableOffsetSupply for HostedConfirmedSupply { + fn durable_offset(&mut self) -> Result, NoSuchModule> { + self.durability_requested.store(true, Ordering::Release); + self.durable.durable_offset() + } + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + self.db.check_hosted_auth(proof) + } + } + + #[tokio::test] + async fn hosted_queued_delivery_rechecks_committed_fence_and_preserves_ordinary_identity() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, mut receiver, actor) = + hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + sender.send_message(None, empty_tx_update()).unwrap(); + set_fence(&db, 2, false); + assert_receiver_closed(receiver.recv()).await; + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + let (ordinary, mut ordinary_rx) = default_client(db.db.clone()); + ordinary.send_message(None, empty_tx_update()).unwrap(); + assert_received_update(ordinary_rx.recv()).await; + } + + #[tokio::test] + async fn hosted_queued_delivery_rejects_closed_startup_gate_with_unchanged_fence() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, mut receiver, actor) = + hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + sender.send_message(None, empty_tx_update()).unwrap(); + db.hosted_admission().close(); + assert_receiver_closed(receiver.recv()).await; + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + let (ordinary, mut ordinary_rx) = default_client(db.db.clone()); + ordinary.send_message(None, empty_tx_update()).unwrap(); + assert_received_update(ordinary_rx.recv()).await; + } + + #[tokio::test] + async fn hosted_confirmed_delivery_rechecks_fence_after_durability_wait() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let durable = FakeDurableOffset::new(); + let durability_requested = Arc::new(AtomicBool::new(false)); + let supply = HostedConfirmedSupply { + db: db.db.clone(), + durable: durable.clone(), + durability_requested: durability_requested.clone(), + }; + let (sender, mut receiver, actor) = hosted_client(&db, supply, true, std::time::Duration::from_secs(20)); + sender.send_message(Some(7), empty_tx_update()).unwrap(); + let mut receiving = Box::pin(receiver.recv()); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !durability_requested.load(Ordering::Acquire) { + assert_pending(&mut receiving).await; + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + set_fence(&db, 2, false); + durable.mark_durable_at(7); + assert_receiver_closed(receiving).await; + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_idle_connection_expires_without_outbound_traffic() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(2)); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), None); + tokio::time::timeout(std::time::Duration::from_secs(3), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_connection_uses_confirmed_expiry_while_authority_read_is_blocked() { + struct BlockedAuthority; + impl DurableOffsetSupply for BlockedAuthority { + fn durable_offset(&mut self) -> Result, NoSuchModule> { + Ok(None) + } + + fn check_hosted_auth( + &mut self, + _: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + Box::pin(std::future::pending()) + } + } + + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (mut sender, _receiver) = ClientConnectionSender::dummy_with_channel( + ClientActorId::for_test(db.database_identity()), + ClientConfig::for_test(), + db.db.clone(), + ); + let proof = hosted_auth(&db, std::time::Duration::from_secs(20)).hosted.unwrap(); + let confirmed_time = proof.expires_at() - std::time::Duration::from_secs(5); + // Authority is ahead of the receiving wall clock. Almost all of the + // five remaining seconds elapsed while its confirmation was in flight. + let started = std::time::Instant::now() - std::time::Duration::from_millis(4_900); + sender.auth = proof + .constrain_expiration(confirmed_time, started) + .unwrap() + .into_connection_auth() + .unwrap(); + let actor = tokio::spawn(std::future::pending::<()>()); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), BlockedAuthority, None); + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_idle_connection_rechecks_durable_revocation() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), None); + set_fence(&db, 2, false); + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_watchdog_ends_and_unregisters_after_socket_actor_finishes() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + subscriptions.register_hosted_connection(&sender).unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 1); + let watchdog = + spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), Some(subscriptions.clone())); + actor.abort(); + let _ = actor.await; + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 0); + // The registry releases the entry even while another owner retains sender. + assert!(!sender.is_cancelled()); + } + + #[tokio::test] + async fn hosted_registry_retains_cancelled_actor_until_delivery_cleanup_finishes() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (mut sender, _receiver) = default_client(db.db.clone()); + sender.auth = hosted_auth(&db, std::time::Duration::from_secs(20)); + let (release, blocked) = std::sync::mpsc::channel(); + let (started, ready) = oneshot::channel(); + // A started blocking task models cleanup that cannot complete merely + // because abort was requested. The barrier must retain its handle. + let actor = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + let _ = blocked.recv(); + }); + ready.await.unwrap(); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + subscriptions.register_hosted_connection(&sender).unwrap(); + let watchdog = + spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), Some(subscriptions.clone())); + set_fence(&db, 2, false); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !sender.is_cancelled() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 1); + let handles = db.with_read_only(Workload::ForTests, |tx| { + subscriptions.cancel_invalid_hosted_connections(tx) + }); + assert_eq!(handles.len(), 1); + assert!(!handles[0].is_finished()); + release.send(()).unwrap(); + actor.await.unwrap(); + watchdog.await.unwrap(); + assert!(handles[0].is_finished()); + assert_eq!(subscriptions.hosted_connection_count(), 0); + } + + #[tokio::test] + async fn hosted_target_barrier_cancels_connections_without_subscriptions_and_waits_for_actor() { + use crate::db::deployment::install_container_fence; + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + subscriptions.register_hosted_connection(&sender).unwrap(); + let handles = db + .with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + &db, + tx, + &StContainerFenceRow { + source_identity: db.database_identity().into(), + generation: 2, + target_grant_revision: 1, + target_set_hash: spacetimedb_lib::hash_bytes(b"targets"), + allowed: true, + }, + )?; + Ok::<_, anyhow::Error>(subscriptions.cancel_invalid_hosted_connections(tx)) + }) + .unwrap(); + assert_eq!(handles.len(), 1); + assert!(actor.await.unwrap_err().is_cancelled()); + assert!(handles[0].is_finished()); + assert!(subscriptions.register_hosted_connection(&sender).is_err()); + } + + #[tokio::test] + async fn hosted_subscription_rejected_under_transaction_before_query_compilation() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + // There is deliberately no installed fence. The malformed SQL verifies + // authentication fails before query parsing or view materialization. + let result = subscriptions + .add_legacy_subscriber( + None, + sender.clone(), + AuthCtx::new(db.database_identity(), db.database_identity()), + ws_v1::Subscribe { + query_strings: ["invalid SQL".into()].into(), + request_id: 0, + }, + Instant::now(), + None, + ) + .await; + assert!(result.unwrap_err().to_string().contains("container")); + sender.cancel_hosted_connection(); + let _ = actor.await; + } + fn default_client( offset_supply: impl DurableOffsetSupply + 'static, ) -> (ClientConnectionSender, ClientConnectionReceiver) { diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 8dc9ceb7273..9e5b314fcfe 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -175,6 +175,7 @@ impl<'de> serde::Deserialize<'de> for ConfigFile { v8: V8Config { procedure_instance_pool_size: config.v8.procedure_instance_pool_size, heap_policy: config.v8_heap_policy, + execution_timeout: config.v8.execution_timeout, }, }) } @@ -265,6 +266,9 @@ impl Default for WasmConfigToml { pub struct V8Config { pub procedure_instance_pool_size: NonZeroUsize, pub heap_policy: V8HeapPolicyConfig, + /// Wall-clock limit for one JavaScript startup, description or function call. + /// Must be positive and no greater than 120 seconds. + pub execution_timeout: Duration, } impl Default for V8Config { @@ -272,11 +276,20 @@ impl Default for V8Config { Self { procedure_instance_pool_size: default_v8_procedure_instance_pool_size(), heap_policy: V8HeapPolicyConfig::default(), + execution_timeout: default_v8_execution_timeout(), } } } impl V8Config { + pub fn validate_execution_timeout(&self) -> anyhow::Result<()> { + anyhow::ensure!( + !self.execution_timeout.is_zero() && self.execution_timeout <= default_v8_execution_timeout(), + "V8 execution timeout must be positive and no greater than 120 seconds" + ); + Ok(()) + } + pub fn normalized(mut self) -> Self { self.heap_policy = self.heap_policy.normalized(); self @@ -291,16 +304,35 @@ struct V8ConfigToml { deserialize_with = "de_nz_usize" )] pub procedure_instance_pool_size: NonZeroUsize, + #[serde( + default = "default_v8_execution_timeout", + deserialize_with = "de_v8_execution_timeout" + )] + pub execution_timeout: Duration, } impl Default for V8ConfigToml { fn default() -> Self { Self { procedure_instance_pool_size: default_v8_procedure_instance_pool_size(), + execution_timeout: default_v8_execution_timeout(), } } } +fn default_v8_execution_timeout() -> Duration { + Duration::from_secs(120) +} + +fn de_v8_execution_timeout<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let timeout = de_nz_duration(deserializer)? + .filter(|timeout| *timeout <= default_v8_execution_timeout()) + .ok_or_else(|| { + serde::de::Error::custom("V8 execution timeout must be positive and no greater than 120 seconds") + })?; + Ok(timeout) +} + #[derive(Clone, Copy, Debug, serde::Deserialize)] #[serde(rename_all = "kebab-case")] pub struct V8HeapPolicyConfig { @@ -576,6 +608,25 @@ mod tests { .unwrap_err(); } + #[test] + fn v8_execution_timeout_is_finite_and_bounded() { + let defaults: ConfigFile = toml::from_str("").unwrap(); + assert_eq!(defaults.v8.execution_timeout, Duration::from_secs(120)); + for value in ["0", "121", "\"0s\"", "\"121s\""] { + assert!(toml::from_str::(&format!("[v8]\nexecution-timeout = {value}")).is_err()); + } + let config: ConfigFile = toml::from_str("[v8]\nexecution-timeout = \"150ms\"").unwrap(); + assert_eq!(config.v8.execution_timeout, Duration::from_millis(150)); + for timeout in [Duration::ZERO, Duration::from_secs(121)] { + assert!(V8Config { + execution_timeout: timeout, + ..V8Config::default() + } + .validate_execution_timeout() + .is_err()); + } + } + #[test] fn v8_heap_policy_defaults_when_omitted() { let config: ConfigFile = toml::from_str("").unwrap(); diff --git a/crates/core/src/db/container_environment.rs b/crates/core/src/db/container_environment.rs new file mode 100644 index 00000000000..def8e8569ed --- /dev/null +++ b/crates/core/src/db/container_environment.rs @@ -0,0 +1,354 @@ +//! Transactional immutable container environment snapshots. +//! +//! These are host-only operations. Before invoking them, the adapter must +//! authenticate the assigned service and confirm the exact current control +//! intent and authoritative leader. Identity equality and a cached control row +//! are insufficient. Helpers retain the caller's transaction and do no IO. +//! +//! Historical restore must close admission and reconcile newer operational +//! fences before any snapshot request. A missing Ready snapshot is an error, +//! never permission to recapture values from a restored or current `st_env`. + +use super::{ + deployment, environment, + relational_db::{MutTx, RelationalDB}, +}; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + StContainerEnvironmentRow, StContainerFenceRow, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, +}; +use spacetimedb_lib::container::{validate_env_key, validate_exec_size, ContainerSpec, MAX_ENV_KEYS}; +use spacetimedb_lib::container_environment::{EnvironmentSnapshotReceipt, EnvironmentSnapshotScope}; +use spacetimedb_lib::{bsatn, SpacetimeType, Uuid}; +use spacetimedb_primitives::ColId; +use spacetimedb_sats::AlgebraicValue; +use std::{collections::BTreeMap, fmt}; + +pub const MAX_SNAPSHOT_BYTES: usize = 256 * 1024; +pub const MAX_RETAINED_SNAPSHOTS: u64 = 64; + +/// Error text and Debug never include selected values or serialized records. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum EnvironmentSnapshotError { + #[error("container environment request is invalid")] + InvalidScope, + #[error("container environment generation is fenced")] + Fenced, + #[error("container environment deployment revision does not match")] + RevisionConflict, + #[error("container environment snapshot belongs to another immutable scope")] + ScopeConflict, + #[error("container environment snapshot has not been captured")] + NotCaptured, + #[error("required container environment keys are missing: {0:?}")] + MissingKeys(Vec), + #[error("container environment does not satisfy startup limits")] + InvalidEnvironment, + #[error("container environment snapshot capacity exhausted")] + Capacity, + #[error("container environment snapshot metadata is invalid")] + CorruptMetadata, + #[error("container environment storage operation failed")] + Storage, + #[error("container environment requires durable storage")] + DurabilityUnavailable, + #[error("container environment durability could not be confirmed")] + DurabilityFailed, +} + +impl From for EnvironmentSnapshotError { + fn from(_: crate::error::DBError) -> Self { + Self::Storage + } +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +enum Record { + V1(RecordV1), +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct RecordV1 { + receipt: EnvironmentSnapshotReceipt, + selected_values: Vec, +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct CapturedValue { + key: String, + value: String, +} + +/// Selected database values only. The trusted adapter merges verified image +/// defaults and current platform variables, then validates the complete exec. +pub struct SecretEnvironment { + pub receipt: EnvironmentSnapshotReceipt, + pub selected_values: BTreeMap, +} + +impl fmt::Debug for SecretEnvironment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecretEnvironment") + .field("receipt", &self.receipt) + .field("selected_values", &"[redacted]") + .finish() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnvironmentClosedReceipt { + pub scope: EnvironmentSnapshotScope, + pub closed_through_generation: u64, +} + +fn valid_uuid(id: Uuid) -> bool { + matches!( + id.get_version(), + Some(spacetimedb_sats::uuid::Version::V4 | spacetimedb_sats::uuid::Version::V7) + ) +} + +fn validate_scope(db: &RelationalDB, scope: &EnvironmentSnapshotScope) -> Result<(), EnvironmentSnapshotError> { + if db.database_identity() != scope.database_identity + || scope.database_id == 0 + || scope.node_id == 0 + || scope.generation == 0 + || scope.cluster.is_empty() + || scope.cluster.len() > 256 + || scope.cluster.contains('\0') + || !valid_uuid(scope.node_incarnation) + || !valid_uuid(scope.start_request) + || !valid_uuid(scope.env_generation) + || scope.env_keys.len() > MAX_ENV_KEYS + || scope.env_keys.windows(2).any(|keys| keys[0] >= keys[1]) + || scope.env_keys.iter().any(|key| validate_env_key(key).is_err()) + { + return Err(EnvironmentSnapshotError::InvalidScope); + } + Ok(()) +} + +fn fence( + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result { + state + .iter_by_col_eq( + ST_CONTAINER_FENCE_ID, + ColId(0), + &AlgebraicValue::U256(scope.database_identity.to_u256().into()), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .map(StContainerFenceRow::try_from) + .transpose() + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::Fenced) +} + +fn admitted_spec( + db: &RelationalDB, + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result { + validate_scope(db, scope)?; + let current = fence(state, scope)?; + if current.generation != scope.generation || !current.allowed { + return Err(EnvironmentSnapshotError::Fenced); + } + let (revision, deployment) = deployment::current_deployment(state) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if revision != scope.deployment_revision { + return Err(EnvironmentSnapshotError::RevisionConflict); + } + let spec = deployment + .current() + .container + .as_ref() + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if spec.env_keys != scope.env_keys { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + Ok(spec.clone()) +} + +fn lookup( + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result, EnvironmentSnapshotError> { + let Some(row) = state + .iter_by_col_eq( + ST_CONTAINER_ENVIRONMENT_ID, + ColId(0), + &AlgebraicValue::U64(scope.generation), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .map(StContainerEnvironmentRow::try_from) + .transpose() + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + else { + return Ok(None); + }; + if row.payload.len() > MAX_SNAPSHOT_BYTES { + return Err(EnvironmentSnapshotError::CorruptMetadata); + } + let Record::V1(record) = bsatn::from_slice(&row.payload).map_err(|_| EnvironmentSnapshotError::CorruptMetadata)?; + if record.receipt.scope != *scope { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + if !valid_uuid(record.receipt.capture_receipt) + || !record + .selected_values + .iter() + .map(|entry| &entry.key) + .eq(scope.env_keys.iter()) + { + return Err(EnvironmentSnapshotError::CorruptMetadata); + } + Ok(Some(record)) +} + +fn validate_values(spec: &ContainerSpec, values: &BTreeMap) -> Result<(), EnvironmentSnapshotError> { + // The at-most-256 individually bounded values also bound this temporary allocation. + if values + .values() + .any(|value| value.contains('\0') || spacetimedb_lib::environment::validate_value(value).is_err()) + { + return Err(EnvironmentSnapshotError::InvalidEnvironment); + } + let env = values + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>(); + validate_exec_size(&spec.argv, &env).map_err(|_| EnvironmentSnapshotError::InvalidEnvironment) +} + +/// Capture exactly once under an open confirmed control intent. Returning this +/// receipt is not a durability acknowledgment; the host wrapper supplies that. +pub fn capture( + db: &RelationalDB, + tx: &mut MutTx, + scope: &EnvironmentSnapshotScope, +) -> Result { + let spec = admitted_spec(db, tx, scope)?; + if let Some(record) = lookup(tx, scope)? { + validate_values( + &spec, + &record + .selected_values + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect(), + )?; + return Ok(record.receipt); + } + if tx + .table_row_count(ST_CONTAINER_ENVIRONMENT_ID) + .ok_or(EnvironmentSnapshotError::Storage)? + >= MAX_RETAINED_SNAPSHOTS + { + return Err(EnvironmentSnapshotError::Capacity); + } + let mut values = BTreeMap::new(); + let mut missing = Vec::new(); + for key in &scope.env_keys { + match environment::get(tx, key).map_err(|_| EnvironmentSnapshotError::Storage)? { + Some(value) => { + values.insert(key.clone(), value); + } + None => missing.push(key.clone()), + } + } + if !missing.is_empty() { + return Err(EnvironmentSnapshotError::MissingKeys(missing)); + } + validate_values(&spec, &values)?; + let receipt = EnvironmentSnapshotReceipt { + scope: scope.clone(), + capture_receipt: Uuid::from_u128(uuid::Uuid::new_v4().as_u128()), + }; + let payload = bsatn::to_vec(&Record::V1(RecordV1 { + receipt: receipt.clone(), + selected_values: values + .into_iter() + .map(|(key, value)| CapturedValue { key, value }) + .collect(), + })) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)?; + if payload.len() > MAX_SNAPSHOT_BYTES { + return Err(EnvironmentSnapshotError::Capacity); + } + tx.insert_via_serialize_bsatn( + ST_CONTAINER_ENVIRONMENT_ID, + &StContainerEnvironmentRow { + generation: scope.generation, + payload: payload.into(), + }, + ) + .map_err(|_| EnvironmentSnapshotError::Storage)?; + Ok(receipt) +} + +/// Read only an existing receipt after fresh exact control/lease confirmation. +pub fn read( + db: &RelationalDB, + state: &impl StateView, + receipt: &EnvironmentSnapshotReceipt, +) -> Result { + let spec = admitted_spec(db, state, &receipt.scope)?; + let record = lookup(state, &receipt.scope)?.ok_or(EnvironmentSnapshotError::NotCaptured)?; + if record.receipt != *receipt { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + let selected_values = record + .selected_values + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect(); + validate_values(&spec, &selected_values)?; + Ok(SecretEnvironment { + receipt: record.receipt, + selected_values, + }) +} + +/// Delete only after a newer own-source fence irreversibly rejects every old +/// capture. The adapter also confirms positive historical control closure. +/// Missing rows remain idempotently closed; no per-UUID TTL is an authority. +pub fn close( + db: &RelationalDB, + tx: &mut MutTx, + scope: &EnvironmentSnapshotScope, + closed_through_generation: u64, +) -> Result { + validate_scope(db, scope)?; + let current = fence(tx, scope)?; + if closed_through_generation <= scope.generation || current.generation < closed_through_generation { + return Err(EnvironmentSnapshotError::Fenced); + } + if lookup(tx, scope)?.is_some() { + let pointer = tx + .iter_by_col_eq( + ST_CONTAINER_ENVIRONMENT_ID, + ColId(0), + &AlgebraicValue::U64(scope.generation), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .ok_or(EnvironmentSnapshotError::Storage)? + .pointer(); + db.delete(tx, ST_CONTAINER_ENVIRONMENT_ID, [pointer]); + } + Ok(EnvironmentClosedReceipt { + scope: scope.clone(), + closed_through_generation, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/db/container_environment/tests.rs b/crates/core/src/db/container_environment/tests.rs new file mode 100644 index 00000000000..2a1e207d6ce --- /dev/null +++ b/crates/core/src/db/container_environment/tests.rs @@ -0,0 +1,427 @@ +use super::*; +use crate::db::deployment::{ + install_container_fence, install_publication_fence, record_deployment_commit, DeploymentCommit, +}; +use crate::db::relational_db::tests_utils::TestDB; +use crate::host::container_environment as host; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_lib::container::*; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; +use spacetimedb_lib::{hash_bytes, Identity, Timestamp}; +use std::sync::{Arc, Barrier}; + +fn uuid() -> Uuid { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) +} + +fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: keys, + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } + .normalize(&Default::default()) + .unwrap(); + let request = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: db.owner_identity(), + expected_revision: None, + prepared_manifest_hash: hash_bytes(b"prepared"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: Some(spec.clone()), + }), + }; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + record_deployment_commit(tx, &request, Timestamp::now(), &Default::default())?; + install_container_fence(db, tx, &self_fence(db, 1, true))?; + for key in &spec.env_keys { + environment::set(db, tx, key, "before")?; + } + Ok(()) + }) + .unwrap(); + EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: 1, + database_identity: db.database_identity(), + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: request.deployment.revision().unwrap(), + start_request: request.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + } +} + +fn self_fence(db: &RelationalDB, generation: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed, + } +} + +fn tx( + db: &RelationalDB, + action: impl FnOnce(&mut MutTx) -> Result, +) -> Result { + db.with_auto_commit(Workload::ForTests, action) +} + +#[test] +fn container_environment_capture_retry_read_and_durable_reopen() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + let captured = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), scope.clone())) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "A", "changed")?; + environment::delete(&db, tx, "B")?; + Ok(()) + }) + .unwrap(); + let db = db.reopen().unwrap(); + let retried = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), scope)) + .unwrap(); + assert_eq!(retried.receipt, captured.receipt); + assert!(retried.durable_through >= captured.durable_through); + let values = db + .runtime() + .unwrap() + .block_on(host::read(db.db.clone(), retried.receipt)) + .unwrap() + .receipt; + assert_eq!( + values.selected_values, + BTreeMap::from([("A".into(), "before".into()), ("B".into(), "before".into())]) + ); + assert!(!format!("{values:?}").contains("before")); +} + +#[test] +fn container_environment_capture_is_atomic_against_concurrent_environment_mutation() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + let barrier = Arc::new(Barrier::new(2)); + let writer = { + let db = db.db.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "A", "after")?; + environment::set(&db, tx, "B", "after")?; + Ok(()) + }) + .unwrap(); + }) + }; + barrier.wait(); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + writer.join().unwrap(); + let values = db + .with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap(); + assert_eq!(values.selected_values["A"], values.selected_values["B"]); +} + +#[test] +fn container_environment_closure_fences_delayed_capture_and_new_instance_observes_new_values() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let old = setup(&db, vec!["A".into()]); + let captured = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), old.clone())) + .unwrap() + .receipt; + assert_eq!( + tx(&db, |tx| close(&db, tx, &old, 1)), + Err(EnvironmentSnapshotError::Fenced) + ); + assert_eq!( + tx(&db, |tx| close(&db, tx, &old, 2)), + Err(EnvironmentSnapshotError::Fenced) + ); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 2, true))?; + environment::set(&db, tx, "A", "new-boot")?; + Ok(()) + }) + .unwrap(); + db.runtime() + .unwrap() + .block_on(host::close(db.db.clone(), old.clone(), 2)) + .unwrap(); + let db = db.reopen().unwrap(); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &old)), + Err(EnvironmentSnapshotError::Fenced) + ); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &captured)), + Err(EnvironmentSnapshotError::Fenced) + )); + db.runtime() + .unwrap() + .block_on(host::close(db.db.clone(), old.clone(), 2)) + .unwrap(); + let new = EnvironmentSnapshotScope { + generation: 2, + env_generation: uuid(), + ..old + }; + let receipt = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), new)) + .unwrap() + .receipt; + assert_ne!(receipt.capture_receipt, captured.capture_receipt); + assert_eq!( + db.runtime() + .unwrap() + .block_on(host::read(db.db.clone(), receipt)) + .unwrap() + .receipt + .selected_values["A"], + "new-boot" + ); +} + +#[test] +fn container_environment_full_scope_conflicts_and_missing_ready_record_fail_closed() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let mut variants = Vec::new(); + let mut changed = scope.clone(); + changed.cluster = "other".into(); + variants.push(changed); + let mut changed = scope.clone(); + changed.database_id += 1; + variants.push(changed); + let mut changed = scope.clone(); + changed.node_id += 1; + variants.push(changed); + let mut changed = scope.clone(); + changed.node_incarnation = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.start_request = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.env_generation = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.env_keys.clear(); + variants.push(changed); + for changed in variants { + assert_eq!( + tx(&db, |tx| capture(&db, tx, &changed)), + Err(EnvironmentSnapshotError::ScopeConflict) + ); + } + let mut changed = scope.clone(); + changed.deployment_revision = hash_bytes(b"other"); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &changed)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + let mut fork = scope; + fork.database_identity = Identity::from_u256(99u64.into()); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &fork)), + Err(EnvironmentSnapshotError::InvalidScope) + ); + // Simulate a restored/lost Ready record. Resolve may not reinterpret its UUID. + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + tx.clear_table(ST_CONTAINER_ENVIRONMENT_ID)?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)), + Err(EnvironmentSnapshotError::NotCaptured) + )); +} + +#[test] +fn container_environment_missing_empty_invalid_and_capacity_are_atomic_and_redacted() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::delete(&db, tx, "B")?; + Ok(()) + }) + .unwrap(); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::MissingKeys(vec!["B".into()])) + ); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "B", "secret\0value")?; + Ok(()) + }) + .unwrap(); + let failure = tx(&db, |tx| capture(&db, tx, &scope)).unwrap_err(); + assert_eq!(failure, EnvironmentSnapshotError::InvalidEnvironment); + assert!(!format!("{failure:?}: {failure}").contains("secret")); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "B", "")?; + Ok(()) + }) + .unwrap(); + for generation in 1..=MAX_RETAINED_SNAPSHOTS { + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, generation, true))?; + Ok(()) + }) + .unwrap(); + let request = EnvironmentSnapshotScope { + generation, + env_generation: uuid(), + ..scope.clone() + }; + let receipt = tx(&db, |tx| capture(&db, tx, &request)).unwrap(); + assert_eq!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap() + .selected_values["B"], + "" + ); + } + let generation = MAX_RETAINED_SNAPSHOTS + 1; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, generation, true))?; + Ok(()) + }) + .unwrap(); + assert_eq!( + tx(&db, |tx| capture( + &db, + tx, + &EnvironmentSnapshotScope { generation, ..scope } + )), + Err(EnvironmentSnapshotError::Capacity) + ); +} + +#[test] +fn container_environment_history_is_hidden_from_privileged_sql_and_subscriptions() { + use crate::db::sql::ast::SchemaViewer; + use spacetimedb_expr::check::SchemaView; + use spacetimedb_lib::identity::AuthCtx; + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let auth = AuthCtx::for_current(db.owner_identity()); + db.with_read_only(Workload::ForTests, |state| { + let schema = SchemaViewer::new(state, &auth); + assert!(schema.schema_for_table(ST_CONTAINER_ENVIRONMENT_ID).is_none()); + assert!(schema.table_id("st_container_environment").is_none()); + for sql in [ + "SELECT * FROM st_container_environment", + "SELECT h.* FROM st_container_environment h JOIN st_env e ON h.generation = 1", + "DELETE FROM st_container_environment", + "UPDATE st_container_environment SET generation = 2", + ] { + assert!( + spacetimedb_query::compile_sql_stmt(sql, &schema, &auth).is_err(), + "{sql}" + ); + } + assert!(spacetimedb_query::compile_sql_stmt("SELECT * FROM st_env", &schema, &auth).is_ok()); + assert!(crate::subscription::query::compile_read_only_query( + &auth, + state, + "SELECT * FROM st_container_environment" + ) + .is_err()); + assert!(crate::subscription::subscription::get_all( + |db, tx| db.get_all_tables(tx).map(Vec::into_iter), + &db, + state, + &auth + ) + .unwrap() + .is_empty()); + }); +} + +#[test] +fn container_environment_host_api_requires_durability() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec![]); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + assert!(matches!( + rt.block_on(host::capture(db.db.clone(), scope)), + Err(EnvironmentSnapshotError::DurabilityUnavailable) + )); +} + +#[test] +fn container_environment_concurrent_closure_cannot_reopen_collected_generation() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + let barrier = Arc::new(Barrier::new(2)); + let capture_thread = { + let db = db.db.clone(); + let scope = scope.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + tx(&db, |tx| capture(&db, tx, &scope)) + }) + }; + barrier.wait(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 2, false))?; + close(&db, tx, &scope, 2)?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + capture_thread.join().unwrap(), + Ok(_) | Err(EnvironmentSnapshotError::Fenced) + )); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::Fenced) + ); + db.with_read_only(Workload::ForTests, |state| { + assert_eq!(state.table_row_count(ST_CONTAINER_ENVIRONMENT_ID), Some(0)) + }); +} diff --git a/crates/core/src/db/deployment.rs b/crates/core/src/db/deployment.rs new file mode 100644 index 00000000000..8ee66d385c8 --- /dev/null +++ b/crates/core/src/db/deployment.rs @@ -0,0 +1,586 @@ +//! Transactional deployment and hosted-client fences. +//! +//! Only authenticated host operations may call the mutation functions here. +//! Callers retain the same serializable transaction through module migration, +//! deployment recording, and commit. These functions do not contact control, +//! perform process IO, or turn an unverified client Identity into host authority. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + ConnectionIdViaU128, StConnectionAuthRow, StContainerFenceRow, StDeploymentOperationRow, StDeploymentRow, + StPublishFenceRow, ST_CONNECTION_AUTH_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, ST_DEPLOYMENT_OPERATION_ID, + ST_PUBLISH_FENCE_ID, +}; +use spacetimedb_lib::container::ContainerSpecLimits; +use spacetimedb_lib::deployment::{operation_expiry_ms, DeploymentSpec, DeploymentValidationError}; +use spacetimedb_lib::{bsatn, hash_bytes, ConnectionId, Hash, Identity, SpacetimeType, Timestamp, Uuid}; +use spacetimedb_primitives::{ColId, TableId}; +use spacetimedb_sats::AlgebraicValue; + +#[derive(Debug, thiserror::Error)] +pub enum DeploymentError { + #[error("this database requires the deployment publication protocol")] + CoordinatorRequired, + #[error("the supplied module does not match the prepared deployment")] + ProgramMismatch, + #[error("the prepared module must advertise hosted_auth_v1 to attach a container")] + UnsupportedHostedModule, + #[error("the publication coordinator no longer owns the database fence")] + PublicationFenced, + #[error("the expected deployment revision does not match the database")] + RevisionConflict, + #[error("operation ID is already bound to a different publication")] + OperationConflict, + #[error("container generation or grant is not authorized by this database")] + ContainerFenced, + #[error("a conflicting or older container fence cannot replace current authority")] + FenceConflict, + #[error("the receiving host fence revision is exhausted")] + FenceRevisionExhausted, + #[error("deployment metadata is inconsistent")] + CorruptMetadata, + #[error(transparent)] + Validation(#[from] DeploymentValidationError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Prepared by the authorized coordinator after recording durable intent. +/// The publisher is its verified original caller, never a guest-selected claim. +#[derive(Clone, Debug)] +pub struct DeploymentCommit { + pub operation_id: Uuid, + pub publication_epoch: u64, + pub publisher: Identity, + pub expected_revision: Option, + pub prepared_manifest_hash: Hash, + pub deployment: DeploymentSpec, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishResult { + #[serde(with = "spacetimedb_lib::deployment::uuid_json")] + pub operation_id: Uuid, + pub previous_revision: Option, + pub revision: Hash, +} + +#[derive(Clone, Debug, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +enum CommitReceipt { + V1(CommitReceiptV1), +} + +#[derive(Clone, Debug, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct CommitReceiptV1 { + request_hash: Hash, + publisher: Identity, + result: PublishResult, +} + +#[derive(Clone, Debug)] +pub enum CommitAdmission { + /// Return this result without repeating module init/update or any effects. + AlreadyCommitted(PublishResult), + Ready, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AbortResult { + /// The commit won the race. Recovery must converge on this deployment. + AlreadyCommitted(PublishResult), + /// This epoch can no longer admit a commit, and the prior revision remains. + Aborted { previous_revision: Option }, +} + +/// Check the actual program selected by the host, not a caller-provided +/// capability bit. The program bytes are hashed here because `Program` also +/// has a public constructor which accepts a previously computed hash. +pub fn validate_deployment_program( + request: &DeploymentCommit, + program: &spacetimedb_datastore::traits::Program, + module: &spacetimedb_schema::def::ModuleDef, +) -> Result<(), DeploymentError> { + use spacetimedb_datastore::system_tables::ModuleKind; + use spacetimedb_lib::deployment::{ModuleComponent, UserModuleKind}; + if hash_bytes(&program.bytes) != program.hash { + return Err(DeploymentError::ProgramMismatch); + } + match &request.deployment.current().module { + ModuleComponent::User(expected) + if expected.program_hash == program.hash + && matches!( + (expected.kind, program.kind), + (UserModuleKind::Wasm, ModuleKind::WASM) | (UserModuleKind::Js, ModuleKind::JS) + ) => {} + ModuleComponent::SystemEmpty(version) if crate::host::empty_module::matches_program(*version, program) => {} + _ => return Err(DeploymentError::ProgramMismatch), + } + if request.deployment.current().container.is_some() && !module.supports_hosted_auth_v1() { + return Err(DeploymentError::UnsupportedHostedModule); + } + Ok(()) +} + +/// Legacy raw module publication must be routed through the coordinator once +/// a database has deployment metadata or a publication fence, including after +/// an attempt aborts or its container is removed. In particular a legacy +/// request cannot race the first prepared container publication. +/// Call while holding the transaction that changes the program/schema. +pub fn require_unmanaged_publication(tx: &MutTx) -> Result<(), DeploymentError> { + if current_deployment(tx)?.is_some() || singleton(tx, ST_PUBLISH_FENCE_ID)?.is_some() { + return Err(DeploymentError::CoordinatorRequired); + } + Ok(()) +} + +/// Foreign callers depend on the receiving module's bindings just as self +/// callers do. A module replacement cannot silently erase that capability +/// while an admitted generation can still address the database. +pub fn validate_active_hosted_grants( + tx: &MutTx, + module: &spacetimedb_schema::def::ModuleDef, +) -> Result<(), DeploymentError> { + if !module.supports_hosted_auth_v1() { + for row in tx.iter(ST_CONTAINER_FENCE_ID)? { + if StContainerFenceRow::try_from(row)?.allowed { + return Err(DeploymentError::UnsupportedHostedModule); + } + } + } + Ok(()) +} + +fn singleton( + state: &S, + table: TableId, +) -> Result>, DeploymentError> { + Ok(state.iter_by_col_eq(table, ColId(0), &AlgebraicValue::U8(0))?.next()) +} + +pub fn current_deployment(state: &S) -> Result, DeploymentError> { + let Some(row) = singleton(state, ST_DEPLOYMENT_ID)? else { + return Ok(None); + }; + let row = StDeploymentRow::try_from(row)?; + let spec = DeploymentSpec::decode(&row.payload)?; + if spec.revision()? != row.revision { + return Err(DeploymentError::CorruptMetadata); + } + Ok(Some((row.revision, spec))) +} + +/// Monotonic compare-and-set, serialized against user-database commits. +/// Advancing this epoch does not itself quiesce a container or authorize launch. +pub fn install_publication_fence( + tx: &mut MutTx, + publication_epoch: u64, + operation_id: Uuid, +) -> Result<(), DeploymentError> { + if publication_epoch == 0 || operation_id == Uuid::NIL { + return Err(DeploymentError::PublicationFenced); + } + let next = StPublishFenceRow { + key: 0, + publication_epoch, + operation_id: operation_id.as_u128(), + }; + if let Some(current) = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()? + { + if current == next { + return Ok(()); + } + if current.publication_epoch >= publication_epoch { + return Err(DeploymentError::PublicationFenced); + } + } + tx.clear_table(ST_PUBLISH_FENCE_ID)?; + tx.insert_via_serialize_bsatn(ST_PUBLISH_FENCE_ID, &next)?; + Ok(()) +} + +/// Close an admitted publication before reporting an abort to control. Run in +/// one serializable transaction and await its durability before releasing the +/// control operation or resuming the previous container under a fresh generation. +/// A delayed commit and this transaction serialize on the same database fence. +/// +/// The nil operation ID is a closed-epoch marker, never a publish operation. +/// Keeping the epoch makes closure irreversible at that epoch while allowing +/// the next control-allocated epoch to install its own operation normally. +pub fn abort_deployment_commit(tx: &mut MutTx, request: &DeploymentCommit) -> Result { + if let Some(result) = committed_deployment_operation(tx, request)? { + return Ok(AbortResult::AlreadyCommitted(result)); + } + // Recovery must be able to close an expired operation too. Its expiry + // prevents new commits, but cannot substitute for a durable abort fence. + // Control owns the immutable epoch-to-operation mapping; authenticate and + // resolve that recorded operation before calling this function. + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + let fence = fence + .filter(|row| { + row.publication_epoch == request.publication_epoch + && (row.operation_id == request.operation_id.as_u128() || row.operation_id == 0) + }) + .ok_or(DeploymentError::PublicationFenced)?; + if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + if fence.operation_id == 0 { + return Ok(AbortResult::Aborted { + previous_revision: request.expected_revision, + }); + } + tx.clear_table(ST_PUBLISH_FENCE_ID)?; + tx.insert_via_serialize_bsatn( + ST_PUBLISH_FENCE_ID, + &StPublishFenceRow { + key: 0, + publication_epoch: request.publication_epoch, + operation_id: 0, + }, + )?; + Ok(AbortResult::Aborted { + previous_revision: request.expected_revision, + }) +} + +fn normalized_request( + request: &DeploymentCommit, + limits: &ContainerSpecLimits, +) -> Result<(DeploymentSpec, Hash, Hash), DeploymentError> { + let spec = request.deployment.clone().normalize(limits)?; + if spec != request.deployment { + return Err(DeploymentValidationError::InvalidEncoding.into()); + } + let (revision, request_hash) = request_identity(request)?; + Ok((spec, revision, request_hash)) +} + +fn request_identity(request: &DeploymentCommit) -> Result<(Hash, Hash), DeploymentError> { + if request.operation_id.get_version() != Some(spacetimedb_sats::uuid::Version::V7) { + return Err(DeploymentValidationError::InvalidOperationId.into()); + } + if request.publication_epoch == 0 { + return Err(DeploymentError::PublicationFenced); + } + // These bytes were normalized at admission. Do not re-apply today's + // resource eligibility when inspecting yesterday's committed outcome. + let revision = request.deployment.revision()?; + let mut bytes = b"spacetimedb/deployment-operation\0".to_vec(); + bytes.extend_from_slice( + &bsatn::to_vec(&( + request.operation_id, + request.publication_epoch, + request.publisher, + request.expected_revision, + request.prepared_manifest_hash, + revision, + )) + .map_err(|_| DeploymentError::CorruptMetadata)?, + ); + Ok((revision, hash_bytes(bytes))) +} + +/// Call before module execution, while holding the transaction later used for +/// migration. A cache or pre-enqueue check cannot replace this admission check. +pub fn check_deployment_commit( + tx: &MutTx, + request: &DeploymentCommit, + now: Timestamp, + limits: &ContainerSpecLimits, +) -> Result { + let now_ms = u64::try_from(now.to_micros_since_unix_epoch()).map_err(|_| DeploymentError::CorruptMetadata)? / 1000; + operation_expiry_ms(request.operation_id, now_ms)?; + if let Some(result) = committed_deployment_operation(tx, request)? { + return Ok(CommitAdmission::AlreadyCommitted(result)); + } + normalized_request(request, limits)?; + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + if !fence.is_some_and(|f| { + f.publication_epoch == request.publication_epoch && f.operation_id == request.operation_id.as_u128() + }) { + return Err(DeploymentError::PublicationFenced); + } + if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + Ok(CommitAdmission::Ready) +} + +/// Inspect the exact retained commit receipt during host recovery. Unlike +/// admitting a client retry, inspecting an existing outcome does not expire. +/// This never authorizes module execution. An absent receipt is not proof of +/// abort: close the epoch atomically before reporting an abort to control. +/// Retain active operations' receipts until their control recovery completes. +pub fn committed_deployment_operation( + state: &S, + request: &DeploymentCommit, +) -> Result, DeploymentError> { + let (revision, request_hash) = request_identity(request)?; + let operation_key = AlgebraicValue::U128(request.operation_id.as_u128().into()); + if let Some(row) = state + .iter_by_col_eq(ST_DEPLOYMENT_OPERATION_ID, ColId(0), &operation_key)? + .next() + { + let row = StDeploymentOperationRow::try_from(row)?; + let CommitReceipt::V1(receipt) = + bsatn::from_slice(&row.commit_result).map_err(|_| DeploymentError::CorruptMetadata)?; + if receipt.request_hash != request_hash || receipt.publisher != request.publisher { + return Err(DeploymentError::OperationConflict); + } + if receipt.result.revision != revision + || row.committed_revision != revision + || row.previous_revision != receipt.result.previous_revision + || receipt.result.operation_id != request.operation_id + { + return Err(DeploymentError::CorruptMetadata); + } + return Ok(Some(receipt.result)); + } + Ok(None) +} + +/// Recognize the durable closed marker when recovering a control operation +/// whose abort report was lost. The authenticated caller must resolve control's +/// immutable epoch-to-operation binding before using this host-only API. +pub fn deployment_publication_aborted( + state: &S, + request: &DeploymentCommit, +) -> Result { + request_identity(request)?; + let fence = singleton(state, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + if !fence.is_some_and(|row| row.publication_epoch == request.publication_epoch && row.operation_id == 0) { + return Ok(false); + } + if current_deployment(state)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + Ok(true) +} + +/// Record after successful module initialization/migration in that same +/// transaction. An error must roll back the entire transaction, including the +/// module changes. The caller waits for durability before reporting acceptance. +pub fn record_deployment_commit( + tx: &mut MutTx, + request: &DeploymentCommit, + now: Timestamp, + limits: &ContainerSpecLimits, +) -> Result { + if let CommitAdmission::AlreadyCommitted(result) = check_deployment_commit(tx, request, now, limits)? { + return Ok(result); + } + let (spec, revision, request_hash) = normalized_request(request, limits)?; + let result = PublishResult { + operation_id: request.operation_id, + previous_revision: request.expected_revision, + revision, + }; + let receipt = CommitReceipt::V1(CommitReceiptV1 { + request_hash, + publisher: request.publisher, + result: result.clone(), + }); + let receipt = bsatn::to_vec(&receipt) + .map_err(|_| DeploymentError::CorruptMetadata)? + .into_boxed_slice(); + let expires_ms = operation_expiry_ms(request.operation_id, (now.to_micros_since_unix_epoch() / 1000) as u64)?; + let expires_us = i64::try_from(expires_ms * 1000).map_err(|_| DeploymentError::CorruptMetadata)?; + let row = StDeploymentRow { + key: 0, + revision, + last_operation_id: request.operation_id.as_u128(), + payload: spec.encode()?, + }; + tx.clear_table(ST_DEPLOYMENT_ID)?; + tx.insert_via_serialize_bsatn(ST_DEPLOYMENT_ID, &row)?; + tx.insert_via_serialize_bsatn( + ST_DEPLOYMENT_OPERATION_ID, + &StDeploymentOperationRow { + operation_id: request.operation_id.as_u128(), + previous_revision: request.expected_revision, + committed_revision: revision, + commit_result: receipt, + expires_at: Timestamp::from_micros_since_unix_epoch(expires_us).into(), + }, + )?; + Ok(result) +} + +/// Install the frozen generation/grant tuple. Changes at the same generation +/// cannot reopen a revoked grant, even if a stale coordinator changes only the +/// target-set hash. Control must allocate a new generation for every barrier. +pub fn install_container_fence( + db: &RelationalDB, + tx: &mut MutTx, + next: &StContainerFenceRow, +) -> Result<(), DeploymentError> { + if next.generation == 0 { + return Err(DeploymentError::FenceConflict); + } + let key: AlgebraicValue = next.source_identity.into(); + let previous = tx + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(|row| StContainerFenceRow::try_from(row).map(|value| (row.pointer(), value))) + .transpose()?; + if let Some((pointer, previous)) = previous { + if previous == *next { + return Ok(()); + } + if next.generation <= previous.generation || next.target_grant_revision < previous.target_grant_revision { + return Err(DeploymentError::FenceConflict); + } + db.hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + } else { + db.hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + } + tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, next)?; + Ok(()) +} + +/// An exact denial of a previously observed allowed fence. Revision counters +/// belong to this live database open, not to the durable generation namespace. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FenceDenial { + pub row: StContainerFenceRow, + pub revision_before: u64, + pub revision_after: u64, +} + +/// Deny an orphan without lowering or inventing a control generation. +/// +/// The trusted coordinator must establish the source's absence from current +/// control inventory, then recheck that inventory revision while holding this +/// same serializable transaction. This function does not confer authority to +/// any external caller. Only the exact allowed tuple, or its already-denied +/// form, is accepted. Ordinary installation cannot reopen this generation. +pub fn deny_container_fence( + db: &RelationalDB, + tx: &mut MutTx, + expected: &StContainerFenceRow, +) -> Result { + if !expected.allowed || expected.generation == 0 { + return Err(DeploymentError::FenceConflict); + } + let key: AlgebraicValue = expected.source_identity.into(); + let (pointer, current) = tx + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(|row| StContainerFenceRow::try_from(row).map(|value| (row.pointer(), value))) + .transpose()? + .ok_or(DeploymentError::FenceConflict)?; + let denied = StContainerFenceRow { + allowed: false, + ..expected.clone() + }; + let (revision_before, revision_after) = if current == denied { + let revision = db.hosted_admission().fence_revision(); + (revision, revision) + } else { + if current != *expected { + return Err(DeploymentError::FenceConflict); + } + // The gate is closed before the row changes. A rollback still + // conservatively invalidates any concurrently progressing page scan. + let revisions = db + .hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, &denied)?; + revisions + }; + Ok(FenceDenial { + row: denied, + revision_before, + revision_after, + }) +} + +/// Called with verified hosted credentials inside every admitted transaction, +/// including each later transaction of a procedure. Signature, audience, expiry, +/// capability, and interface checks are additional receiving-host requirements. +pub fn check_container_fence( + state: &S, + source: Identity, + generation: u64, + target_grant_revision: u64, +) -> Result<(), DeploymentError> { + let key = AlgebraicValue::U256(source.to_u256().into()); + let fence = state + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(StContainerFenceRow::try_from) + .transpose()?; + if !fence + .is_some_and(|f| f.allowed && f.generation == generation && f.target_grant_revision == target_grant_revision) + { + return Err(DeploymentError::ContainerFenced); + } + Ok(()) +} + +/// Capture validated hosted connection authority in the transaction inserting st_client. +/// This is host-only metadata; ordinary connections leave no row and retain flags zero. +pub(crate) fn record_connection_auth( + tx: &mut MutTx, + connection_id: ConnectionId, + sender: Identity, + call_auth_flags: u32, +) -> Result<(), DeploymentError> { + tx.insert_via_serialize_bsatn( + ST_CONNECTION_AUTH_ID, + &StConnectionAuthRow { + connection_id: connection_id.into(), + sender_identity: sender.into(), + call_auth_flags, + }, + )?; + Ok(()) +} + +/// Recover captured authority for a host-dispatched disconnect event. It is not +/// a new container admission and does not require a still-valid credential/lease. +pub(crate) fn connection_auth_flags( + state: &S, + connection_id: ConnectionId, + sender: Identity, +) -> Result { + let key: AlgebraicValue = ConnectionIdViaU128::from(connection_id).into(); + let row = state + .iter_by_col_eq(ST_CONNECTION_AUTH_ID, ColId(0), &key)? + .next() + .map(StConnectionAuthRow::try_from) + .transpose()?; + let Some(row) = row else { return Ok(0) }; + if row.sender_identity.0 != sender { + return Err(DeploymentError::CorruptMetadata); + } + Ok(row.call_auth_flags) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/db/deployment/tests.rs b/crates/core/src/db/deployment/tests.rs new file mode 100644 index 00000000000..c027d017136 --- /dev/null +++ b/crates/core/src/db/deployment/tests.rs @@ -0,0 +1,633 @@ +use super::*; +use crate::db::relational_db::tests_utils::{begin_mut_tx, TestDB}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::{ + StEnvRow, ST_CLIENT_ID, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, ST_ENV_ID, +}; +use spacetimedb_durability::Durability; +use spacetimedb_lib::deployment::{ + DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind, PUBLISH_RETRY_WINDOW_MS, +}; + +fn request(sequence: u64, previous: Option) -> DeploymentCommit { + DeploymentCommit { + operation_id: Uuid::from_u128(0x01991ec4000070008000000000000000 | u128::from(sequence)), + publication_epoch: sequence, + publisher: Identity::from_u256(55u64.into()), + expected_revision: previous, + prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(sequence.to_le_bytes()), + }), + container: None, + }), + } +} + +fn now() -> Timestamp { + Timestamp::from_micros_since_unix_epoch(((request(1, None).operation_id.as_u128() >> 80) as i64) * 1000) +} + +fn transact( + db: &RelationalDB, + f: impl FnOnce(&mut MutTx) -> Result, +) -> Result { + db.with_auto_commit(Workload::ForTests, f) +} + +#[test] +fn deployment_retry_returns_original_result_after_later_publish_without_mutation() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let accepted = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let second = request(2, Some(accepted.revision)); + let later = transact(&db, |tx| { + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!(check_deployment_commit(tx, &first, now(), &limits)?, CommitAdmission::AlreadyCommitted(ref r) if r == &accepted)); + assert_eq!(record_deployment_commit(tx, &first, now(), &limits)?, accepted); + assert_eq!(current_deployment(tx)?.unwrap().0, later.revision); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(2)); + Ok(()) + }).unwrap(); +} + +#[test] +fn deployment_fence_and_revision_conflicts_fail_before_module_execution() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let mut second = request(2, Some(hash_bytes(b"not current"))); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, second.publication_epoch, second.operation_id) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits), + Err(DeploymentError::RevisionConflict) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + second.expected_revision = None; + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn deployment_and_module_effects_roll_back_together() { + let db = TestDB::in_memory().unwrap(); + let request = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); + let failed: Result<(), DeploymentError> = transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + // A second persistent table stands in for migration effects in the same + // transaction. Integration must additionally execute Wasm/JS migrations. + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: "MIGRATED".into(), + value: "yes".into(), + }, + )?; + record_deployment_commit(tx, &request, now(), &limits)?; + Err(DeploymentError::Database(DBError::Other(anyhow::anyhow!( + "injected failure before commit" + )))) + }); + assert!(failed.is_err()); + transact(&db, |tx| { + assert!(current_deployment(tx)?.is_none()); + assert_eq!(tx.table_row_count(ST_ENV_ID), Some(0)); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(0)); + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_closes_delayed_commits_and_cannot_reopen_its_epoch() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id) + }) + .unwrap(); + let aborted = AbortResult::Aborted { + previous_revision: None, + }; + assert_eq!( + transact(&db, |tx| abort_deployment_commit(tx, &first)).unwrap(), + aborted + ); + transact(&db, |tx| { + assert_eq!(abort_deployment_commit(tx, &first)?, aborted); + assert!(matches!( + record_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch + 1, Uuid::NIL), + Err(DeploymentError::PublicationFenced) + )); + assert!(current_deployment(tx)?.is_none()); + let second = request(2, None); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + let result = record_deployment_commit(tx, &second, now(), &limits)?; + assert!(matches!( + abort_deployment_commit(tx, &first), + Err(DeploymentError::PublicationFenced) + )); + assert_eq!(current_deployment(tx)?.unwrap().0, result.revision); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_commit_winning_abort_race_is_never_reported_as_aborted() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let second = request(2, Some(committed.revision)); + transact(&db, |tx| { + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed.clone()) + ); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits)?; + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed) + ); + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits)?, + CommitAdmission::AlreadyCommitted(_) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_rollback_does_not_report_a_closed_fence() { + let db = TestDB::in_memory().unwrap(); + let request = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); + let failed: Result<(), DeploymentError> = transact(&db, |tx| { + abort_deployment_commit(tx, &request)?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(failed.is_err()); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn recovery_of_expired_publication_preserves_commits_and_closes_uncommitted_epochs() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let expired = Timestamp::from_micros_since_unix_epoch( + now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS * 1000) as i64, + ); + let second = request(2, Some(committed.revision)); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &first, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed.clone()) + ); + let mut wrong_publisher = first.clone(); + wrong_publisher.publisher = Identity::ZERO; + assert!(matches!( + committed_deployment_operation(tx, &wrong_publisher), + Err(DeploymentError::OperationConflict) + )); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + assert!(matches!( + check_deployment_commit(tx, &second, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + assert_eq!( + abort_deployment_commit(tx, &second)?, + AbortResult::Aborted { + previous_revision: Some(committed.revision) + } + ); + // Clock rollback cannot make a delayed old commit admissible again. + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_and_commit_receipts_survive_commitlog_replay() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + abort_deployment_commit(tx, &first) + }) + .unwrap(); + let db = db.reopen().unwrap(); + let second = request(2, None); + let committed = transact(&db, |tx| { + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::Aborted { + previous_revision: None + } + ); + assert!(matches!( + record_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits) + }) + .unwrap(); + let db = db.reopen().unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!( + committed_deployment_operation(tx, &second).unwrap(), + Some(committed.clone()) + ); + assert_eq!(current_deployment(tx).unwrap().unwrap().0, committed.revision); + }); + assert_eq!( + transact(&db, |tx| abort_deployment_commit(tx, &second)).unwrap(), + AbortResult::AlreadyCommitted(committed) + ); +} + +#[test] +fn publication_recovery_does_not_reapply_tightened_resource_admission() { + use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, + }; + let db = TestDB::in_memory().unwrap(); + let mut first = request(1, None); + let DeploymentSpec::V1(spec) = &mut first.deployment; + spec.container = Some(ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 10_000, + }); + let original_limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &original_limits) + }) + .unwrap(); + let mut tightened = original_limits; + tightened.resources.cpu_millicores = 500; + let mut second = request(2, Some(committed.revision)); + second.deployment = first.deployment.clone(); + transact(&db, |tx| { + assert!(first.deployment.clone().normalize(&tightened).is_err()); + assert!(matches!(check_deployment_commit(tx, &first, now(), &tightened)?, CommitAdmission::AlreadyCommitted(ref result) if result == &committed)); + assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!(abort_deployment_commit(tx, &first)?, AbortResult::AlreadyCommitted(committed.clone())); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + assert!(check_deployment_commit(tx, &second, now(), &tightened).is_err()); + assert_eq!(abort_deployment_commit(tx, &second)?, AbortResult::Aborted { previous_revision: Some(committed.revision) }); + assert!(deployment_publication_aborted(tx, &second)?); + Ok(()) + }).unwrap(); +} + +#[test] +fn deployment_operation_cannot_be_reused_by_another_publisher_or_changed_request() { + let db = TestDB::in_memory().unwrap(); + let original = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, original.publication_epoch, original.operation_id)?; + record_deployment_commit(tx, &original, now(), &limits) + }) + .unwrap(); + transact(&db, |tx| { + let mut changed = original.clone(); + changed.publisher = Identity::from_u256(99u64.into()); + assert!(matches!( + check_deployment_commit(tx, &changed, now(), &limits), + Err(DeploymentError::OperationConflict) + )); + changed = original.clone(); + changed.deployment = DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(1), + container: None, + }); + assert!(matches!( + check_deployment_commit(tx, &changed, now(), &limits), + Err(DeploymentError::OperationConflict) + )); + // A retained row does not extend the advertised retry window. + let expired = Timestamp::from_micros_since_unix_epoch( + now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS as i64) * 1000, + ); + assert!(matches!( + check_deployment_commit(tx, &original, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn container_fence_revokes_copied_credentials_and_does_not_reopen_at_same_generation() { + let db = TestDB::in_memory().unwrap(); + let source = Identity::from_u256(777u64.into()); + let first = StContainerFenceRow { + source_identity: source.into(), + generation: 1, + target_grant_revision: 3, + target_set_hash: hash_bytes(b"targets1"), + allowed: true, + }; + transact(&db, |tx| { + assert!(matches!( + check_container_fence(tx, source, 1, 3), + Err(DeploymentError::ContainerFenced) + )); + install_container_fence(&db, tx, &first)?; + install_container_fence(&db, tx, &first)?; + check_container_fence(tx, source, 1, 3)?; + assert!(matches!( + check_container_fence(tx, source, 1, 4), + Err(DeploymentError::ContainerFenced) + )); + Ok(()) + }) + .unwrap(); + let revoked = StContainerFenceRow { + generation: 2, + target_grant_revision: 4, + target_set_hash: hash_bytes(b"targets2"), + allowed: false, + ..first.clone() + }; + transact(&db, |tx| { + install_container_fence(&db, tx, &revoked)?; + assert!(matches!( + check_container_fence(tx, source, 1, 3), + Err(DeploymentError::ContainerFenced) + )); + assert!(matches!( + check_container_fence(tx, source, 2, 4), + Err(DeploymentError::ContainerFenced) + )); + let reopen = StContainerFenceRow { + allowed: true, + ..revoked.clone() + }; + assert!(matches!( + install_container_fence(&db, tx, &reopen), + Err(DeploymentError::FenceConflict) + )); + assert!(matches!( + install_container_fence(&db, tx, &first), + Err(DeploymentError::FenceConflict) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn captured_connection_authority_survives_replay_without_inferring_sender_authority() { + let db = TestDB::durable().unwrap(); + let self_sender = db.database_identity(); + let foreign_sender = Identity::ONE; + let self_connection = ConnectionId::from_u128(41); + let foreign_connection = ConnectionId::from_u128(42); + let ordinary_connection = ConnectionId::from_u128(43); + transact(&db, |tx| { + for (connection, sender, flags) in [ + (self_connection, self_sender, 1), + (foreign_connection, foreign_sender, 0), + ] { + tx.insert_st_client( + sender, + connection, + r#"{"iss":"platform","sub":"previously-admitted","exp":1}"#, + )?; + record_connection_auth(tx, connection, sender, flags)?; + } + tx.insert_st_client(self_sender, ordinary_connection, "ordinary JWT")?; + Ok(()) + }) + .unwrap(); + // TestDB::reopen expects zero connected clients. Reopen the same committed + // log explicitly to exercise crash recovery with outstanding connections. + let (db, durability, runtime, directory) = db.into_parts(); + let runtime = runtime.unwrap(); + let directory = directory.unwrap(); + let durability = durability.unwrap(); + runtime.block_on(db.shutdown()); + drop(db); + runtime.block_on(durability.close()); + drop(durability); + let _runtime_guard = runtime.enter(); + let (db, durability) = TestDB::open_existing_durable( + &directory, + runtime.handle().clone(), + 0, + TestDB::DATABASE_IDENTITY, + TestDB::OWNER, + true, + ) + .unwrap(); + transact(&db, |tx| { + assert_eq!(connection_auth_flags(tx, self_connection, self_sender)?, 1); + assert_eq!(connection_auth_flags(tx, foreign_connection, foreign_sender)?, 0); + // Equal sender/database identities do not invent internal authority. + assert_eq!(connection_auth_flags(tx, ordinary_connection, self_sender)?, 0); + assert_eq!(tx.table_row_count(ST_CONNECTION_AUTH_ID), Some(2)); + assert!(matches!( + connection_auth_flags(tx, self_connection, foreign_sender), + Err(DeploymentError::CorruptMetadata) + )); + Ok(()) + }) + .unwrap(); + db.clear_all_clients().unwrap(); + transact(&db, |tx| { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(0)); + } + Ok(()) + }) + .unwrap(); + runtime.block_on(db.shutdown()); + drop(db); + runtime.block_on(durability.close()); +} + +#[test] +fn connection_auth_and_client_rows_share_connect_and_cleanup_transactions() { + let db = TestDB::in_memory().unwrap(); + let sender = db.database_identity(); + let connection = ConnectionId::from_u128(77); + let rejected: Result<(), DeploymentError> = transact(&db, |tx| { + tx.insert_st_client(sender, connection, "JWT")?; + record_connection_auth(tx, connection, sender, 1)?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(rejected.is_err()); + transact(&db, |tx| { + assert!(tx.st_client_row(sender, connection).is_none()); + assert_eq!(connection_auth_flags(tx, connection, sender)?, 0); + tx.insert_st_client(sender, connection, "JWT")?; + record_connection_auth(tx, connection, sender, 1)?; + Ok(()) + }) + .unwrap(); + // A failed callback transaction cannot partially delete its captured auth. + let failed_callback: Result<(), DeploymentError> = transact(&db, |tx| { + tx.delete_st_client(sender, connection, db.database_identity())?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(failed_callback.is_err()); + transact(&db, |tx| { + assert!(tx.st_client_row(sender, connection).is_some()); + assert_eq!(connection_auth_flags(tx, connection, sender)?, 1); + // Both successful callbacks and fallback cleanup use this deletion path. + tx.delete_st_client(sender, connection, db.database_identity())?; + Ok(()) + }) + .unwrap(); + transact(&db, |tx| { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(0)); + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn container_fence_installation_serializes_with_admitted_transactions() { + let db = TestDB::in_memory().unwrap(); + let source = Identity::from_u256(778u64.into()); + let first = StContainerFenceRow { + source_identity: source.into(), + generation: 1, + target_grant_revision: 0, + target_set_hash: hash_bytes(b"self"), + allowed: true, + }; + transact(&db, |tx| install_container_fence(&db, tx, &first)).unwrap(); + let admitted = begin_mut_tx(&db); + check_container_fence(&admitted, source, 1, 0).unwrap(); + // A fence cannot commit midway through an already admitted transaction. + assert!(db + .try_begin_mut_tx( + spacetimedb_datastore::traits::IsolationLevel::Serializable, + Workload::ForTests + ) + .is_none()); + let _ = db.rollback_mut_tx(admitted); + transact(&db, |tx| { + install_container_fence(&db, tx, &StContainerFenceRow { generation: 2, ..first }) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!( + check_container_fence(tx, source, 1, 0), + Err(DeploymentError::ContainerFenced) + )); + check_container_fence(tx, source, 2, 0) + }) + .unwrap(); +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index a7117db5b71..88a553da487 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -26,6 +26,15 @@ pub mod update { pub use spacetimedb_engine::update::*; } +pub mod container_environment; +pub mod deployment; +pub mod hosted_admission { + pub use spacetimedb_engine::hosted_admission::*; +} + +#[cfg(test)] +mod retained_shutdown_tests; + /// Whether SpacetimeDB is run in memory, or persists objects and /// a message log to disk. #[derive(Clone, Copy)] diff --git a/crates/core/src/db/retained_shutdown_tests.rs b/crates/core/src/db/retained_shutdown_tests.rs new file mode 100644 index 00000000000..5e9211378ef --- /dev/null +++ b/crates/core/src/db/retained_shutdown_tests.rs @@ -0,0 +1,50 @@ +use crate::db::{environment, relational_db::tests_utils::TestDB}; +use crate::error::DBError; +use crate::host::module_host::{DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall}; +use crate::subscription::module_subscription_actor::ModuleSubscriptions; +use spacetimedb_datastore::{execution_context::Workload, traits::IsolationLevel}; + +#[test] +fn operation_drain_retained_sql_and_subscription_handles_reject_after_writer_close() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let fixture = TestDB::durable().unwrap(); + let db = fixture.db.clone(); + runtime.block_on(async { + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.clone()); + db.shutdown().await; + let auth = spacetimedb_lib::identity::AuthCtx::new(db.owner_identity(), db.owner_identity()); + for statement in ["SELECT * FROM st_client", "SET env.LATE = 'denied'"] { + let error = crate::sql::execute::run( + db.clone(), + statement.into(), + auth.clone(), + Some(subscriptions.clone()), + None, + &mut vec![], + ) + .await + .err() + .unwrap(); + assert!(matches!(error, DBError::DatabaseClosed), "{error}"); + } + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Unsubscribe); + environment::set(&db, &mut tx, "LATE", "denied").unwrap(); + let event = ModuleEvent { + timestamp: spacetimedb_lib::Timestamp::now(), + caller_identity: db.owner_identity(), + caller_connection_id: None, + function_call: ModuleFunctionCall::update(), + status: EventStatus::Committed(DatabaseUpdate::default()), + reducer_return_value: None, + execution_budget_used: spacetimedb_client_api_messages::energy::FunctionBudget::ZERO, + host_execution_duration: std::time::Duration::ZERO, + request_id: None, + timer: None, + }; + let error = subscriptions.commit_and_broadcast_event(None, event, tx).err().unwrap(); + assert!(matches!(error, DBError::DatabaseClosed)); + let tx = db.begin_tx(Workload::Internal); + assert_eq!(environment::get(&tx, "LATE").unwrap(), None); + let _ = db.release_tx(tx); + }); +} diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 0f8d5621b3f..7417927fe7f 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -34,6 +34,8 @@ pub enum NodesError { InvalidEnvironmentKey, #[error("too many outstanding byte sources for environment read")] EnvironmentSourceLimit, + #[error("hosted invocation rejected: {0}")] + HostedInvocationRejected(String), #[error("Failed to decode row: {0}")] DecodeRow(#[source] DecodeError), #[error("Failed to decode value: {0}")] diff --git a/crates/core/src/host/container_environment.rs b/crates/core/src/host/container_environment.rs new file mode 100644 index 00000000000..45535df1a09 --- /dev/null +++ b/crates/core/src/host/container_environment.rs @@ -0,0 +1,188 @@ +//! Bounded, durable host operations for immutable container environments. +//! +//! The caller must confirm current control authority and authoritative leader +//! before calling. These local host APIs are not an external authentication +//! interface. Historical restore must keep admission closed until current +//! operational fences have been reconciled. No in-memory production fallback. + +use crate::db::container_environment::{ + self as storage, EnvironmentClosedReceipt, EnvironmentSnapshotError, SecretEnvironment, +}; +use crate::db::relational_db::{MutTx, RelationalDB}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::container_environment::{EnvironmentSnapshotReceipt, EnvironmentSnapshotScope}; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Semaphore; + +/// Bounds queued/blocked transactions even when the async caller is cancelled. +static OPERATIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(8))); + +/// The barrier is specific to this proof. It can advance on an exact retry. +#[derive(Debug)] +pub struct Durable { + pub receipt: T, + pub durable_through: u64, +} + +pub type DurableSnapshotReceipt = Durable; +pub type DurableEnvironmentValues = Durable; +pub type DurableClosedReceipt = Durable; + +pub async fn capture( + db: Arc, + scope: EnvironmentSnapshotScope, +) -> Result { + let action_db = db.clone(); + mutate(db, move |tx| storage::capture(&action_db, tx, &scope)).await +} + +pub async fn read( + db: Arc, + receipt: EnvironmentSnapshotReceipt, +) -> Result { + read_with_capacity(db, receipt, OPERATIONS.clone()).await +} + +async fn read_with_capacity( + db: Arc, + receipt: EnvironmentSnapshotReceipt, + capacity: Arc, +) -> Result { + let mut durability = db + .durable_tx_offset() + .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| EnvironmentSnapshotError::Capacity)?; + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_tx(Workload::Internal); + let result = storage::read(&action_db, &tx, &receipt); + let (offset, metrics, reducer) = action_db.release_tx(tx); + action_db.report_read_tx_metrics(reducer, metrics); + result.map(|result| (offset, result)) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)? +} + +pub async fn close( + db: Arc, + scope: EnvironmentSnapshotScope, + closed_through_generation: u64, +) -> Result { + let action_db = db.clone(); + mutate(db, move |tx| { + storage::close(&action_db, tx, &scope, closed_through_generation) + }) + .await +} + +async fn mutate( + db: Arc, + action: impl FnOnce(&mut MutTx) -> Result + Send + 'static, +) -> Result, EnvironmentSnapshotError> { + mutate_with_capacity(db, OPERATIONS.clone(), action).await +} + +async fn mutate_with_capacity( + db: Arc, + capacity: Arc, + action: impl FnOnce(&mut MutTx) -> Result + Send + 'static, +) -> Result, EnvironmentSnapshotError> { + let mut durability = db + .durable_tx_offset() + .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| EnvironmentSnapshotError::Capacity)?; + // A cancelled waiter drops only this JoinHandle. The actual operation + // keeps its database and finite slot until commit and durability finish. + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, result) = action_db.with_auto_rollback(tx, action)?; + let (offset, data, metrics, reducer) = action_db + .commit_tx(tx) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .ok_or(EnvironmentSnapshotError::Storage)?; + action_db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, EnvironmentSnapshotError>((offset, result)) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)? +} + +#[cfg(test)] +#[path = "container_environment/durability_tests.rs"] +mod durability_tests; + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use std::time::Duration; + + #[test] + fn container_environment_cancelled_transaction_retains_capacity_until_worker_finishes() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let (entered, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release, release_rx) = std::sync::mpsc::sync_channel(1); + let blocked = db + .runtime() + .unwrap() + .spawn(mutate_with_capacity(db.db.clone(), capacity.clone(), move |_| { + entered.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Err::<(), _>(EnvironmentSnapshotError::InvalidEnvironment) + })); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + blocked.abort(); + db.runtime().unwrap().block_on(async { + assert!(blocked.await.unwrap_err().is_cancelled()); + let retry = mutate_with_capacity(db.db.clone(), capacity.clone(), |_| Ok(())).await; + assert!(matches!(retry, Err(EnvironmentSnapshotError::Capacity))); + }); + release.send(()).unwrap(); + db.runtime().unwrap().block_on(async { + // This waits for the actual cancelled caller's blocking worker to + // return, not merely for cancellation of its async JoinHandle. + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + mutate_with_capacity(db.db.clone(), capacity, |_| Ok(())).await.unwrap(); + }); + } +} diff --git a/crates/core/src/host/container_environment/durability_tests.rs b/crates/core/src/host/container_environment/durability_tests.rs new file mode 100644 index 00000000000..d8512b39c7a --- /dev/null +++ b/crates/core/src/host/container_environment/durability_tests.rs @@ -0,0 +1,291 @@ +//! Real local writes with only their durability acknowledgment delayed. +use super::*; +use crate::db::deployment::{ + install_container_fence, install_publication_fence, record_deployment_commit, DeploymentCommit, +}; +use crate::db::relational_db::{ + local_durability, + tests_utils::{TempReplicaDir, TestDB}, + LocalDurability, +}; +use crate::db::{environment, persistence::Persistence}; +use spacetimedb_datastore::system_tables::StContainerFenceRow; +use spacetimedb_durability::{Close, Durability, DurableOffset, PreparedTx}; +use spacetimedb_lib::container::*; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; +use spacetimedb_lib::{hash_bytes, Timestamp, Uuid}; +use std::time::Duration; + +fn uuid() -> Uuid { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) +} + +fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: keys, + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } + .normalize(&Default::default()) + .unwrap(); + let request = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: db.owner_identity(), + expected_revision: None, + prepared_manifest_hash: hash_bytes(b"prepared"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: Some(spec.clone()), + }), + }; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + record_deployment_commit(tx, &request, Timestamp::now(), &Default::default())?; + install_container_fence(db, tx, &self_fence(db, 1, true))?; + for key in &spec.env_keys { + environment::set(db, tx, key, "before")?; + } + Ok(()) + }) + .unwrap(); + EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: 1, + database_identity: db.database_identity(), + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: request.deployment.revision().unwrap(), + start_request: request.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + } +} + +fn self_fence(db: &RelationalDB, generation: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed, + } +} + +struct DelayedAcknowledgment { + writer: LocalDurability, + acknowledged: DurableOffset, +} +impl Durability for DelayedAcknowledgment { + type TxData = crate::db::relational_db::Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.writer.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.acknowledged.clone() + } + fn close(&self) -> Close { + self.writer.close() + } +} + +struct Fixture { + db: Arc, + writer: LocalDurability, + acknowledge: tokio::sync::watch::Sender>, + _directory: TempReplicaDir, +} +impl Fixture { + async fn new() -> Self { + let directory = TempReplicaDir::new().unwrap(); + let (writer, disk_size) = + local_durability((*directory).clone(), spacetimedb_runtime::Handle::tokio_current(), None) + .await + .unwrap(); + let (acknowledge, acknowledged) = tokio::sync::watch::channel(None); + let db = Arc::new( + TestDB::open_db( + writer.as_history(), + Some(Persistence { + durability: Arc::new(DelayedAcknowledgment { + writer: writer.clone(), + acknowledged: acknowledged.into(), + }), + disk_size, + snapshots: None, + runtime: spacetimedb_runtime::Handle::tokio_current(), + }), + None, + 0, + ) + .unwrap(), + ); + Self { + db, + writer, + acknowledge, + _directory: directory, + } + } + async fn acknowledge_current(&self) -> u64 { + let tx = self.db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = self.db.release_tx(tx); + self.db.report_read_tx_metrics(reducer, metrics); + let mut actual = self.writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + self.acknowledge.send_replace(Some(durable)); + offset + } + async fn finish(self) { + self.db.shutdown().await; + drop(self.db); + self.writer.close().await; + } +} + +async fn wait_for_slot(capacity: &Arc) { + tokio::time::timeout(Duration::from_secs(5), async { + while capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +async fn released(capacity: &Arc) { + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_capture_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let capacity = Arc::new(Semaphore::new(1)); + let action_db = db.clone(); + let (captured, result) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(mutate_with_capacity(db.clone(), capacity.clone(), move |tx| { + let receipt = storage::capture(&action_db, tx, &scope)?; + captured.send(receipt.clone()).unwrap(); + Ok(receipt) + })); + let captured = tokio::time::timeout(Duration::from_secs(5), result) + .await + .unwrap() + .unwrap(); + // This read waits for the actual capture transaction to commit. + db.with_read_only(Workload::ForTests, |tx| storage::read(&db, tx, &captured)) + .unwrap(); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + mutate_with_capacity(db.clone(), capacity.clone(), |_| Ok(())).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + let values = read(db.clone(), captured).await.unwrap(); + assert_eq!(values.receipt.selected_values["SECRET"], "before"); + drop(db); + fixture.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_read_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let receipt = db + .with_auto_commit(Workload::ForTests, |tx| storage::capture(&db, tx, &scope)) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(read_with_capacity(db.clone(), receipt.clone(), capacity.clone())); + wait_for_slot(&capacity).await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + read_with_capacity(db.clone(), receipt.clone(), capacity.clone()).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + let values = read_with_capacity(db.clone(), receipt, capacity).await.unwrap(); + assert_eq!(values.receipt.selected_values["SECRET"], "before"); + drop(db); + fixture.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_close_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let receipt = db + .with_auto_commit(Workload::ForTests, |tx| storage::capture(&db, tx, &scope)) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &self_fence(&db, 2, false)) + }) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let action_db = db.clone(); + let (closed, result) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(mutate_with_capacity(db.clone(), capacity.clone(), move |tx| { + let receipt = storage::close(&action_db, tx, &scope, 2)?; + closed.send(()).unwrap(); + Ok(receipt) + })); + tokio::time::timeout(Duration::from_secs(5), result) + .await + .unwrap() + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + assert_eq!( + tx.table_row_count(spacetimedb_datastore::system_tables::ST_CONTAINER_ENVIRONMENT_ID), + Some(0) + ); + }); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + mutate_with_capacity(db.clone(), capacity.clone(), |_| Ok(())).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + assert!(matches!( + read(db.clone(), receipt).await, + Err(EnvironmentSnapshotError::Fenced) + )); + drop(db); + fixture.finish().await; +} diff --git a/crates/core/src/host/container_fence.rs b/crates/core/src/host/container_fence.rs new file mode 100644 index 00000000000..4b872fbcea1 --- /dev/null +++ b/crates/core/src/host/container_fence.rs @@ -0,0 +1,204 @@ +//! Bounded host-only inspection and durable denial of retained source fences. +//! +//! The authenticated platform coordinator owns control membership checks and +//! actor drainage. These APIs do not authorize an external client or expose the +//! protected fence table through SQL, subscriptions, or module syscalls. + +use crate::db::deployment::{deny_container_fence, DeploymentError, FenceDenial}; +use crate::db::relational_db::RelationalDB; +use crate::host::container_environment::Durable; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::ScanOrIndex; +use spacetimedb_datastore::system_tables::{StContainerFenceRow, ST_CONTAINER_FENCE_ID}; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::Identity; +use spacetimedb_primitives::ColId; +use spacetimedb_sats::AlgebraicValue; +use std::ops::Bound; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Semaphore; + +pub const FENCE_PAGE_SIZE: usize = 64; +static OPERATIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(8))); + +#[derive(Debug, thiserror::Error)] +pub enum FenceOperationError { + #[error("receiving host fence operation capacity exhausted")] + Capacity, + #[error("receiving host fence storage is unavailable")] + Storage, + #[error("receiving host fence durability is unavailable")] + DurabilityUnavailable, + #[error("receiving host fence durability failed")] + DurabilityFailed, + #[error("receiving host fence index is unavailable")] + IndexUnavailable, + #[error("receiving host fence inventory changed")] + RevisionChanged, + #[error(transparent)] + Deployment(#[from] DeploymentError), +} + +#[derive(Debug)] +pub struct FencePage { + /// Includes denied rows so every bounded physical page advances its cursor. + pub rows: Vec, + /// Last source in this page, or the input cursor if the page is empty. + pub next_source: Option, + pub complete: bool, + /// Read under the same database transaction as the rows. Restart the scan + /// if this differs from the previous page or a known local denial result. + pub fence_revision: u64, +} + +/// Read at most 64 rows using the protected source Identity B-tree. We refuse +/// the datastore's scan fallback: row order and physical work must be bounded. +pub async fn page(db: Arc, after: Option) -> Result { + let permit = OPERATIONS + .clone() + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let tx = db.begin_tx(Workload::Internal); + let result = (|| { + let lower = after.map_or(Bound::Unbounded, |identity| { + Bound::Excluded(AlgebraicValue::U256(identity.to_u256().into())) + }); + let range = db + .iter_by_col_range(&tx, ST_CONTAINER_FENCE_ID, ColId(0), (lower, Bound::Unbounded)) + .map_err(|_| FenceOperationError::Storage)?; + let ScanOrIndex::Index(mut range) = range else { + return Err(FenceOperationError::IndexUnavailable); + }; + let rows = range + .by_ref() + .take(FENCE_PAGE_SIZE) + .map(|row| StContainerFenceRow::try_from(row).map_err(|_| FenceOperationError::Storage)) + .collect::, _>>()?; + let complete = range.next().is_none(); + let next_source = rows.last().map(|row| row.source_identity.into()).or(after); + Ok(FencePage { + rows, + next_source, + complete, + fence_revision: db.hosted_admission().fence_revision(), + }) + })(); + let (_, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + result + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +/// The coordinator must have excluded this exact source from current control +/// authority. For a check that must serialize with an inventory lock, use the +/// synchronous `deployment::deny_container_fence` inside the caller's own +/// serializable transaction instead. Neither API performs that control check. +pub async fn deny_orphan( + db: Arc, + expected: StContainerFenceRow, +) -> Result, FenceOperationError> { + deny_with_capacity(db, expected, OPERATIONS.clone()).await +} + +/// Confirm durability of all fences visible at one exact physical revision. +/// In particular, a retry that sees a previous owner's committed denial must +/// still wait for that denial's storage acknowledgment. The coordinator must +/// recheck the revision under its final database transaction before admission; +/// this receipt does not freeze future mutations or confer control authority. +pub async fn confirm_revision( + db: Arc, + expected_physical: u64, +) -> Result, FenceOperationError> { + confirm_with_capacity(db, expected_physical, OPERATIONS.clone()).await +} + +async fn confirm_with_capacity( + db: Arc, + expected_physical: u64, + capacity: Arc, +) -> Result, FenceOperationError> { + let mut durability = db + .durable_tx_offset() + .ok_or(FenceOperationError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let durable_through = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_tx(Workload::Internal); + let physical = action_db.hosted_admission().fence_revision(); + let (offset, metrics, reducer) = action_db.release_tx(tx); + action_db.report_read_tx_metrics(reducer, metrics); + if physical != expected_physical || physical == u64::MAX { + return Err(FenceOperationError::RevisionChanged); + } + Ok(offset) + }) + .await + .map_err(|_| FenceOperationError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| FenceOperationError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt: (), + durable_through, + }) + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +async fn deny_with_capacity( + db: Arc, + expected: StContainerFenceRow, + capacity: Arc, +) -> Result, FenceOperationError> { + let mut durability = db + .durable_tx_offset() + .ok_or(FenceOperationError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + // A caller's cancellation drops only this JoinHandle. The owned task keeps + // both the permit and database alive through physical commit and durability. + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, receipt) = + action_db.with_auto_rollback(tx, |tx| deny_container_fence(&action_db, tx, &expected))?; + let (offset, data, metrics, reducer) = action_db + .commit_tx(tx) + .map_err(|_| FenceOperationError::Storage)? + .ok_or(FenceOperationError::Storage)?; + action_db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, FenceOperationError>((offset, receipt)) + }) + .await + .map_err(|_| FenceOperationError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| FenceOperationError::DurabilityFailed)?; + // Do not drop the physical database before the owned durability wait. + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/host/container_fence/tests.rs b/crates/core/src/host/container_fence/tests.rs new file mode 100644 index 00000000000..41e203a072a --- /dev/null +++ b/crates/core/src/host/container_fence/tests.rs @@ -0,0 +1,392 @@ +use super::*; +use crate::db::deployment::{check_container_fence, install_container_fence}; +use crate::db::relational_db::tests_utils::TestDB; +use spacetimedb_lib::hash_bytes; +use std::time::Duration; + +fn fence(source: u64) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: Identity::from_u256(source.into()).into(), + generation: 7, + target_grant_revision: 3, + target_set_hash: hash_bytes(b"retained target set"), + allowed: true, + } +} + +fn install(db: &RelationalDB, row: &StContainerFenceRow) { + db.with_auto_commit(Workload::ForTests, |tx| install_container_fence(db, tx, row)) + .unwrap(); +} + +#[tokio::test] +async fn container_fence_pages_follow_identity_index_and_include_denied_rows() { + let db = TestDB::in_memory().unwrap(); + // Reverse insertion order deliberately differs from source index order. + for source in (0..130).rev() { + install( + &db, + &StContainerFenceRow { + allowed: source % 3 == 0, + ..fence(source) + }, + ); + } + let revision = db.hosted_admission().fence_revision(); + let mut after = None; + let mut identities = Vec::new(); + for expected_size in [64, 64, 2] { + let page = page(db.db.clone(), after).await.unwrap(); + assert_eq!(page.rows.len(), expected_size); + assert_eq!(page.complete, expected_size == 2); + assert_eq!(page.fence_revision, revision); + identities.extend(page.rows.iter().map(|row| Identity::from(row.source_identity))); + after = page.next_source; + assert_eq!(after, identities.last().copied()); + } + assert_eq!( + identities, + (0..130u64).map(|n| Identity::from_u256(n.into())).collect::>() + ); + let empty = page(db.db.clone(), after).await.unwrap(); + assert!(empty.rows.is_empty() && empty.complete); + assert_eq!(empty.next_source, after); +} + +#[tokio::test] +async fn container_fence_missing_index_refuses_unbounded_scan_and_memory_refuses_denial() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(17); + install(&db, &expected); + assert!(matches!( + deny_orphan(db.db.clone(), expected).await, + Err(FenceOperationError::DurabilityUnavailable) + )); + db.with_auto_commit(Workload::ForTests, |tx| { + db.drop_index(tx, spacetimedb_primitives::IndexId(34)) + }) + .unwrap(); + assert!(matches!( + page(db.db.clone(), None).await, + Err(FenceOperationError::IndexUnavailable) + )); +} + +#[test] +fn container_fence_orphan_denial_is_durable_exact_and_cannot_reopen_generation() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let expected = fence(19); + install(&db, &expected); + db.hosted_admission().begin().unwrap().complete().unwrap(); + let before = db.hosted_admission().fence_revision(); + let first = db + .runtime() + .unwrap() + .block_on(deny_orphan(db.db.clone(), expected.clone())) + .unwrap(); + assert_eq!(first.receipt.revision_before, before); + assert_eq!(first.receipt.revision_after, before + 1); + assert!(!db.hosted_admission().is_open()); + assert_eq!( + first.receipt.row, + StContainerFenceRow { + allowed: false, + ..expected.clone() + } + ); + assert!(db.durable_tx_offset().unwrap().get().unwrap().unwrap() >= first.durable_through); + let retry = db + .runtime() + .unwrap() + .block_on(deny_orphan(db.db.clone(), expected.clone())) + .unwrap(); + assert_eq!(retry.receipt.revision_before, before + 1); + assert_eq!(retry.receipt.revision_after, before + 1); + let db = db.reopen().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + assert!(matches!( + check_container_fence( + tx, + expected.source_identity.into(), + expected.generation, + expected.target_grant_revision + ), + Err(DeploymentError::ContainerFenced) + )); + assert!(matches!( + install_container_fence(&db, tx, &expected), + Err(DeploymentError::FenceConflict) + )); + let retry = deny_container_fence(&db, tx, &expected)?; + assert_eq!(retry.row, first.receipt.row); + assert_eq!(retry.revision_before, retry.revision_after); + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + install( + &db, + &StContainerFenceRow { + generation: expected.generation + 1, + ..expected + }, + ); +} + +#[test] +fn container_fence_orphan_cas_rejects_missing_changed_and_invalid_observations() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(20); + db.with_auto_commit(Workload::ForTests, |tx| { + assert!(matches!( + deny_container_fence(&db, tx, &expected), + Err(DeploymentError::FenceConflict) + )); + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + install(&db, &expected); + let next = StContainerFenceRow { + generation: expected.generation + 1, + ..expected.clone() + }; + install(&db, &next); + let revision = db.hosted_admission().fence_revision(); + db.with_auto_commit(Workload::ForTests, |tx| { + for stale in [ + expected, + StContainerFenceRow { + allowed: false, + ..next.clone() + }, + StContainerFenceRow { + target_set_hash: hash_bytes(b"different"), + ..next.clone() + }, + StContainerFenceRow { + target_grant_revision: 4, + ..next.clone() + }, + ] { + assert!(matches!( + deny_container_fence(&db, tx, &stale), + Err(DeploymentError::FenceConflict) + )); + } + check_container_fence( + tx, + next.source_identity.into(), + next.generation, + next.target_grant_revision, + )?; + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + assert_eq!(db.hosted_admission().fence_revision(), revision); +} + +#[tokio::test] +async fn container_fence_physical_change_behind_cursor_invalidates_completion() { + let db = TestDB::in_memory().unwrap(); + install(&db, &fence(100)); + let ticket = db.hosted_admission().begin().unwrap(); + let first = page(db.db.clone(), None).await.unwrap(); + assert!(first.complete); + install(&db, &fence(1)); + assert!(ticket.complete_with_fence_revision(first.fence_revision).is_err()); + assert!(!db.hosted_admission().is_open()); + let restarted = page(db.db.clone(), None).await.unwrap(); + assert_eq!(restarted.rows.len(), 2); + assert_ne!(first.fence_revision, restarted.fence_revision); +} + +#[test] +fn container_fence_orphan_rollback_keeps_row_and_invalidates_physical_scan() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(21); + install(&db, &expected); + let before = db.hosted_admission().fence_revision(); + let ticket = db.hosted_admission().begin().unwrap(); + let result = db.with_auto_commit(Workload::ForTests, |tx| { + deny_container_fence(&db, tx, &expected)?; + Err::<(), _>(DeploymentError::CorruptMetadata) + }); + assert!(result.is_err()); + assert!(ticket.complete_with_fence_revision(before).is_err()); + db.with_auto_commit(Workload::ForTests, |tx| { + check_container_fence( + tx, + expected.source_identity.into(), + expected.generation, + expected.target_grant_revision, + ) + }) + .unwrap(); +} + +/// Real local storage with its durable acknowledgment deliberately withheld. +/// This models a lagging quorum without replacing the transaction or writer. +struct DelayedAcknowledgment { + writer: Arc>, + acknowledged: spacetimedb_durability::DurableOffset, +} + +impl spacetimedb_durability::Durability for DelayedAcknowledgment { + type TxData = crate::db::relational_db::Txdata; + fn append_tx(&self, tx: spacetimedb_durability::PreparedTx) { + self.writer.append_tx(tx); + } + fn durable_tx_offset(&self) -> spacetimedb_durability::DurableOffset { + self.acknowledged.clone() + } + fn close(&self) -> spacetimedb_durability::Close { + self.writer.close() + } +} + +async fn delayed_storage() -> ( + crate::db::relational_db::tests_utils::TempReplicaDir, + Arc, + crate::db::relational_db::LocalDurability, + tokio::sync::watch::Sender>, +) { + use crate::db::persistence::Persistence; + use crate::db::relational_db::{local_durability, tests_utils::TempReplicaDir}; + let directory = TempReplicaDir::new().unwrap(); + let (writer, disk_size) = + local_durability((*directory).clone(), spacetimedb_runtime::Handle::tokio_current(), None) + .await + .unwrap(); + let (acknowledge, acknowledged) = tokio::sync::watch::channel(None); + let db = Arc::new( + TestDB::open_db( + writer.as_history(), + Some(Persistence { + durability: Arc::new(DelayedAcknowledgment { + writer: writer.clone(), + acknowledged: acknowledged.into(), + }), + disk_size, + snapshots: None, + runtime: spacetimedb_runtime::Handle::tokio_current(), + }), + None, + 0, + ) + .unwrap(), + ); + (directory, db, writer, acknowledge) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_fence_cancelled_denial_retains_capacity_through_actual_durability_wait() { + use spacetimedb_durability::Durability; + let (_directory, db, writer, acknowledge) = delayed_storage().await; + let expected = fence(22); + install(&db, &expected); + let capacity = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(deny_with_capacity(db.clone(), expected.clone(), capacity.clone())); + let row = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let page = page(db.clone(), None).await.unwrap(); + if let Some(row) = page.rows.first().filter(|row| !row.allowed) { + break row.clone(); + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!row.allowed); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + deny_with_capacity(db.clone(), expected.clone(), capacity.clone()).await, + Err(FenceOperationError::Capacity) + )); + let tx = db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + let mut actual = writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + acknowledge.send_replace(Some(durable)); + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + // The detached owner finished only once its physical acknowledgment arrived. + db.shutdown().await; + drop(db); + writer.close().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_fence_retry_barrier_waits_for_prior_commit_and_retains_cancelled_capacity() { + use spacetimedb_durability::Durability; + let (_directory, db, writer, acknowledge) = delayed_storage().await; + let expected = fence(23); + install(&db, &expected); + // A previous coordinator has committed but never confirmed durability. + let denied = db + .with_auto_commit(Workload::ForTests, |tx| deny_container_fence(&db, tx, &expected)) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + // Changed inventory fails immediately, even while acknowledgments lag. + assert!(matches!( + tokio::time::timeout( + Duration::from_secs(5), + confirm_with_capacity(db.clone(), denied.revision_before, capacity.clone()) + ) + .await + .unwrap(), + Err(FenceOperationError::RevisionChanged) + )); + let caller = tokio::spawn(confirm_with_capacity( + db.clone(), + denied.revision_after, + capacity.clone(), + )); + tokio::time::timeout(Duration::from_secs(5), async { + while capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!caller.is_finished()); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert!(matches!( + confirm_with_capacity(db.clone(), denied.revision_after, capacity.clone()).await, + Err(FenceOperationError::Capacity) + )); + let tx = db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + let mut actual = writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + acknowledge.send_replace(Some(durable)); + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + let proof = confirm_revision(db.clone(), denied.revision_after).await.unwrap(); + assert_eq!(proof.durable_through, offset); + // The proof is only a barrier, so finalization still checks current revision. + install(&db, &fence(24)); + assert!(matches!( + confirm_revision(db.clone(), denied.revision_after).await, + Err(FenceOperationError::RevisionChanged) + )); + db.shutdown().await; + drop(db); + writer.close().await; +} diff --git a/crates/core/src/host/empty_module.rs b/crates/core/src/host/empty_module.rs new file mode 100644 index 00000000000..c0310ac8628 --- /dev/null +++ b/crates/core/src/host/empty_module.rs @@ -0,0 +1,47 @@ +//! Versioned built-in module for a database published with only a container. +//! +//! This is a real Wasm program, not an empty byte string. It declares an empty +//! V10 user schema and hosted_auth_v1, so normal database initialization, system +//! tables, subscriptions, and later migration use the existing host machinery. +//! Its required reducer ABI entry point traps because no reducer is declared. +//! +//! Version 1 bytes are immutable. Reproduce them with +//! `python3 crates/core/src/host/empty_module/generate.py --check` from public/. +//! New program bytes require a new version, preserving old deployment replay. + +use spacetimedb_datastore::{system_tables::ModuleKind, traits::Program}; +use spacetimedb_lib::{hash_bytes, Hash}; +use std::sync::OnceLock; + +pub const VERSION_1: u32 = 1; +pub use spacetimedb_lib::deployment::SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH as VERSION_1_PROGRAM_HASH; + +pub use spacetimedb_lib::deployment::SYSTEM_EMPTY_MODULE_V1_BYTES as VERSION_1_BYTES; + +/// Return the exact bundled program for a recognized system module version. +/// Unknown versions fail closed instead of silently selecting the latest one. +pub fn program(version: u32) -> Option { + (version == VERSION_1 && v1_hash() == VERSION_1_PROGRAM_HASH).then(|| Program { + hash: v1_hash(), + bytes: VERSION_1_BYTES.into(), + kind: ModuleKind::WASM, + }) +} + +/// Validate a system-empty deployment against immutable platform bytes, not +/// against an empty-looking schema supplied by a publisher or a claimed hash. +pub fn matches_program(version: u32, candidate: &Program) -> bool { + version == VERSION_1 + && candidate.kind == ModuleKind::WASM + && candidate.hash == VERSION_1_PROGRAM_HASH + && v1_hash() == VERSION_1_PROGRAM_HASH + && candidate.bytes.as_ref() == VERSION_1_BYTES +} + +fn v1_hash() -> Hash { + static HASH: OnceLock = OnceLock::new(); + *HASH.get_or_init(|| hash_bytes(VERSION_1_BYTES)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/host/empty_module/README.md b/crates/core/src/host/empty_module/README.md new file mode 100644 index 00000000000..72f3b0a74b2 --- /dev/null +++ b/crates/core/src/host/empty_module/README.md @@ -0,0 +1,23 @@ +# System empty module, version 1 + +The host uses this real Wasm module for a database published with a container and no user module. It declares V10 with an empty typespace and `hosted_auth_v1`; there are no user tables, reducers, procedures, views, schedules, or HTTP handlers. + +`generate.py` is the complete source. It uses only the Python standard library and explicitly encodes the Wasm sections and BSATN description. It produces a 250-byte Wasm binary with one fixed memory page, a schema describer, and the required reducer ABI entry point. The reducer entry point always traps because no reducer is declared. The `spacetime_10.6::get_call_auth_flags` import requires the host ABI corresponding to the advertised capability; the empty module has no invocation contexts to construct. + +From the public repository, reproduce and verify the checked-in files with: + +```sh +python3 crates/core/src/host/empty_module/generate.py --check +``` + +Omit `--check` to regenerate the three files. This does not require a Rust module compiler, WASI toolchain, WAT compiler, or container image builder. + +The SHA-256 checksum of `v1.wasm` is recorded in `v1.sha256`. SpacetimeDB's separate Keccak-256 program identity is: + +```text +83cc1cc8794f7a9a540a0743d0f874bf613b76676ca9e8ecbdc4cfca19c709a5 +``` + +The unreleased Stage 1 version-1 bundle was generated against the current V10 layout, where `Capabilities` is section 15 and upstream sections 13/14 retain their existing meanings. It supersedes the proposal-only bytes from the old base; no released version-1 deployment exists. After release, version 1 is immutable. A change to its schema or Wasm requires a new version and new files. Keep version 1 available for existing deployments and replay. The Rust helper checks the known program hash as well as the exact bytes, kind, and version, so a different publisher-supplied module with an empty-looking schema cannot qualify as the system empty module. `Program::empty` is also rejected. + +Core tests compare the BSATN fixture against the current V10 Rust wire types, load the Wasm through the actual host, and initialize a real database through `HostController`. They verify that initialization stores the bundled program and the database's metadata without invoking a user reducer. diff --git a/crates/core/src/host/empty_module/generate.py b/crates/core/src/host/empty_module/generate.py new file mode 100644 index 00000000000..e7ca62e679c --- /dev/null +++ b/crates/core/src/host/empty_module/generate.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Reproduce the version-1 system empty module using only Python's standard library. + +The binary has one page of memory, a V10 schema describer, and a reducer entry +point which always traps. There are no declared user tables or functions. + +Equivalent code (the data payload below is BSATN, not WebAssembly encoding): + (import "spacetime_10.0" "bytes_sink_write" (func (param i32 i32 i32) (result i32))) + (import "spacetime_10.6" "get_call_auth_flags" (func (result i32))) + (memory (export "memory") 1 1) + (func (export "__describe_module__") (param i32) + local.get 0 i32.const 32 i32.const 16 call 0 + if unreachable end) + (func (export "__call_reducer__") + (param i32 i64 i64 i64 i64 i64 i64 i64 i32 i32) (result i32) + unreachable) + +The unused 10.6 import explicitly requires the host ABI which supports captured +invocation flags. No function or invocation context exists in this module. +""" + +import argparse +import hashlib +from pathlib import Path +import struct + + +def leb(value): + encoded = bytearray() + while value >= 128: + encoded.append((value & 127) | 128) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def string(value): + value = value.encode("utf-8") + return leb(len(value)) + value + + +def vector(items): + return leb(len(items)) + b"".join(items) + + +def section(tag, payload): + return bytes([tag]) + leb(len(payload)) + payload + + +def function_type(params, results): + return b"\x60" + vector(params) + vector(results) + + +def generate(): + u32 = lambda value: struct.pack("!#HF0uu| literal 0 HcmV?d00001 diff --git a/crates/core/src/host/empty_module/v1.sha256 b/crates/core/src/host/empty_module/v1.sha256 new file mode 100644 index 00000000000..36b30eddeb3 --- /dev/null +++ b/crates/core/src/host/empty_module/v1.sha256 @@ -0,0 +1 @@ +02bd77cf25b47908f4064c3ddf0fa7eccca8c6165d0a41c4455469546b14ce02 v1.wasm diff --git a/crates/core/src/host/empty_module/v1.wasm b/crates/core/src/host/empty_module/v1.wasm new file mode 100644 index 0000000000000000000000000000000000000000..e2b57b1e4ffa90a21dbb30d3cec1feee3d843818 GIT binary patch literal 250 zcmY+9L2JV>427TNY=Z_eLw6qSA1En>>; -/// The registry of all running hosts. -type Hosts = Arc>>; +mod lifecycle; +mod registry; +use registry::{Hosts, Registration}; + +#[cfg(test)] +mod lifecycle_tests; + +#[cfg(test)] +mod deployment_tests; + +#[cfg(test)] +mod execution_deadline_tests; + +#[cfg(test)] +static FAIL_NEXT_DEPLOYMENT_ACTIVATION: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); pub type ExternalDurability = (Arc>, DiskSizeFn); #[async_trait] pub trait ExternalStorage: Send + Sync + 'static { async fn lookup(&self, program_hash: Hash) -> anyhow::Result>>; + + /// Resolve the authorized initial deployment from the same control + /// transaction that created the database. A failed lookup must return an + /// error, never None: None is reserved for legacy module-only databases. + /// Called only when no initialized program exists in the user database. + async fn initial_deployment(&self, _database: &Database) -> anyhow::Result> { + Ok(None) + } } #[async_trait] impl ExternalStorage for F @@ -490,14 +511,14 @@ impl HostController { // Note that `tokio::spawn` only cancels its tasks when the runtime shuts down, // at which point we won't be calling `try_init_host` again anyways. let rx = tokio::spawn(async move { - let initialized = this.try_init_host(database, replica_id).await?; + let initialized = this.try_init_host(database, replica_id, &guard).await?; let HostInit { host, bootstrap_completion, } = initialized; let rx = host.module.subscribe(); - *guard = Some(host); + guard.install(host); Ok::<_, anyhow::Error>((rx, bootstrap_completion)) }) @@ -556,6 +577,23 @@ impl HostController { replica_id: u64, program_bytes: Box<[u8]>, policy: MigrationPolicy, + ) -> anyhow::Result { + self.update_module_host_with_deployment(database, host_type, replica_id, program_bytes, policy, None) + .await + } + + /// Authorized coordinator entry point. The deployment and its operation + /// receipt are committed in the module migration transaction. Process + /// startup and the control database mirror remain separate reconciliation. + #[allow(clippy::too_many_arguments)] + pub async fn update_module_host_with_deployment( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { let program = Program::from_bytes(host_type.into(), program_bytes); trace!( @@ -593,27 +631,57 @@ impl HostController { let mut host = match guard.take() { None => { trace!("host not running, try_init"); - this.try_init_host(database, replica_id).await?.host + this.try_init_host(database, replica_id, &guard).await?.host } Some(host) => { trace!("host found, updating"); host } }; - let update_result = host - .update_module( - this.runtimes.clone(), - program, - policy, - this.energy_monitor.clone(), - this.unregister_fn(replica_id, database_identity), - this.db_cores.take(), - ) - .await?; - - *guard = Some(host); - - Ok::<_, anyhow::Error>(update_result) + let mut database_committed = false; + let registration = guard.registration(); + let update_result = std::panic::AssertUnwindSafe(host.update_module( + this.runtimes.clone(), + program, + policy, + deployment, + this.energy_monitor.clone(), + this.unregister_fn(registration.clone(), database_identity), + registration, + this.db_cores.take(), + &mut database_committed, + )) + .catch_unwind() + .await; + let update_result = match update_result { + Ok(result) => result, + Err(panic) => { + if let Err(error) = lifecycle::close_host(host).await { + if matches!(error, lifecycle::CloseFailure::WriterUnconfirmed) { + guard.quarantine(); + } + return Err(error.into()); + } + std::panic::resume_unwind(panic); + } + }; + + if update_result.is_err() && database_committed { + // Schema/program/receipt already committed. The previous + // executable cannot be retained after activation failure. + // Close clients/scheduler and reconstruct from stored program + // on the next leader lookup or reconciliation attempt. + if let Err(error) = lifecycle::close_host(host).await { + if matches!(error, lifecycle::CloseFailure::WriterUnconfirmed) { + guard.quarantine(); + } + return Err(error.into()); + } + } else { + guard.install(host); + } + + update_result }) .await??; @@ -658,58 +726,22 @@ impl HostController { /// Release all resources of the [`ModuleHost`] identified by `replica_id`, /// and deregister it from the controller. + /// + /// A timeout returns an error while the owned close continues. Only success + /// confirms the configured storage writer has completed shutdown. #[tracing::instrument(level = "trace", skip_all)] pub async fn exit_module_host(&self, replica_id: u64, timeout: Duration) -> Result<(), anyhow::Error> { - let Some(lock) = self.hosts.lock().remove(&replica_id) else { + let Some(request) = registry::close(&self.hosts, replica_id) else { return Ok(()); }; - // To debug the potential deadlock issue reported in - // https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - // we'll log a warning every 5s if we can't acquire an exclusive lock. - let start = Instant::now(); - let mut t = interval_at(start + Duration::from_secs(5), Duration::from_secs(5)); - let warn_blocked = tokio::spawn(async move { - loop { - t.tick().await; - warn!( - "blocked waiting to exit module for replica {} since {}s", - replica_id, - start.elapsed().as_secs_f32() - ); - } - }); - defer!(warn_blocked.abort()); - - let shutdown = tokio::time::timeout(timeout, async { - let mut guard = lock.write_owned().await; - let Some(host) = guard.take() else { - return; - }; - let module = host.module.borrow().clone(); - let info = module.info(); - - let database_identity = info.database_identity; - let table_names = info.module_def.tables().map(|t| t.name.deref()); - - // Ensure we clear the metrics even if the future is cancelled. - defer!(remove_database_gauges(&database_identity, table_names)); - - info!("replica={replica_id} database={database_identity} exiting module"); - module.exit().await; - info!("replica={replica_id} database={database_identity} exiting database"); - module.relational_db().shutdown().await; - info!("replica={replica_id} database={database_identity} module host exited"); - }) - .await; - - if shutdown.is_err() { - warn!( - "replica={replica_id} shutdown timed out after {}s", - start.elapsed().as_secs_f32() - ); + if let Some(owner) = request.owner { + // The close owns its registry pin independently of this waiter. + // Cancellation or timeout cannot release a still-running writer. + tokio::spawn(owner.run()); } - - Ok(()) + tokio::time::timeout(timeout, request.completion.wait()) + .await + .map_err(|_| anyhow!("replica {replica_id} shutdown is still pending after {timeout:?}"))? } /// Get the [`ModuleHost`] identified by `replica_id` or return an error @@ -766,45 +798,47 @@ impl HostController { self.hosts.lock().keys().copied().collect() } - /// On-panic callback passed to [`ModuleHost`]s created by this controller. - /// - /// Removes the module with the given `replica_id` from this controller. - fn unregister_fn(&self, replica_id: u64, database_identity: Identity) -> impl Fn() + Send + Sync + 'static + use<> { - let hosts = Arc::downgrade(&self.hosts); + /// On-panic callbacks are scoped to one installed executable and cell. + /// A stale callback cannot unregister an updated or reopened database. + fn unregister_fn( + &self, + registration: Registration, + database_identity: Identity, + ) -> impl Fn() + Send + Sync + 'static + use<> { + let runtime = tokio::runtime::Handle::current(); move || { - let unregistered = hosts - .upgrade() - .is_some_and(|hosts| hosts.lock().remove(&replica_id).is_some()); - if unregistered { + if let Some(request) = registration.close_if_current() + && let Some(owner) = request.owner + { record_module_host_unexpected_exit(database_identity); + runtime.spawn(owner.run()); } } } - /// Acquire a write lock on the [HostCell] for `replica_id`. - /// - /// This will time out after 5s to aid debugging of - /// https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - async fn acquire_write_lock(&self, replica_id: u64) -> Result>, Elapsed> { - let lock = self.hosts.lock().entry(replica_id).or_default().clone(); - timeout(Duration::from_secs(5), lock.write_owned()).await + /// The total wait includes any ongoing close and cell reacquisition. + async fn acquire_write_lock(&self, replica_id: u64) -> anyhow::Result { + timeout(Duration::from_secs(5), registry::Pin::write(&self.hosts, replica_id)).await? } - /// Acquire a read lock on the [HostCell] for `replica_id`. - /// - /// This will time out after 5s to aid debugging of - /// https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - async fn acquire_read_lock(&self, replica_id: u64) -> Result>, Elapsed> { - let lock = self.hosts.lock().entry(replica_id).or_default().clone(); - timeout(Duration::from_secs(5), lock.read_owned()).await + async fn acquire_read_lock(&self, replica_id: u64) -> anyhow::Result { + timeout(Duration::from_secs(5), registry::Pin::read(&self.hosts, replica_id)).await? } - async fn try_init_host(&self, database: Database, replica_id: u64) -> anyhow::Result { + async fn try_init_host( + &self, + database: Database, + replica_id: u64, + guard: ®istry::WriteGuard, + ) -> anyhow::Result { let database_identity = database.database_identity; record_module_host_init_attempt(database_identity); - Host::try_init(self, database, replica_id) - .await + let result = Host::try_init(self, database, replica_id, guard.registration()).await; + if result.as_ref().is_err_and(lifecycle::writer_unconfirmed) { + guard.quarantine(); + } + result .inspect_err(|error| { let cause = HostInitError::metric_cause(error); match cause { @@ -1033,36 +1067,31 @@ fn repair_stale_view_backing_tables_on_launch(launched: &LaunchedModule) -> anyh /// If the `db` is not initialized yet (i.e. its program hash is `None`), /// return an error. /// -/// Otherwise, if `db.program_hash` matches the given `program_hash`, do -/// nothing and return an empty `UpdateDatabaseResult`. -/// -/// Otherwise, invoke `module.update_database` and return the result. +/// Admission, including unchanged programs and idempotent deployment retries, +/// is serialized inside the migration transaction. async fn update_module( db: &RelationalDB, module: &ModuleHost, program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { let addr = db.database_identity(); match stored_program_hash(db)? { None => Err(anyhow!("database `{addr}` not yet initialized")), Some(stored) => { - let res = if stored == program.hash { - info!("database `{}` up to date with program `{}`", addr, program.hash); - UpdateDatabaseResult::NoUpdateNeeded - } else { - info!("updating `{}` from {} to {}", addr, stored, program.hash); - module.update_database(program, old_module_info, policy).await? - }; - - Ok(res) + info!("publishing `{}` from {} to {}", addr, stored, program.hash); + module + .update_database_with_deployment(program, old_module_info, policy, deployment) + .await } } } /// Encapsulates a database, associated module, and auxiliary state. struct Host { + registration: Registration, /// The [`ModuleHost`], providing the callable reducer API. /// /// Modules may be updated via [`Host::update_module`]. @@ -1088,18 +1117,23 @@ struct Host { /// Handle to the task responsible for cleaning up old views. /// The task is aborted when [`Host`] is dropped. view_cleanup_task: AbortHandle, + + // Runtime that owns this host and its physical shutdown. + runtime: spacetimedb_runtime::Handle, } impl Host { /// Attempt to instantiate a [`Host`] from persistent storage. /// - /// Note that this does **not** run module initialization routines, but may - /// create on-disk artifacts if the host / database did not exist. + /// This executes the stored module and initializes a new database when + /// necessary. It may create on-disk artifacts. Retained cleanup uses the + /// separate metadata-only open path instead. #[tracing::instrument(level = "debug", skip_all)] async fn try_init( host_controller: &HostController, database: Database, replica_id: u64, + registration: Registration, ) -> anyhow::Result { let database_identity = database.database_identity; let HostController { @@ -1108,8 +1142,6 @@ impl Host { program_storage, energy_monitor, runtimes, - persistence, - page_pool, bsatn_rlb_pool, memory_observer, .. @@ -1119,234 +1151,230 @@ impl Host { let (tx_metrics_queue, tx_metrics_recorder_task) = spawn_tx_metrics_recorder(&runtime); let tx_metrics_recorder_task = guard(tx_metrics_recorder_task, |task| task.abort()); - let (db, connected_clients) = match config.storage { - db::Storage::Memory => RelationalDB::open( - database.database_identity, - database.owner_identity, - EmptyHistory::new(), - None, - Some(tx_metrics_queue), - page_pool.clone(), - )?, - db::Storage::Disk => { - // Replay from the local state. - let history = relational_db::local_history(&replica_dir, &runtime).await?; - let persistence_db = db::persistence::Database { - id: database.id, - database_identity: database.database_identity, - owner_identity: database.owner_identity, - }; - let persistence = persistence.persistence(&persistence_db, replica_id).await?; - // Loading a database from persistent storage involves heavy - // blocking I/O. `asyncify` to avoid blocking the async worker. - let (db, clients) = asyncify({ - let database_identity = database.database_identity; - let owner_identity = database.owner_identity; - let page_pool = page_pool.clone(); - move || { - RelationalDB::open( - database_identity, - owner_identity, - history, - Some(persistence), - Some(tx_metrics_queue), - page_pool, - ) - } - }) - .await - // Make sure we log the source chain of the error - // as a single line, with the help of `anyhow`. - .map_err(anyhow::Error::from) - .inspect_err(|e| { - tracing::error!( - database = %database.database_identity, - replica = replica_id, - "Failed to open database: {e:#}" - ); - })?; - - (db, clients) - } - }; - let db = db.with_memory_observer(memory_observer.clone()); - let (mut program, program_needs_init) = match db.program()? { - // Launch module with program from existing database. - Some(program) => { - info!( - "loaded program {} from the database host-type={}", - program.hash, - HostType::from(program.kind) - ); - (program, false) - } - // Database is empty, load program from external storage and run - // initialization. - None => { - info!( - "loading program {} from external storage host-type={}", - database.initial_program, database.host_type - ); - let program_bytes = load_program(program_storage, database.initial_program).await?; - let program = Program { - hash: database.initial_program, - bytes: program_bytes, - kind: database.host_type.into(), - }; - (program, true) - } - }; + let (db, connected_clients, joined) = + lifecycle::open_database(host_controller, &database, replica_id, Some(tx_metrics_queue), &runtime).await?; let bootstrap_generation = database.bootstrap_generation; let mut bootstrap_completion = Some(BootstrapCompletion::durable(bootstrap_generation)); - - let relational_db = Arc::new(db); - let (program, launched) = match HostType::from(program.kind) { - HostType::Js => { - ModuleLauncher { - database, - replica_id, - program, - on_panic: host_controller.unregister_fn(replica_id, database_identity), - relational_db, - energy_monitor: energy_monitor.clone(), - memory_observer: memory_observer.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), + let initialized = std::panic::AssertUnwindSafe(async { + let (mut program, program_needs_init, initial_deployment) = match db.program()? { + // Launch module with program from existing database. + Some(program) => { + info!( + "loaded program {} from the database host-type={}", + program.hash, + HostType::from(program.kind) + ); + (program, false, None) } - .launch_module() - .await? - } - HostType::Wasm => { - // Prior to https://github.com/clockworklabs/SpacetimeDB/pull/4549 - // the host type in `st_module` was always set to wasm. - // We now correctly use the host type from the database, but the - // module may in fact be a JS module. - // So if launching it as a wasm module fails, try JS instead. - // If this succeeds, the module is definitely a JS module, so - // attempt to repair `st_module` in this case. - // - // TODO: This code should eventually be removed once all - // databases have been repaired. - let launch_wasm_result = ModuleLauncher { - database: database.clone(), - replica_id, - program: program.clone(), - on_panic: host_controller.unregister_fn(replica_id, database_identity), - relational_db: relational_db.clone(), - energy_monitor: energy_monitor.clone(), - memory_observer: memory_observer.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.clone().module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), + // Database is empty, load program from external storage and run + // initialization. + None => { + info!( + "loading program {} from external storage host-type={}", + database.initial_program, database.host_type + ); + let initial_deployment = program_storage.initial_deployment(&database).await?; + let program_bytes = load_program(program_storage, database.initial_program).await?; + let program = Program { + hash: database.initial_program, + bytes: program_bytes, + kind: database.host_type.into(), + }; + (program, true, initial_deployment) } - .launch_module() - .await; - match launch_wasm_result { - Ok(program_and_module_host) => program_and_module_host, - Err(e) => { - warn!("failed to launch wasm module, trying js: {e:#}"); - - program.kind = ModuleKind::JS; - let res = ModuleLauncher { - database, - replica_id, - program: program.clone(), - on_panic: host_controller.unregister_fn(replica_id, database_identity), - relational_db: relational_db.clone(), - energy_monitor: energy_monitor.clone(), - memory_observer: memory_observer.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), - } - .launch_module() - .await; + }; - if res.is_ok() { - let _ = relational_db - .with_auto_commit(Workload::Internal, |tx| relational_db.update_program(tx, program)); + let relational_db = db.clone(); + let (program, launched) = match HostType::from(program.kind) { + HostType::Js => { + ModuleLauncher { + database, + replica_id, + program, + on_panic: host_controller.unregister_fn(registration.clone(), database_identity), + relational_db, + energy_monitor: energy_monitor.clone(), + memory_observer: memory_observer.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await? + } + HostType::Wasm => { + // Prior to https://github.com/clockworklabs/SpacetimeDB/pull/4549 + // the host type in `st_module` was always set to wasm. + // We now correctly use the host type from the database, but the + // module may in fact be a JS module. + // Retry JS only for an existing stored module whose database + // declaration explicitly identifies it as JS. A new publication + // or declared Wasm module must preserve its Wasm validation error. + // If the legacy retry succeeds, repair `st_module`. + // + // TODO: This code should eventually be removed once all + // databases have been repaired. + let launch_wasm_result = ModuleLauncher { + database: database.clone(), + replica_id, + program: program.clone(), + on_panic: host_controller.unregister_fn(registration.clone(), database_identity), + relational_db: relational_db.clone(), + energy_monitor: energy_monitor.clone(), + memory_observer: memory_observer.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.clone().module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await; + match launch_wasm_result { + Ok(program_and_module_host) => program_and_module_host, + Err(e) => { + if program_needs_init || database.host_type != HostType::Js { + return Err(e); + } + warn!("failed to launch wasm module, trying js: {e:#}"); + + program.kind = ModuleKind::JS; + let res = ModuleLauncher { + database, + replica_id, + program: program.clone(), + on_panic: host_controller.unregister_fn(registration.clone(), database_identity), + relational_db: relational_db.clone(), + energy_monitor: energy_monitor.clone(), + memory_observer: memory_observer.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await; + + if res.is_ok() { + let _ = relational_db.with_auto_commit(Workload::Internal, |tx| { + relational_db.update_program(tx, program) + }); + } + + res.map_err(|js_error| { + e.context(format!("legacy JS host-type repair also failed: {js_error:#}")) + })? } - - res? } } - } - }; + }; - if program_needs_init { - let InitDatabaseResult { reducer, tx_offset } = launched.module_host.init_database(program).await?; - if let Some(call_result) = reducer { - validate_init_reducer_call_result(call_result)?; + if program_needs_init { + let InitDatabaseResult { reducer, tx_offset } = launched + .module_host + .init_database_with_deployment(program, initial_deployment) + .await?; + if let Some(call_result) = reducer { + validate_init_reducer_call_result(call_result)?; + } + bootstrap_completion = Some(BootstrapCompletion::pending( + bootstrap_generation, + tx_offset, + launched.module_host.durable_tx_offset(), + )); + } else { + repair_stale_view_backing_tables_on_launch(&launched)?; + drop(program) } - bootstrap_completion = Some(BootstrapCompletion::pending( - bootstrap_generation, - tx_offset, - launched.module_host.durable_tx_offset(), - )); - } else { - repair_stale_view_backing_tables_on_launch(&launched)?; - drop(program) - } - let LaunchedModule { - replica_ctx, - module_host, - scheduler, - scheduler_starter, - } = launched; - - // Disconnect dangling clients. - // No need to clear view tables here since we do it in `clear_all_clients`. - for (identity, connection_id) in connected_clients { - module_host - .call_identity_disconnected(identity, connection_id) - .await - .with_context(|| { - format!( - "Error calling disconnect for {} {} on {}", - identity, connection_id, replica_ctx.database_identity - ) - })?; - } - // We should have no clients left, but we do this just in case. - // This should only matter if we crashed with something in st_client_credentials, - // then restarted with an older version of the code that doesn't use st_client_credentials. - // That case would cause some permanently dangling st_client_credentials. - // Since we have no clients on startup, this should be safe to do regardless. - module_host.clear_all_clients().await?; - - scheduler_starter.start(&module_host)?; - let disk_metrics_recorder_task: spacetimedb_runtime::AbortHandle = - tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle().into(); - let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone(), &runtime); - - let module = watch::Sender::new(module_host); - let tx_metrics_recorder_task = scopeguard::ScopeGuard::into_inner(tx_metrics_recorder_task); - - Ok(HostInit { - host: Host { - module, + let LaunchedModule { replica_ctx, + module_host, scheduler, - disk_metrics_recorder_task, - tx_metrics_recorder_task, - view_cleanup_task, - }, - bootstrap_completion, + scheduler_starter, + } = launched; + + // Disconnect dangling clients. + // No need to clear view tables here since we do it in `clear_all_clients`. + for (identity, connection_id) in connected_clients { + module_host + .call_identity_disconnected(identity, connection_id) + .await + .with_context(|| { + format!( + "Error calling disconnect for {} {} on {}", + identity, connection_id, replica_ctx.database_identity + ) + })?; + } + // We should have no clients left, but we do this just in case. + // This should only matter if we crashed with something in st_client_credentials, + // then restarted with an older version of the code that doesn't use st_client_credentials. + // That case would cause some permanently dangling st_client_credentials. + // Since we have no clients on startup, this should be safe to do regardless. + module_host.clear_all_clients().await?; + + // Scheduled work can run before this Host is installed in its cell. + // Its callback must already identify this executable; the close + // owner will wait for our pinned guard before taking the Host. + registration.activate(); + scheduler_starter.start(&module_host)?; + #[cfg(test)] + if lifecycle::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + .lock() + .remove(&replica_ctx.database_identity) + { + host_controller.unregister_fn(registration.clone(), database_identity)(); + } + let disk_metrics_recorder_task: AbortHandle = + tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle().into(); + let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone(), &runtime); + + let module = watch::Sender::new(module_host); + + let tx_metrics_recorder_task = scopeguard::ScopeGuard::into_inner(tx_metrics_recorder_task); + Ok(HostInit { + host: Host { + registration, + runtime: runtime.clone(), + module, + replica_ctx, + scheduler, + disk_metrics_recorder_task, + tx_metrics_recorder_task, + view_cleanup_task, + }, + bootstrap_completion, + }) }) + .catch_unwind() + .await; + match initialized { + Ok(Ok(host)) => Ok(host), + Ok(Err(error)) => { + if let Some(joined) = joined { + joined.join().await?; + } + drop(db); + Err(error) + } + Err(panic) => { + if let Some(joined) = joined { + joined.join().await?; + } + drop(db); + std::panic::resume_unwind(panic) + } + } } /// Construct an in-memory instance of `database` running `program`. @@ -1414,9 +1442,12 @@ impl Host { runtimes: Arc, program: Program, policy: MigrationPolicy, + deployment: Option, energy_monitor: Arc, on_panic: impl Fn() + Send + Sync + 'static, + registration: Registration, core: AllocatedJobCore, + database_committed: &mut bool, ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); @@ -1435,43 +1466,72 @@ impl Host { // Get the old module info to diff against when building a migration plan. let old_module_info = self.module.borrow().info.clone(); - let update_result = - update_module(replica_ctx.relational_db(), &module, program, old_module_info, policy).await?; + let update_result = update_module( + replica_ctx.relational_db(), + &module, + program, + old_module_info, + policy, + deployment, + ) + .await?; + + *database_committed = matches!( + update_result, + UpdateDatabaseResult::UpdatePerformed { .. } + | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } + ); + + #[cfg(test)] + if *database_committed + && FAIL_NEXT_DEPLOYMENT_ACTIVATION + .lock() + .remove(&replica_ctx.database_identity) + { + bail!("injected scheduler activation failure after deployment commit"); + } // Only replace the module + scheduler if the update succeeded. // Otherwise, we want the database to continue running with the old state. match update_result { UpdateDatabaseResult::NoUpdateNeeded | UpdateDatabaseResult::UpdatePerformed { .. } => { - self.scheduler = scheduler; + registration.activate(); scheduler_starter.start(&module)?; + self.scheduler = scheduler; + self.registration = registration.clone(); let old_module = self.module.send_replace(module); old_module.exit().await; } // In this case, we need to disconnect all clients connected to the old module UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } => { + self.registration = registration; + self.registration.activate(); // Replace the module first, so that new clients get the new module. let old_watcher = std::mem::replace(&mut self.module, watch::Sender::new(module.clone())); + let old_module = old_watcher.borrow().clone(); // Disconnect all clients connected to the old module. - let connected_clients = replica_ctx.relational_db().connected_clients()?; - for (identity, connection_id) in connected_clients { - let client_actor_id = ClientActorId { - identity, - connection_id, - name: ClientName(0), - }; - //NOTE: This will call disconnect reducer of the new module, not the old one. - //It makes sense, as relationaldb is already updated to the new module. - module.disconnect_client(client_actor_id).await; + let activation = async { + let connected_clients = replica_ctx.relational_db().connected_clients()?; + for (identity, connection_id) in connected_clients { + let client_actor_id = ClientActorId { + identity, + connection_id, + name: ClientName(0), + }; + // Disconnect uses the newly committed module. + module.disconnect_client(client_actor_id).await; + } + scheduler_starter.start(&module)?; + Ok::<_, anyhow::Error>(()) } - - self.scheduler = scheduler; - scheduler_starter.start(&module)?; + .await; // exit the old module, drop the `old_watcher` afterwards, // which will signal websocket clients that the module is gone. - let old_module = old_watcher.borrow().clone(); old_module.exit().await; + activation?; + self.scheduler = scheduler; } _ => {} } @@ -1710,3 +1770,6 @@ mod tests { assert_eq!(HostInitError::metric_cause(&error), ModuleHostInitFailureCause::Other); } } + +#[cfg(test)] +mod invocation_flags_tests; diff --git a/crates/core/src/host/host_controller/deployment_tests.rs b/crates/core/src/host/host_controller/deployment_tests.rs new file mode 100644 index 00000000000..e67233e4c47 --- /dev/null +++ b/crates/core/src/host/host_controller/deployment_tests.rs @@ -0,0 +1,215 @@ +use super::*; +use crate::db::deployment::{current_deployment, install_publication_fence}; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::empty_module; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ST_DEPLOYMENT_OPERATION_ID, ST_PUBLISH_FENCE_ID}; +use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; +use spacetimedb_lib::{hash_bytes, Uuid}; +use spacetimedb_paths::FromPathUnchecked; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +struct InitialStorage { + program: Program, + request: Option, + initial_lookups: AtomicUsize, +} + +#[async_trait] +impl ExternalStorage for InitialStorage { + async fn lookup(&self, hash: Hash) -> anyhow::Result>> { + Ok((self.program.hash == hash).then(|| self.program.bytes.clone())) + } + async fn initial_deployment(&self, _: &Database) -> anyhow::Result> { + self.initial_lookups.fetch_add(1, Ordering::SeqCst); + Ok(self.request.clone()) + } +} + +fn request(program: &Program, sequence: u64, initial: bool) -> DeploymentCommit { + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + DeploymentCommit { + operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(sequence)), + publication_epoch: sequence, + publisher: Identity::ONE, + expected_revision: None, + prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: if initial { + ModuleComponent::SystemEmpty(1) + } else { + ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: program.hash, + }) + }, + container: Some(ContainerSpec { + image_manifest: OciDigest::sha256([4; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 1000, + }), + }), + } +} + +fn fixture(id: u64, bootstrap: bool) -> (tempfile::TempDir, HostController, Database, Arc) { + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = empty_module::program(1).unwrap(); + let storage = Arc::new(InitialStorage { + request: bootstrap.then(|| request(&program, 1, true)), + program: program.clone(), + initial_lookups: AtomicUsize::new(0), + }); + let controller = HostController::new( + data_dir.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + storage.clone(), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + bootstrap_generation: 0, + }; + (directory, controller, database, storage) +} + +#[tokio::test(flavor = "multi_thread")] +async fn bootstrap_commits_fence_program_and_deployment_and_reopens_from_disk() { + let (_directory, controller, database, storage) = fixture(0xdd01, true); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let expected = storage.request.as_ref().unwrap().deployment.revision().unwrap(); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, expected); + assert_eq!(tx.table_row_count(ST_PUBLISH_FENCE_ID), Some(1)); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + assert_eq!( + module.relational_db().program().unwrap().unwrap().hash, + database.initial_program + ); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!( + storage.initial_lookups.load(Ordering::SeqCst), + 1, + "replay must not re-run bootstrap intent" + ); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, expected); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn activation_failure_after_commit_closes_old_host_and_recovers_committed_program() { + let (_directory, controller, database, _storage) = fixture(0xdd02, false); + let old_module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let watcher = controller.watch_module_host(database.id).await.unwrap(); + let mut bytes = empty_module::VERSION_1_BYTES.to_vec(); + bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let newer = Program::from_bytes(ModuleKind::WASM, bytes); + let publication = request(&newer, 1, false); + old_module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, publication.publication_epoch, publication.operation_id) + }) + .unwrap(); + FAIL_NEXT_DEPLOYMENT_ACTIVATION + .lock() + .insert(database.database_identity); + let error = controller + .update_module_host_with_deployment( + database.clone(), + HostType::Wasm, + database.id, + newer.bytes.clone(), + MigrationPolicy::Compatible, + Some(publication.clone()), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("injected scheduler activation failure")); + assert!( + controller.get_module_host(database.id).await.is_err(), + "old executable must not remain available" + ); + assert!(watcher.has_changed().is_err(), "old client watcher must close"); + assert_eq!(old_module.relational_db().program().unwrap().unwrap().hash, newer.hash); + old_module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!( + current_deployment(tx).unwrap().unwrap().0, + publication.deployment.revision().unwrap() + ); + }); + drop(watcher); + drop(old_module); + let recovered = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(recovered.info.module_hash, newer.hash); + recovered.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!( + current_deployment(tx).unwrap().unwrap().0, + publication.deployment.revision().unwrap() + ); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + drop(recovered); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} diff --git a/crates/core/src/host/host_controller/execution_deadline_tests.rs b/crates/core/src/host/host_controller/execution_deadline_tests.rs new file mode 100644 index 00000000000..c1ca0ca1821 --- /dev/null +++ b/crates/core/src/host/host_controller/execution_deadline_tests.rs @@ -0,0 +1,413 @@ +//! Actual V8 module hosts, transactions and updates. No network or external service. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::FunctionArgs; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_lib::db::raw_def::{v10::RawModuleDefV10Builder, v9::Lifecycle}; +use spacetimedb_paths::FromPathUnchecked; +use spacetimedb_sats::{AlgebraicType, ProductType}; + +fn config() -> HostRuntimeConfig { + HostRuntimeConfig { + v8: V8Config { + execution_timeout: Duration::from_millis(200), + ..V8Config::default() + }, + ..HostRuntimeConfig::default() + } +} + +fn program(loop_in_first_init: bool) -> Program { + program_with_startup_cutoff(loop_in_first_init, u64::MAX) +} + +fn program_with_startup_cutoff(loop_in_first_init: bool, cutoff_millis: u64) -> Program { + let mut schema = RawModuleDefV10Builder::new(); + schema + .build_table_with_new_type("rows", [("value", AlgebraicType::U64)], true) + .finish(); + schema.add_lifecycle_reducer(Lifecycle::Init, "init", ProductType::unit()); + schema.add_reducer("loop", ProductType::unit()); + schema.add_reducer("good", ProductType::unit()); + schema.add_procedure("task", ProductType::unit(), AlgebraicType::U64); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{ register_hooks, table_id_from_name, datastore_insert_bsatn }} from "spacetime:sys@1.0"; + import {{ register_hooks as register_procedure_hooks }} from "spacetime:sys@1.2"; + if (Date.now() > {cutoff_millis}) {{ for (;;) {{}} }} + let initCalls = 0; + let nextValue = 0; + register_hooks({{ + __describe_module__: function() {{ return new Uint8Array({schema:?}); }}, + __call_reducer__: function(id) {{ + const row = new Uint8Array(8); + row[0] = ++nextValue; + datastore_insert_bsatn(table_id_from_name("rows"), row); + if (id === 1 || (id === 0 && ++initCalls === 1 && {loop_in_first_init})) {{ for (;;) {{}} }} + return {{ tag: "ok" }}; + }}, + }}); + let retained; + register_procedure_hooks({{ __call_procedure__: function() {{ + if (Date.now() > {cutoff_millis}) {{ retained = new Array(4 * 1024 * 1024).fill(1); }} + return new Uint8Array(8); + }} }}); + "# + ) + .into_bytes(), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_procedure_startup_failure_keeps_main_module_registered() { + use std::time::{SystemTime, UNIX_EPOCH}; + let cutoff = SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + Duration::from_secs(3); + let program = program_with_startup_cutoff(false, cutoff.as_millis() as u64); + let (_directory, controller, database) = controller_fixture( + 0xed05, + &program, + HostRuntimeConfig { + v8: V8Config { + // One slot makes a leaked admission permit observable on retry. + procedure_instance_pool_size: std::num::NonZeroUsize::new(1).unwrap(), + ..config().v8 + }, + ..config() + }, + ); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert!(call(&module, "good").await.is_ok()); + let until_cutoff = cutoff.saturating_sub(SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + tokio::time::sleep(until_cutoff + Duration::from_millis(50)).await; + // The module's main isolate already ran startup. Only newly created + // procedure isolates encounter the now-hostile startup branch. + for _ in 0..2 { + let result = timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary), + ) + .await + .unwrap(); + let error = result.result.unwrap_err(); + assert!(error.to_string().contains("wall-clock limit"), "{error}"); + let registered = controller.get_module_host(database.id).await.unwrap(); + assert!(call(®istered, "good").await.is_ok()); + } + assert_eq!(row_count(&module), Some(4)); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +fn database(id: u64, program: &Program) -> Database { + Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Js, + initial_program: program.hash, + bootstrap_generation: 0, + } +} + +fn controller_fixture( + id: u64, + program: &Program, + config: HostRuntimeConfig, +) -> (tempfile::TempDir, HostController, Database) { + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let initial = program.clone(); + let storage = move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }; + let controller = HostController::new( + data_dir.clone(), + db::Config { + storage: db::Storage::Memory, + page_pool_max_size: None, + }, + config, + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + (directory, controller, database(id, program)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_failed_procedure_recreation_keeps_main_module_registered() { + use std::time::{SystemTime, UNIX_EPOCH}; + let cutoff = SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + Duration::from_secs(3); + let program = program_with_startup_cutoff(false, cutoff.as_millis() as u64); + let (_directory, controller, database) = controller_fixture( + 0xed06, + &program, + HostRuntimeConfig { + v8: V8Config { + procedure_instance_pool_size: std::num::NonZeroUsize::new(1).unwrap(), + heap_policy: crate::config::V8HeapPolicyConfig { + heap_limit_bytes: 64 * 1024 * 1024, + heap_check_request_interval: Some(1), + heap_gc_trigger_fraction: 0.2, + heap_retire_fraction: 0.2, + ..Default::default() + }, + ..config().v8 + }, + ..config() + }, + ); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + // Populate the one-slot procedure pool before the startup branch changes. + assert!(module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + .result + .is_ok()); + let remaining = cutoff.saturating_sub(SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + tokio::time::sleep(remaining + Duration::from_millis(50)).await; + // This invocation runs in the existing isolate and leaves32MiB alive. The + // real post-call heap policy retires it and tries a new, now-looping startup. + assert!(module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + .result + .is_ok()); + let result = timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary), + ) + .await + .unwrap(); + let error = result.result.unwrap_err(); + assert!( + error.to_string().contains("procedure isolate startup failed"), + "{error}" + ); + assert!(error.to_string().contains("wall-clock limit"), "{error}"); + let registered = controller.get_module_host(database.id).await.unwrap(); + assert!(call(®istered, "good").await.is_ok()); + // A subsequent checkout also completes, proving the dead instance's slot + // was released instead of leaked after its replacement failed. + assert!(timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary,) + ) + .await + .unwrap() + .result + .is_err()); + assert!(call(®istered, "good").await.is_ok()); + drop(registered); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +async fn launch(id: u64, program: Program) -> anyhow::Result<(Program, LaunchedModule)> { + timeout( + Duration::from_secs(5), + Host::try_init_in_memory_to_check( + &HostRuntimes::new(None, config()), + PagePool::new(None), + database(id, &program), + program, + AllocatedJobCore::default(), + BsatnRowListBuilderPool::new(), + ), + ) + .await? +} + +fn row_count(module: &ModuleHost) -> Option { + module.relational_db().with_read_only(Workload::Internal, |tx| { + tx.table_id_from_name("rows") + .unwrap() + .and_then(|table| tx.table_row_count(table)) + }) +} + +async fn call(module: &ModuleHost, name: &str) -> ReducerCallResult { + timeout( + Duration::from_secs(5), + module.call_reducer(Identity::ONE, None, None, None, None, name, FunctionArgs::Nullary), + ) + .await + .unwrap() + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_rolls_back_init_and_reducer_and_preserves_isolate() { + let (program, launched) = launch(0xed01, program(true)).await.unwrap(); + let module = launched.module_host; + let result = module.init_database(program.clone()).await.unwrap().reducer.unwrap(); + assert!(result.is_err(), "infinite init must fail"); + assert!(module.relational_db().program().unwrap().is_none()); + assert_eq!( + row_count(&module), + None, + "failed init must roll back the schema and inserted row" + ); + + assert!(module.init_database(program).await.unwrap().reducer.unwrap().is_ok()); + assert_eq!(row_count(&module), Some(1)); + let failed = call(&module, "loop").await; + assert!(failed.is_err()); + assert_eq!( + row_count(&module), + Some(1), + "timed-out reducer must roll back its write" + ); + + for _ in 0..64 { + assert!(call(&module, "good").await.is_ok()); + } + // Cross the original timer boundary before reusing the same isolate again. + tokio::time::sleep(Duration::from_millis(250)).await; + assert!(call(&module, "good").await.is_ok()); + assert_eq!(row_count(&module), Some(66)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_bounds_startup_description_and_failed_update() { + let valid = program(false); + let looping_startup = Program::from_bytes(ModuleKind::JS, b"for (;;) {}".to_vec()); + let looping_description = Program::from_bytes( + ModuleKind::JS, + br#"import {register_hooks} from "spacetime:sys@1.0"; + register_hooks({__describe_module__: function() {for (;;) {}}, __call_reducer__: function() {}});"# + .to_vec(), + ); + for (id, bad) in [(0xed02, &looping_startup), (0xed03, &looping_description)] { + let error = match launch(id, bad.clone()).await { + Ok(_) => panic!("infinite JavaScript unexpectedly launched"), + Err(error) => error, + }; + assert!(format!("{error:#}").contains("wall-clock limit"), "{error:#}"); + } + + let (_directory, controller, database) = controller_fixture(0xed04, &valid, config()); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(row_count(&module), Some(1)); + for bad in [looping_startup, looping_description] { + assert!(timeout( + Duration::from_secs(5), + controller.update_module_host( + database.clone(), + HostType::Js, + database.id, + bad.bytes, + MigrationPolicy::Compatible, + ) + ) + .await + .unwrap() + .is_err()); + assert_eq!(module.relational_db().program().unwrap().unwrap().hash, valid.hash); + assert!(call(&module, "good").await.is_ok()); + } + let mut newer = valid.bytes.to_vec(); + newer.extend_from_slice(b"\n// Valid replacement after two deadline failures.\n"); + let newer = Program::from_bytes(ModuleKind::JS, newer); + controller + .update_module_host( + database.clone(), + HostType::Js, + database.id, + newer.bytes.clone(), + MigrationPolicy::Compatible, + ) + .await + .unwrap(); + let replacement = controller.get_module_host(database.id).await.unwrap(); + assert_eq!(replacement.relational_db().program().unwrap().unwrap().hash, newer.hash); + assert!(call(&replacement, "good").await.is_ok()); + drop(replacement); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn javascript_logging_handles_direct_startup_and_preserves_wrapped_call_locations() { + use futures::TryStreamExt as _; + + let mut schema = RawModuleDefV10Builder::new(); + schema.add_reducer("log", ProductType::unit()); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + // Direct top-level logging has exactly one JS frame. It previously passed + // index1 to V8's unchecked GetFrame and could crash the entire host process. + let source = format!( + r#"import {{ register_hooks, console_log }} from "spacetime:sys@1.0"; +console_log(2, "direct-startup"); +function wrapped(message) {{ console_log(2, message); }} +wrapped("wrapped-startup"); +register_hooks({{ + __describe_module__: () => new Uint8Array({schema:?}), + __call_reducer__: () => {{ + console_log(2, "direct-reducer"); + wrapped("wrapped-reducer"); + return {{ tag: "ok" }}; + }}, +}}); +"# + ); + let program = Program::from_bytes(ModuleKind::JS, source.into_bytes()); + let (_directory, controller, database) = controller_fixture(0xed08, &program, config()); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert!(call(&module, "log").await.is_ok()); + let chunks: Vec<_> = module + .database_logger() + .tail(None, false) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let bytes = chunks.into_iter().flatten().collect::>(); + let records = String::from_utf8(bytes) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + for (message, line) in [ + ("direct-startup", 2), + ("wrapped-startup", 4), + ("direct-reducer", 8), + ("wrapped-reducer", 9), + ] { + let record = records.iter().find(|record| record["message"] == message).unwrap(); + assert_eq!(record["line_number"], line, "wrong call location for {message}"); + assert!(record["filename"].as_str().is_some_and(|filename| !filename.is_empty())); + } +} diff --git a/crates/core/src/host/host_controller/lifecycle.rs b/crates/core/src/host/host_controller/lifecycle.rs new file mode 100644 index 00000000000..e29871a66d2 --- /dev/null +++ b/crates/core/src/host/host_controller/lifecycle.rs @@ -0,0 +1,157 @@ +//! Shared physical writer completion for ordinary module initialization and +//! shutdown. Provider snapshot and archival services keep their own ownership. + +use super::*; +use crate::db::persistence::Persistence; +use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; +use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; +use std::panic::{resume_unwind, AssertUnwindSafe}; +use std::sync::OnceLock; + +#[derive(Debug, thiserror::Error)] +pub(super) enum CloseFailure { + #[error("module exit panicked; the storage writer was positively closed")] + ModuleExit, + #[error("storage writer close panicked; the replica remains quarantined")] + WriterUnconfirmed, +} + +#[cfg(test)] +pub(super) static FAIL_NEXT_MODULE_EXIT: parking_lot::Mutex> = + parking_lot::Mutex::new(std::collections::BTreeSet::new()); + +#[cfg(test)] +pub(super) static PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START: parking_lot::Mutex> = + parking_lot::Mutex::new(std::collections::BTreeSet::new()); + +pub(super) fn writer_unconfirmed(error: &anyhow::Error) -> bool { + matches!( + error.downcast_ref::(), + Some(CloseFailure::WriterUnconfirmed) + ) +} + +/// Some providers hand the first close caller the writer's JoinHandle and let +/// later calls finish immediately. Share that *first* physical close instead. +/// In particular, partial RelationalDB replay may start close from Drop before +/// our owning task sees its error. That task must join the same close future. +pub(super) struct JoinedDurability { + inner: Arc>, + close: OnceLock>>>, +} + +impl JoinedDurability { + pub fn wrap(persistence: &mut Persistence) -> Arc { + let joined = Arc::new(Self { + inner: persistence.durability.clone(), + close: OnceLock::new(), + }); + persistence.durability = joined.clone(); + joined + } + + pub async fn join(&self) -> Result, CloseFailure> { + AssertUnwindSafe(async { self.close().await }) + .catch_unwind() + .await + .map_err(|_| CloseFailure::WriterUnconfirmed) + } +} + +impl Durability for JoinedDurability { + type TxData = Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.inner.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.inner.durable_tx_offset() + } + fn close(&self) -> Close { + self.close.get_or_init(|| self.inner.close().shared()).clone().boxed() + } +} + +/// A failed replay joins the same physical writer close before another normal +/// initialization can acquire the replica's canonical registry cell. +pub(super) async fn open_database( + controller: &HostController, + database: &Database, + replica_id: u64, + tx_metrics_queue: Option, + runtime: &spacetimedb_runtime::Handle, +) -> anyhow::Result<( + Arc, + relational_db::ConnectedClients, + Option>, +)> { + if matches!(controller.default_config.storage, db::Storage::Memory) { + let (db, clients) = RelationalDB::open( + database.database_identity, + database.owner_identity, + EmptyHistory::new(), + None, + tx_metrics_queue, + controller.page_pool.clone(), + )?; + return Ok(( + Arc::new(db.with_memory_observer(controller.memory_observer.clone())), + clients, + None, + )); + } + + let replica_dir = controller.data_dir.replica(replica_id); + let history = relational_db::local_history(&replica_dir, runtime).await?; + let persistence_db = crate::db::persistence::Database { + id: database.id, + database_identity: database.database_identity, + owner_identity: database.owner_identity, + }; + let mut persistence = controller.persistence.persistence(&persistence_db, replica_id).await?; + let joined = JoinedDurability::wrap(&mut persistence); + let identity = database.database_identity; + let owner = database.owner_identity; + let page_pool = controller.page_pool.clone(); + let opened = AssertUnwindSafe(asyncify(move || { + RelationalDB::open(identity, owner, history, Some(persistence), tx_metrics_queue, page_pool) + })) + .catch_unwind() + .await; + let (db, clients) = match opened { + Ok(Ok(opened)) => opened, + Ok(Err(error)) => { + joined.join().await?; + return Err(error.into()); + } + Err(panic) => { + joined.join().await?; + resume_unwind(panic); + } + }; + let db = Arc::new(db.with_memory_observer(controller.memory_observer.clone())); + Ok((db, clients, Some(joined))) +} + +pub(super) async fn close_host(host: Host) -> Result<(), CloseFailure> { + let module = host.module.borrow().clone(); + let info = module.info(); + let identity = info.database_identity; + let table_names = info.module_def.tables().map(|table| table.name.deref()); + defer!(remove_database_gauges(&identity, table_names)); + let exited = AssertUnwindSafe(async { + module.exit().await; + #[cfg(test)] + if FAIL_NEXT_MODULE_EXIT.lock().remove(&identity) { + panic!("injected module exit failure"); + } + }) + .catch_unwind() + .await; + let writer = AssertUnwindSafe(module.relational_db().shutdown_with_runtime(&host.runtime)) + .catch_unwind() + .await; + drop(host); + writer.map_err(|_| CloseFailure::WriterUnconfirmed)?; + exited.map_err(|_| CloseFailure::ModuleExit) +} diff --git a/crates/core/src/host/host_controller/lifecycle_tests.rs b/crates/core/src/host/host_controller/lifecycle_tests.rs new file mode 100644 index 00000000000..360b05862e0 --- /dev/null +++ b/crates/core/src/host/host_controller/lifecycle_tests.rs @@ -0,0 +1,403 @@ +//! Real local durability writers and module hosts. No network or external +//! configuration. The probe counts writers until their first close completes. +use super::*; +use crate::db::persistence::{LocalPersistenceProvider, Persistence}; +use crate::host::empty_module; +use futures::FutureExt; +use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; +use spacetimedb_paths::FromPathUnchecked; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use tokio::sync::Semaphore; + +struct Probe { + opened: AtomicUsize, + active: AtomicUsize, + maximum: AtomicUsize, + block_close: AtomicBool, + panic_close: AtomicBool, + close_started: Semaphore, + release_close: Semaphore, +} + +impl Default for Probe { + fn default() -> Self { + Self { + opened: AtomicUsize::new(0), + active: AtomicUsize::new(0), + maximum: AtomicUsize::new(0), + block_close: AtomicBool::new(false), + panic_close: AtomicBool::new(false), + close_started: Semaphore::new(0), + release_close: Semaphore::new(0), + } + } +} + +struct ProbedProvider { + local: LocalPersistenceProvider, + probe: Arc, +} + +#[async_trait] +impl PersistenceProvider for ProbedProvider { + async fn persistence( + &self, + database: &crate::db::persistence::Database, + replica: u64, + ) -> anyhow::Result { + let mut persistence = self.local.persistence(database, replica).await?; + let active = self.probe.active.fetch_add(1, Ordering::SeqCst) + 1; + self.probe.maximum.fetch_max(active, Ordering::SeqCst); + self.probe.opened.fetch_add(1, Ordering::SeqCst); + persistence.durability = Arc::new(ProbedWriter { + inner: persistence.durability, + probe: self.probe.clone(), + closing: AtomicBool::new(false), + }); + Ok(persistence) + } +} + +struct ProbedWriter { + inner: Arc>, + probe: Arc, + closing: AtomicBool, +} + +impl Durability for ProbedWriter { + type TxData = Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.inner.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.inner.durable_tx_offset() + } + fn close(&self) -> Close { + // Reproduce the actual first-caller-owns-join behavior deliberately. + // A second close must not let another database open before this one. + if self.closing.swap(true, Ordering::SeqCst) { + let offset = self.inner.durable_tx_offset().last_seen(); + return async move { offset }.boxed(); + } + let close = self.inner.close(); + let probe = self.probe.clone(); + async move { + probe.close_started.add_permits(1); + if probe.block_close.load(Ordering::SeqCst) { + probe.release_close.acquire().await.unwrap().forget(); + } + assert!( + !probe.panic_close.load(Ordering::SeqCst), + "injected writer close failure" + ); + let offset = close.await; + probe.active.fetch_sub(1, Ordering::SeqCst); + offset + } + .boxed() + } +} + +fn fixture( + id: u64, +) -> ( + tempfile::TempDir, + HostController, + Database, + Arc, + Arc, +) { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = empty_module::program(1).unwrap(); + let initial = program.clone(); + let lookups = Arc::new(AtomicUsize::new(0)); + let lookup_count = lookups.clone(); + let storage = move |hash| { + let initial = initial.clone(); + lookup_count.fetch_add(1, Ordering::SeqCst); + async move { Ok((hash == initial.hash).then_some(initial.bytes)) } + }; + let probe = Arc::new(Probe::default()); + let controller = HostController::new( + data.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(()), + Arc::new(ProbedProvider { + local: LocalPersistenceProvider::new(data), + probe: probe.clone(), + }), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + bootstrap_generation: 0, + }; + (directory, controller, database, probe, lookups) +} + +async fn closed_seed(controller: &HostController, database: &Database, probe: &Probe) { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + probe.close_started.acquire().await.unwrap().forget(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); +} + +async fn wait_for_close(probe: &Probe) { + timeout(Duration::from_secs(5), probe.close_started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_idle_module_and_queued_relookup_do_not_break_positive_close() { + let (_directory, controller, database, probe, _) = fixture(0xc002); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + probe.block_close.store(true, Ordering::SeqCst); + let exiting = { + let controller = controller.clone(); + tokio::spawn(async move { controller.exit_module_host(0xc002, Duration::from_secs(5)).await }) + }; + wait_for_close(&probe).await; + let lookup = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + }) + }; + assert!( + controller + .exit_module_host(database.id, Duration::from_millis(10)) + .await + .is_err(), + "a timeout is not positive closure" + ); + exiting.abort(); + assert!(exiting.await.unwrap_err().is_cancelled()); + assert_eq!(probe.opened.load(Ordering::SeqCst), 1); + assert_eq!(probe.active.load(Ordering::SeqCst), 1); + probe.block_close.store(false, Ordering::SeqCst); + probe.release_close.add_permits(1); + let current = timeout(Duration::from_secs(5), lookup).await.unwrap().unwrap().unwrap(); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + assert!(!Arc::ptr_eq(idle.relational_db(), current.relational_db())); + let seen_live = controller.get_module_host(database.id).await.unwrap(); + assert!(Arc::ptr_eq(seen_live.relational_db(), current.relational_db())); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); + drop((idle, current)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_failed_normal_initialization_joins_writer_before_retry() { + let (_directory, controller, database, probe, _) = fixture(0xc008); + closed_seed(&controller, &database, &probe).await; + let mut wrong_owner = database.clone(); + wrong_owner.owner_identity = Identity::from_u256(123456_u64.into()); + probe.block_close.store(true, Ordering::SeqCst); + let failed = { + let controller = controller.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(wrong_owner.clone(), wrong_owner.id) + .await + }) + }; + wait_for_close(&probe).await; + let next = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + }) + }; + assert!(!failed.is_finished()); + probe.release_close.add_permits(1); + assert!(failed.await.unwrap().is_err()); + let reopened = next.await.unwrap().unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + probe.block_close.store(false, Ordering::SeqCst); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + drop(reopened); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_module_exit_panic_reports_error_after_positive_writer_close_and_allows_reopen() { + let (_directory, controller, database, probe, _) = fixture(0xc00a); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + lifecycle::FAIL_NEXT_MODULE_EXIT + .lock() + .insert(database.database_identity); + let error = controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(error.to_string().contains("module exit panicked")); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + drop(idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_unconfirmed_writer_close_reports_error_and_quarantines_replica() { + let (_directory, controller, database, probe, _) = fixture(0xc00b); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + probe.panic_close.store(true, Ordering::SeqCst); + let error = controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(error.to_string().contains("writer close panicked")); + let error = timeout( + Duration::from_secs(1), + controller.get_or_launch_module_host(database.clone(), database.id), + ) + .await + .unwrap() + .expect_err("quarantined host must not reopen"); + assert!(error.to_string().contains("unable to lock")); + assert_eq!(probe.opened.load(Ordering::SeqCst), 1); + assert!(controller + .exit_module_host(database.id, Duration::from_secs(1)) + .await + .unwrap_err() + .to_string() + .contains("quarantined")); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + drop(module); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_panic_callback_before_initial_host_install_still_owns_cleanup() { + let (_directory, controller, database, probe, _) = fixture(0xc00c); + lifecycle::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + .lock() + .insert(database.database_identity); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + // No explicit exit is requested until the injected scheduler interleaving + // has independently started the actual storage writer close. + wait_for_close(&probe).await; + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.get_module_host(database.id).await.is_err()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + drop(idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn lifecycle_stale_panic_callback_does_not_unregister_updated_or_reopened_host() { + let (_directory, controller, database, probe, _) = fixture(0xc006); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let old_callback = { + let guard = controller.acquire_read_lock(database.id).await.unwrap(); + controller.unregister_fn(guard.as_ref().unwrap().registration.clone(), database.database_identity) + }; + let mut newer = empty_module::VERSION_1_BYTES.to_vec(); + newer.extend_from_slice(&[0, 3, 1, b'x', 1]); + let newer_hash = spacetimedb_lib::hash_bytes(&newer); + controller + .update_module_host( + database.clone(), + HostType::Wasm, + database.id, + newer.into(), + MigrationPolicy::Compatible, + ) + .await + .unwrap(); + old_callback(); + assert_eq!( + controller.get_module_host(database.id).await.unwrap().info.module_hash, + newer_hash + ); + assert_eq!(probe.active.load(Ordering::SeqCst), 1); + let current_callback = { + let guard = controller.acquire_read_lock(database.id).await.unwrap(); + controller.unregister_fn(guard.as_ref().unwrap().registration.clone(), database.database_identity) + }; + current_callback(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert!(controller.get_module_host(database.id).await.is_err()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + current_callback(); + assert!(controller.get_module_host(database.id).await.is_ok()); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); +} diff --git a/crates/core/src/host/host_controller/registry.rs b/crates/core/src/host/host_controller/registry.rs new file mode 100644 index 00000000000..030a1d7c3fe --- /dev/null +++ b/crates/core/src/host/host_controller/registry.rs @@ -0,0 +1,333 @@ +//! Canonical host cells, pinned by queued and active operations. +//! +//! Lock order: the registry mutex is never held across an await. A cell guard +//! may briefly acquire the registry mutex to publish its resident-host state. +//! It releases the cell guard before releasing its operation pin. The last pin +//! removes an empty, non-closing entry without waiting for unrelated Arc owners. + +use super::{Host, HostCell}; +use parking_lot::Mutex; +use spacetimedb_data_structures::map::IntMap; +use std::ops::{Deref, DerefMut}; +use std::sync::{Arc, Weak}; +use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard}; + +pub(super) type Hosts = Arc>>; + +pub(super) struct Entry { + cell: HostCell, + pins: usize, + resident: bool, + registration: Option>, + closing: Option>, +} + +/// Completion means the storage writer has actually closed, not that the +/// caller's wait expired. Provider-owned snapshot/archival services are separate. +pub(super) struct Closing { + result: watch::Sender>>>, +} + +impl Closing { + pub async fn wait(&self) -> anyhow::Result<()> { + let mut receiver = self.result.subscribe(); + let result = receiver.wait_for(Option::is_some).await?.clone().unwrap(); + result.map_err(|message| anyhow::anyhow!(message.to_string())) + } +} + +/// A pin counts an operation, including time queued for the cell lock. Idle +/// ModuleHost references are not pins. Raw cells never authorize an operation. +pub(super) struct Pin { + hosts: Weak>>, + replica: u64, + cell: HostCell, +} + +impl Pin { + pub fn acquire(hosts: &Hosts, replica: u64) -> Self { + let mut entries = hosts.lock(); + let entry = entries.entry(replica).or_insert_with(|| Entry { + cell: HostCell::default(), + pins: 0, + resident: false, + registration: None, + closing: None, + }); + entry.pins += 1; + Self { + hosts: Arc::downgrade(hosts), + replica, + cell: entry.cell.clone(), + } + } + + fn closing(&self) -> Option> { + let hosts = self.hosts.upgrade()?; + let entries = hosts.lock(); + let entry = entries.get(&self.replica)?; + debug_assert!(Arc::ptr_eq(&entry.cell, &self.cell)); + entry.closing.clone() + } + + pub async fn read(hosts: &Hosts, replica: u64) -> anyhow::Result { + loop { + let pin = Self::acquire(hosts, replica); + let guard = pin.cell.clone().read_owned().await; + if let Some(closing) = pin.closing() { + drop(guard); + drop(pin); + closing.wait().await?; + continue; + } + return Ok(ReadGuard { + guard: Some(guard), + pin: Some(pin), + }); + } + } + + pub async fn write(hosts: &Hosts, replica: u64) -> anyhow::Result { + loop { + let pin = Self::acquire(hosts, replica); + let guard = pin.cell.clone().write_owned().await; + if let Some(closing) = pin.closing() { + drop(guard); + drop(pin); + closing.wait().await?; + continue; + } + return Ok(WriteGuard { + guard: Some(guard), + pin: Some(pin), + }); + } + } + + fn publish(&self, host: Option<&Host>) { + let Some(hosts) = self.hosts.upgrade() else { return }; + let mut entries = hosts.lock(); + let Some(entry) = entries.get_mut(&self.replica) else { + return; + }; + debug_assert!(Arc::ptr_eq(&entry.cell, &self.cell)); + entry.resident = host.is_some(); + entry.registration = host.map(|host| host.registration.token.clone()); + } + + fn registration(&self) -> Registration { + Registration { + hosts: self.hosts.clone(), + replica: self.replica, + cell: Arc::downgrade(&self.cell), + token: Arc::new(()), + } + } +} + +impl Drop for Pin { + fn drop(&mut self) { + let Some(hosts) = self.hosts.upgrade() else { return }; + let removed = { + let mut entries = hosts.lock(); + let Some(entry) = entries.get_mut(&self.replica) else { + return; + }; + assert!(Arc::ptr_eq(&entry.cell, &self.cell), "replaced a pinned host cell"); + entry.pins -= 1; + if entry.pins == 0 && !entry.resident && entry.closing.is_none() { + entries.remove(&self.replica) + } else { + None + } + }; + drop(removed); + } +} + +pub(super) struct ReadGuard { + guard: Option>>, + pin: Option, +} + +impl Deref for ReadGuard { + type Target = Option; + fn deref(&self) -> &Self::Target { + self.guard.as_ref().unwrap() + } +} + +impl Drop for ReadGuard { + fn drop(&mut self) { + drop(self.guard.take()); + drop(self.pin.take()); + } +} + +pub(super) struct WriteGuard { + guard: Option>>, + pin: Option, +} + +impl WriteGuard { + pub fn registration(&self) -> Registration { + self.pin.as_ref().unwrap().registration() + } + + pub fn install(&mut self, host: Host) { + **self = Some(host); + self.pin.as_ref().unwrap().publish(self.as_ref()); + } + + pub fn quarantine(&self) { + let pin = self.pin.as_ref().unwrap(); + let Some(hosts) = pin.hosts.upgrade() else { return }; + let mut entries = hosts.lock(); + let entry = entries.get_mut(&pin.replica).expect("pinned cell exists"); + if let Some(closing) = &entry.closing { + closing + .result + .send_replace(Some(Err("storage writer close is unconfirmed".into()))); + } + entry.closing = Some(Arc::new(Closing { + result: watch::Sender::new(Some(Err( + "storage writer close is unconfirmed; replica is quarantined".into() + ))), + })); + } +} + +impl Deref for WriteGuard { + type Target = Option; + fn deref(&self) -> &Self::Target { + self.guard.as_ref().unwrap() + } +} +impl DerefMut for WriteGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.guard.as_mut().unwrap() + } +} +impl Drop for WriteGuard { + fn drop(&mut self) { + self.pin.as_ref().unwrap().publish(self.as_ref()); + drop(self.guard.take()); + drop(self.pin.take()); + } +} + +/// The token changes with each installed executable, even when its cell and +/// database stay the same. A late panic from an old executable is not authority +/// to close its replacement. +#[derive(Clone)] +pub(super) struct Registration { + hosts: Weak>>, + replica: u64, + cell: Weak>>, + token: Arc<()>, +} + +impl Registration { + pub fn activate(&self) { + let (Some(hosts), Some(cell)) = (self.hosts.upgrade(), self.cell.upgrade()) else { + return; + }; + let mut entries = hosts.lock(); + if let Some(entry) = entries.get_mut(&self.replica) + && Arc::ptr_eq(&entry.cell, &cell) + { + entry.registration = Some(self.token.clone()); + } + } + + pub fn close_if_current(&self) -> Option { + let (Some(hosts), Some(cell)) = (self.hosts.upgrade(), self.cell.upgrade()) else { + return None; + }; + let mut entries = hosts.lock(); + let entry = entries.get_mut(&self.replica)?; + if !Arc::ptr_eq(&entry.cell, &cell) + || !entry + .registration + .as_ref() + .is_some_and(|token| Arc::ptr_eq(token, &self.token)) + { + return None; + } + Some(request_close(&hosts, self.replica, entry)) + } +} + +pub(super) struct CloseOwner { + pin: Pin, + completion: Arc, +} + +pub(super) struct CloseRequest { + pub completion: Arc, + pub owner: Option, +} + +pub(super) fn close(hosts: &Hosts, replica: u64) -> Option { + let mut entries = hosts.lock(); + let entry = entries.get_mut(&replica)?; + Some(request_close(hosts, replica, entry)) +} + +fn request_close(hosts: &Hosts, replica: u64, entry: &mut Entry) -> CloseRequest { + if let Some(completion) = &entry.closing { + return CloseRequest { + completion: completion.clone(), + owner: None, + }; + } + let completion = Arc::new(Closing { + result: watch::Sender::new(None), + }); + entry.closing = Some(completion.clone()); + entry.pins += 1; + CloseRequest { + completion: completion.clone(), + owner: Some(CloseOwner { + pin: Pin { + hosts: Arc::downgrade(hosts), + replica, + cell: entry.cell.clone(), + }, + completion, + }), + } +} + +impl CloseOwner { + pub async fn run(self) { + let mut guard = self.pin.cell.clone().write_owned().await; + // A prior cold owner can report an unconfirmed writer before this + // queued close obtains its guard. An empty cell is not proof of closure. + if self.completion.result.borrow().is_some() { + return; + } + let result = match guard.take() { + Some(host) => super::lifecycle::close_host(host).await, + None => Ok(()), + }; + let writer_closed = !matches!(result, Err(super::lifecycle::CloseFailure::WriterUnconfirmed)); + // Publish under the cell lock, then release it before final registry + // unpin/removal. Closing stays set during both steps, so reopen waits. + self.pin.publish(None); + drop(guard); + if let Some(hosts) = self.pin.hosts.upgrade() { + let mut entries = hosts.lock(); + if let Some(entry) = entries.get_mut(&self.pin.replica) { + debug_assert!(Arc::ptr_eq(&entry.cell, &self.pin.cell)); + if writer_closed { + entry.closing = None; + } + } + } + self.completion + .result + .send_replace(Some(result.map_err(|error| Arc::from(error.to_string())))); + // self.pin drops last. An unrelated idle Arc cannot delay completion. + } +} diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 42262c991c0..8b083a050ba 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,4 +1,6 @@ use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::check_hosted_admission; use crate::database_logger::{BacktraceFrame, BacktraceProvider, LogLevel, ModuleBacktrace, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; use crate::error::{DBError, DatastoreError, IndexError, NodesError}; @@ -50,11 +52,14 @@ pub struct InstanceEnv { pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. pub func_name: Option, + /// Set by trusted host dispatch for this invocation, independently of JWTs + /// and connection IDs. Cleared before each new function call. + call_auth_flags: u32, + hosted_auth: Option>, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. procedure_last_tx_offset: Option, - call_auth_flags: u32, } /// `InstanceEnv` needs to be `Send` because it is created on the host thread @@ -237,9 +242,10 @@ impl InstanceEnv { // run a function func_type: FuncCallType::Reducer, func_name: None, + call_auth_flags: 0, + hosted_auth: None, in_anon_tx: false, procedure_last_tx_offset: None, - call_auth_flags: 0, } } @@ -255,12 +261,17 @@ impl InstanceEnv { self.func_type = func_type; self.func_name = Some(name); self.call_auth_flags = 0; + self.hosted_auth = None; } pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { self.call_auth_flags = flags; } + pub(crate) fn set_hosted_auth(&mut self, auth: Option>) { + self.hosted_auth = auth; + } + pub(crate) fn get_call_auth_flags(&self) -> u32 { self.call_auth_flags } @@ -308,13 +319,22 @@ impl InstanceEnv { return Err(NodesError::NotInTransaction); } self.relational_db().with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, self.relational_db(), self.hosted_auth.as_deref()) + .map_err(|err| NodesError::HostedInvocationRejected(err.to_string()))?; environment::get(tx, key).map_err(|err| NodesError::from(DBError::Other(err.into()))) }) } pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { - let tx = &mut *self.get_tx()?; - Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?) + if let Ok(tx) = self.get_tx() { + return Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?); + } + // Procedures may inspect authentication before opening their first + // transaction. Use a short read transaction without manufacturing an + // internal caller or dropping the real connection's JWT. + Ok(self.relational_db().with_read_only(Workload::Internal, |tx| { + tx.get_jwt_payload(connection_id).map_err(DBError::from) + })?) } #[tracing::instrument(level = "trace", skip_all)] @@ -386,7 +406,8 @@ impl InstanceEnv { count } - /// Environment values are reachable only through their dedicated host interface. + /// Engine-owned deployment/authentication records and environment values + /// are reachable only through their dedicated host interfaces. fn require_module_table(table_id: TableId) -> Result<(), NodesError> { if is_module_restricted_table(table_id) { Err(NodesError::TableNotFound) @@ -792,6 +813,10 @@ impl InstanceEnv { let tx = self .relational_db() .begin_mut_tx(IsolationLevel::Serializable, Workload::Procedure); + if let Err(err) = check_hosted_admission(&tx, self.relational_db(), self.hosted_auth.as_deref()) { + let _ = tx.rollback(); + return Err(NodesError::HostedInvocationRejected(err.to_string())); + } self.tx.set_raw(tx); self.in_anon_tx = true; @@ -1529,12 +1554,107 @@ mod test { } #[test] - fn module_cannot_access_environment_by_guessed_table_and_index_ids() -> Result<()> { - use spacetimedb_datastore::system_tables::ST_ENV_ID; + fn procedure_environment_snapshot_rechecks_durable_fence_and_expiry() -> Result<()> { + use crate::auth::{ + hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, + JwtKeys, + }; + use crate::db::{deployment::install_container_fence, environment}; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + use std::time::SystemTime; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + env.func_type = FuncCallType::Procedure; + let keys = JwtKeys::generate()?; + let validator = HostedTokenValidator::new([("platform.test".into(), keys.public)])?; + let mint = |issued: SystemTime| -> Result<_> { + let binding = HostedTokenBinding { + source_database: db.database_identity(), + target_database: db.database_identity(), + generation: 1, + grant_revision: 1, + lease_expires_at: issued + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "platform.test", + &binding, + issued, + issued + Duration::from_secs(20), + "env-test", + )?; + Ok(Arc::new(validator.validate_token( + &token, + db.database_identity(), + issued, + |_, _, _| Some(binding), + )?)) + }; + let fence = |generation, allowed| StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: generation, + target_set_hash: Hash::ZERO, + allowed, + }; + db.with_auto_commit(Workload::ForTests, |tx| -> Result<()> { + install_container_fence(&db, tx, &fence(1, true))?; + environment::set(&db, tx, "VALUE", "available")?; + Ok(()) + })?; + db.with_read_only(Workload::ForTests, |_| db.hosted_admission().begin()?.complete())?; + env.set_hosted_auth(Some(mint(SystemTime::now())?)); + assert_eq!(env.env_get("VALUE")?.as_deref(), Some("available")); + env.set_hosted_auth(Some(mint(SystemTime::now() - Duration::from_secs(60))?)); + assert!(matches!( + env.env_get("VALUE"), + Err(NodesError::HostedInvocationRejected(_)) + )); + env.set_hosted_auth(Some(mint(SystemTime::now())?)); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &fence(2, false)) + })?; + // Reconciled database admission does not override a denied source fence. + db.with_read_only(Workload::ForTests, |_| db.hosted_admission().begin()?.complete())?; + assert!(matches!( + env.env_get("VALUE"), + Err(NodesError::HostedInvocationRejected(_)) + )); + env.set_hosted_auth(None); + assert_eq!(env.env_get("VALUE")?.as_deref(), Some("available")); + Ok(()) + } + + /// Generate a `ProductValue` for use in [create_table_with_index] + fn product_row(i: usize) -> ProductValue { + let str = i.to_string(); + let str = str.repeat(i); + let id = i as u64; + product!(id, str) + } + + #[test] + fn module_cannot_access_hosted_system_records_by_guessed_ids() -> Result<()> { + use spacetimedb_datastore::system_tables::{ + ST_CONNECTION_AUTH_ID, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, + ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, ST_PUBLISH_FENCE_ID, + }; let db = relational_db()?; let (env, _runtime) = instance_env(db.clone())?; let mut slot = env.tx.clone(); - let protected = [(ST_ENV_ID, "st_env", to_vec("TOKEN")?)]; + let protected = [ + (ST_ENV_ID, "st_env", to_vec("TOKEN")?), + (ST_DEPLOYMENT_ID, "st_deployment", to_vec(&0u8)?), + (ST_PUBLISH_FENCE_ID, "st_publish_fence", to_vec(&0u8)?), + (ST_DEPLOYMENT_OPERATION_ID, "st_deployment_operation", to_vec(&0u128)?), + (ST_CONNECTION_AUTH_ID, "st_connection_auth", to_vec(&0u128)?), + (ST_CONTAINER_ENVIRONMENT_ID, "st_container_environment", to_vec(&0u64)?), + ( + ST_CONTAINER_FENCE_ID, + "st_container_fence", + to_vec(&spacetimedb_sats::u256::ZERO)?, + ), + ]; let tx = begin_mut_tx(&db); let (tx, result) = slot.set(tx, || -> Result<()> { for (table, name, point) in &protected { @@ -1599,14 +1719,6 @@ mod test { result } - /// Generate a `ProductValue` for use in [create_table_with_index] - fn product_row(i: usize) -> ProductValue { - let str = i.to_string(); - let str = str.repeat(i); - let id = i as u64; - product!(id, str) - } - /// Generate a BSATN encoded row for use in [create_table_with_index] fn bsatn_row(i: usize) -> Result> { Ok(to_vec(&product_row(i))?) diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 06e19c367b9..4b8618e672a 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -10,7 +10,10 @@ use spacetimedb_lib::ProductValue; use spacetimedb_schema::def::deserialize::{ArgsSeed, FunctionDef}; use spacetimedb_schema::def::ModuleDef; +pub mod container_environment; +pub mod container_fence; mod disk_storage; +pub mod empty_module; mod host_controller; mod module_common; #[allow(clippy::too_many_arguments)] @@ -191,8 +194,8 @@ pub enum AbiCall { Identity, JwtLength, GetJwt, - EnvGet, GetCallAuthFlags, + EnvGet, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/module_common.rs b/crates/core/src/host/module_common.rs index 49f1126b54b..5bd4e61d927 100644 --- a/crates/core/src/host/module_common.rs +++ b/crates/core/src/host/module_common.rs @@ -19,7 +19,7 @@ use std::sync::Arc; pub fn build_common_module_from_raw( mcc: ModuleCreationContext, raw_def: RawModuleDef, -) -> Result { +) -> Result> { // Perform a bunch of validation on the raw definition. let def: ModuleDef = raw_def.try_into()?; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 4089b88ec5d..90b7bab8215 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2,9 +2,12 @@ use super::{ ArgsTuple, FunctionArgs, InvalidProcedureArguments, InvalidReducerArguments, ReducerCallResult, ReducerCallResultWithTxOffset, ReducerId, ReducerOutcome, Scheduler, }; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::{check_hosted_admission, InvocationCaller, SqlCallAuth}; use crate::client::messages::{OneOffQueryResponseMessage, ProcedureResultMessage, SerializableMessage}; use crate::client::{ClientActorId, ClientConnectionSender, WsVersion}; use crate::database_logger::{DatabaseLogger, LogLevel, Record}; +use crate::db::deployment::{self, CommitAdmission, DeploymentCommit, PublishResult as DeploymentPublishResult}; use crate::db::relational_db::{RelationalDB, Tx}; use crate::db::sql::ast::SchemaViewer; use crate::error::DBError; @@ -73,11 +76,16 @@ use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; use std::fmt; use std::num::NonZeroUsize; -use std::sync::atomic::AtomicBool; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; use tokio::sync::{oneshot, OwnedSemaphorePermit, Semaphore}; +#[cfg(test)] +mod drain_tests; +mod operations; +use operations::ModuleOperations; +pub(in crate::host) use operations::OperationLease; + #[derive(Debug, Default, Clone, From)] pub struct DatabaseUpdate { pub tables: SmallVec<[DatabaseTableUpdate; 1]>, @@ -472,6 +480,7 @@ impl WasmtimeModuleHost { label: &str, on_panic: Arc, timer_guard: CallTimerGuard, + operation: OperationLease, arg: A, wasm: impl FnOnce(A, &mut ModuleInstance) + Send + 'static, ) where @@ -479,6 +488,7 @@ impl WasmtimeModuleHost { { let label = label.to_owned(); self.executor.enqueue_sync_job(move |state| { + let _operation = operation; scopeguard::defer_on_unwind!({ log::error!("wasm main operation {label} panicked"); on_panic(); @@ -496,15 +506,20 @@ impl WasmtimeModuleHost { label: &str, on_panic: Arc, timer_guard: CallTimerGuard, + operation: OperationLease, arg: A, wasm: impl AsyncFnOnce(A, &mut ModuleInstance) + Send + 'static, ) where A: Send + 'static, { let instance_manager = self.procedure_instances.clone(); - let ModuleInstanceLease { instance, slot } = instance_manager.get_instance().await; + let ModuleInstanceLease { instance, slot } = instance_manager + .get_instance(Some(operation.clone())) + .await + .unwrap_or_else(|never| match never {}); let label = label.to_owned(); self.executor.enqueue_async_job(async move || { + let _operation = operation; scopeguard::defer_on_unwind!({ log::error!("wasm procedure {label} panicked"); on_panic(); @@ -527,7 +542,8 @@ struct V8ModuleHost { /// A module; used as a bound on `InstanceManager`. trait GenericModule { type Instance: GenericModuleInstance; - async fn create_instance(&self) -> Self::Instance; + type CreationError; + async fn create_instance(&self, operation: Option) -> Result; fn host_type(&self) -> HostType; } @@ -557,8 +573,10 @@ impl GenericModuleInstance for Box { impl GenericModule for Arc { type Instance = Box; - async fn create_instance(&self) -> Self::Instance { - Box::new((**self).create_instance()) + type CreationError = std::convert::Infallible; + async fn create_instance(&self, operation: Option) -> Result { + let _operation = operation; + Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { HostType::Wasm @@ -567,8 +585,10 @@ impl GenericModule for Arc { impl GenericModule for Arc { type Instance = Box; - async fn create_instance(&self) -> Self::Instance { - Box::new((**self).create_instance()) + type CreationError = std::convert::Infallible; + async fn create_instance(&self, operation: Option) -> Result { + let _operation = operation; + Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { HostType::Wasm @@ -577,8 +597,9 @@ impl GenericModule for Arc { impl GenericModule for super::v8::JsModule { type Instance = super::v8::JsProcedureInstance; - async fn create_instance(&self) -> Self::Instance { - self.create_instance().await + type CreationError = anyhow::Error; + async fn create_instance(&self, operation: Option) -> Result { + self.create_instance_for_operation(operation).await } fn host_type(&self) -> HostType { HostType::Js @@ -607,18 +628,33 @@ fn extract_trapped(res: Result<(T, bool), E>) -> (Result, bool) { pub(crate) fn init_database( replica_ctx: &ReplicaContext, module_def: &ModuleDef, + module_hash: Hash, program: Program, + deployment: Option, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> (anyhow::Result, bool) { - extract_trapped(init_database_inner(replica_ctx, module_def, program, call_reducer)) + extract_trapped(init_database_inner( + replica_ctx, + module_def, + module_hash, + program, + deployment, + call_reducer, + )) } fn init_database_inner( replica_ctx: &ReplicaContext, module_def: &ModuleDef, + module_hash: Hash, program: Program, + deployment: Option, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResultWithTxOffset, bool), ) -> anyhow::Result<(InitDatabaseResult, bool)> { + anyhow::ensure!( + module_hash == program.hash && spacetimedb_lib::hash_bytes(&program.bytes) == program.hash, + "program does not match the instantiated module" + ); log::debug!("init database"); let timestamp = Timestamp::now(); let stdb = replica_ctx.relational_db(); @@ -627,6 +663,34 @@ fn init_database_inner( let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); let auth_ctx = AuthCtx::for_current(owner_identity); + let (tx, admission) = stdb.with_auto_rollback(tx, |tx| { + if let Some(request) = &deployment { + deployment::validate_deployment_program(request, &program, module_def)?; + // New databases install the bootstrap fence, schema, program and + // receipt in this one transaction. An interrupted initialization + // cannot leave a successful partial deployment behind. + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use spacetimedb_datastore::system_tables::ST_MODULE_ID; + if tx.iter(ST_MODULE_ID)?.next().is_none() { + deployment::install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + } + deployment::check_deployment_commit(tx, request, timestamp, &Default::default()) + } else { + deployment::require_unmanaged_publication(tx)?; + Ok(CommitAdmission::Ready) + } + })?; + if matches!(admission, CommitAdmission::AlreadyCommitted(_)) { + let (offset, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + return Ok(( + InitDatabaseResult { + reducer: None, + tx_offset: from_tx_offset(offset), + }, + false, + )); + } let (tx, ()) = stdb .with_auto_rollback(tx, |tx| { // Create all in-memory tables defined by the module (including submodules), @@ -676,6 +740,10 @@ fn init_database_inner( stdb.set_initialized(tx, program)?; + if let Some(request) = &deployment { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default())?; + } + anyhow::Ok(()) }) .inspect_err(|e| log::error!("{e:?}"))?; @@ -741,6 +809,13 @@ pub fn call_identity_connected( stdb.report_mut_tx_metrics(reducer_name, metrics, None); }); + let caller = InvocationCaller::from(&caller_auth); + let flags = caller + .flags_for(module.database_identity, &module.module_def) + .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; + check_hosted_admission(&*mut_tx, stdb, caller.hosted.as_deref()) + .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; + mut_tx .insert_st_client( caller_auth.claims.identity, @@ -750,13 +825,23 @@ pub fn call_identity_connected( .map_err(DBError::from) .map_err(Box::new)?; + if caller.hosted.is_some() { + crate::db::deployment::record_connection_auth( + &mut mut_tx, + caller_connection_id, + caller_auth.claims.identity, + flags, + ) + .map_err(|err| ClientConnectedError::DBError(Box::new(DBError::Other(err.into()))))?; + } + if let Some((reducer_id, reducer_def)) = reducer_lookup { // The module defined a lifecycle reducer to handle new connections. // Call this reducer. // If the call fails (as in, something unexpectedly goes wrong with guest execution), // abort the connection: we can't really recover. let tx = Some(ScopeGuard::into_inner(mut_tx)); - let params = ModuleHost::call_reducer_params( + let mut params = ModuleHost::call_reducer_params( &module.module_def, caller_auth.claims.identity, Some(caller_connection_id), @@ -768,6 +853,8 @@ pub fn call_identity_connected( FunctionArgs::Nullary, ) .map_err(ReducerCallError::from)?; + params.call_auth_flags = flags; + params.hosted_auth = caller.hosted; let (reducer_outcome, trapped) = call_reducer(tx, params); *trapped_slot = trapped; @@ -812,7 +899,9 @@ pub struct CallReducerParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + /// Verified invocation authority. Bit 0 is internal; never client-decoded. pub(crate) call_auth_flags: u32, + pub(crate) hosted_auth: Option>, pub client: Option>, pub request_id: Option, pub timer: Option, @@ -834,6 +923,7 @@ impl CallReducerParams { caller_identity, caller_connection_id: ConnectionId::ZERO, call_auth_flags: 1, + hosted_auth: None, client: None, request_id: None, timer: None, @@ -1039,7 +1129,7 @@ impl ViewCommandErrorTarget { pub(in crate::host) struct SqlCommand { pub(in crate::host) db: Arc, pub(in crate::host) sql_text: String, - pub(in crate::host) auth: AuthCtx, + pub(in crate::host) auth: SqlCallAuth, pub(in crate::host) subs: Option, } @@ -1214,6 +1304,7 @@ pub struct CallProcedureParams { pub caller_identity: Identity, pub caller_connection_id: ConnectionId, pub(crate) call_auth_flags: u32, + pub(crate) hosted_auth: Option>, pub timer: Option, pub procedure_id: ProcedureId, pub args: ArgsTuple, @@ -1233,6 +1324,7 @@ impl CallProcedureParams { caller_identity, caller_connection_id: ConnectionId::ZERO, call_auth_flags: 1, + hosted_auth: None, timer: None, procedure_id, args, @@ -1264,7 +1356,7 @@ struct ModuleInstanceManager { struct ModuleInstanceLease { instance: I, - slot: Option, + slot: Option>, } /// Holds the single shared instance used by the JS main execution path. @@ -1437,22 +1529,18 @@ impl ModuleInstanceManager { } } - async fn with_instance(&self, f: impl AsyncFnOnce(M::Instance) -> (R, M::Instance)) -> R { - let ModuleInstanceLease { instance, slot } = self.get_instance().await; - let (res, instance) = f(instance).await; - self.return_instance(ModuleInstanceLease { instance, slot }); - res - } - - async fn get_instance(&self) -> ModuleInstanceLease { + async fn get_instance( + &self, + operation: Option, + ) -> Result, M::CreationError> { let slot = if let Some(instance_slots) = &self.instance_slots { - Some( + Some(Arc::new( instance_slots .clone() .acquire_owned() .await .expect("module instance slot semaphore should not close"), - ) + )) } else { None }; @@ -1465,13 +1553,16 @@ impl ModuleInstanceManager { instance } else { let start_time = std::time::Instant::now(); - let res = self.module.create_instance().await; + let res = self + .module + .create_instance(operation.map(|operation| operation.with_pool_slot(slot.clone()))) + .await?; let elapsed_time = start_time.elapsed(); self.metrics.observe_instance_created(elapsed_time); res }; - ModuleInstanceLease { instance, slot } + Ok(ModuleInstanceLease { instance, slot }) } fn return_instance(&self, lease: ModuleInstanceLease) { @@ -1510,10 +1601,10 @@ pub struct ModuleHost { /// Called whenever a reducer call on this host panics. on_panic: Arc, - /// Marks whether this module has been closed by [`Self::exit`]. - /// - /// When this is true, most operations will fail with [`NoSuchModule`]. - closed: Arc, + /// Shared admission and physical-operation drainage for this module. + /// [`Self::exit`] closes admission, rejects new work with [`NoSuchModule`], + /// and waits for accepted executor jobs, including cancelled callers. + operations: Arc, } impl fmt::Debug for ModuleHost { @@ -1529,12 +1620,19 @@ pub struct WeakModuleHost { info: Arc, inner: Weak, on_panic: Weak, - closed: Weak, + operations: Weak, } #[derive(Debug)] pub enum UpdateDatabaseResult { NoUpdateNeeded, + /// A prior successful commit is returned without replacing the current + /// module, which may already be newer than this retried publication. + DeploymentAlreadyCommitted { + result: DeploymentPublishResult, + tx_offset: TransactionOffset, + durable_offset: Option, + }, UpdatePerformed { /// The transaction offset of the successful database update. tx_offset: TransactionOffset, @@ -1559,6 +1657,7 @@ impl UpdateDatabaseResult { self, UpdateDatabaseResult::UpdatePerformed { .. } | UpdateDatabaseResult::NoUpdateNeeded + | UpdateDatabaseResult::DeploymentAlreadyCommitted { .. } | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } ) } @@ -1568,6 +1667,45 @@ impl UpdateDatabaseResult { #[error("no such module")] pub struct NoSuchModule; +#[derive(thiserror::Error, Debug)] +enum PooledCallError { + #[error(transparent)] + NoSuchModule(#[from] NoSuchModule), + #[error("module instance startup failed: {0}")] + Startup(anyhow::Error), +} + +impl From for ProcedureCallError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => { + Self::InternalError(format!("module instance startup failed: {error:#}")) + } + } + } +} + +impl From for HttpHandlerCallError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => { + Self::InternalError(format!("module instance startup failed: {error:#}")) + } + } + } +} + +impl From for CallScheduledFunctionError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => Self::InstanceStartup(error), + } + } +} + #[derive(thiserror::Error, Debug)] pub enum ReducerCallError { #[error(transparent)] @@ -1826,7 +1964,7 @@ impl ModuleHost { info, inner, on_panic, - closed: Arc::new(AtomicBool::new(false)), + operations: Arc::new(ModuleOperations::default()), } } @@ -1845,20 +1983,6 @@ impl ModuleHost { matches!(&*self.inner, ModuleHostInner::Js(_)) } - fn is_marked_closed(&self) -> bool { - // `self.closed` isn't used for any synchronization, it's just a shared flag, - // so `Ordering::Relaxed` is sufficient. - self.closed.load(std::sync::atomic::Ordering::Relaxed) - } - - fn guard_closed(&self) -> Result<(), NoSuchModule> { - if self.is_marked_closed() { - Err(NoSuchModule) - } else { - Ok(()) - } - } - fn start_call_timer(&self, label: &str) -> CallTimerGuard { // Record the time until our function starts running. let queue_timer = WORKER_METRICS @@ -1896,7 +2020,7 @@ impl ModuleHost { R: Send + 'static, A: Send + 'static, { - self.guard_closed()?; + let operation = self.operations.begin()?; let timer_guard = self.start_call_timer(label); scopeguard::defer_on_unwind!({ @@ -1909,6 +2033,7 @@ impl ModuleHost { let executor = host.executor.clone(); executor .run_sync_job(move |state| { + let _operation = operation; state.with_instance(move |inst| { drop(timer_guard); wasm(arg, inst) @@ -1919,7 +2044,7 @@ impl ModuleHost { ModuleHostInner::Js(host) => { drop(timer_guard); host.main_instance - .with_instance(|inst| async move { js(arg, &inst).await }) + .with_instance(|inst| async move { js(arg, &inst.with_operation(operation)).await }) .await } }) @@ -1935,12 +2060,12 @@ impl ModuleHost { arg: A, wasm: impl AsyncFnOnce(A, &mut ModuleInstance) -> R + Send + 'static, js: impl AsyncFnOnce(A, &JsProcedureInstance) -> R, - ) -> Result + ) -> Result where R: Send + 'static, A: Send + 'static, { - self.guard_closed()?; + let operation = self.operations.begin()?; let timer_guard = self.start_call_timer(label); scopeguard::defer_on_unwind!({ @@ -1951,27 +2076,35 @@ impl ModuleHost { Ok(match &*self.inner { ModuleHostInner::Wasm(host) => { let executor = host.executor.clone(); - let instance_manager = host.procedure_instances.clone(); - instance_manager - .with_instance(async move |mut inst| { - executor - .run_async_job(async move || { - drop(timer_guard); - let res = wasm(arg, &mut inst).await; - (res, inst) - }) - .await + let manager = host.procedure_instances.clone(); + let ModuleInstanceLease { mut instance, slot } = manager + .get_instance(Some(operation.clone())) + .await + .unwrap_or_else(|never| match never {}); + let operation = operation.with_pool_slot(slot); + executor + .run_async_job(async move || { + let _operation = operation; + drop(timer_guard); + let result = wasm(arg, &mut instance).await; + manager.return_instance(ModuleInstanceLease { instance, slot: None }); + result }) .await } ModuleHostInner::Js(host) => { - host.procedure_instances - .with_instance(async |inst| { - drop(timer_guard); - let res = js(arg, &inst).await; - (res, inst) - }) + let mut lease = host + .procedure_instances + .get_instance(Some(operation.clone())) .await + .map_err(PooledCallError::Startup)?; + lease + .instance + .set_operation(Some(operation.with_pool_slot(lease.slot.take()))); + drop(timer_guard); + let result = js(arg, &lease.instance).await; + self.return_js_procedure_instance(lease); + result } }) } @@ -1982,7 +2115,7 @@ impl ModuleHost { label: &str, arg: A, js: impl FnOnce(A, JsMainInstance, JsFatalHook) -> JsFut, - wasm: impl FnOnce(A, &WasmtimeModuleHost, JsFatalHook, CallTimerGuard) -> Result<(), NoSuchModule>, + wasm: impl FnOnce(A, &WasmtimeModuleHost, JsFatalHook, CallTimerGuard, OperationLease) -> Result<(), NoSuchModule>, ) -> Result<(), NoSuchModule> where A: Send + 'static, @@ -1994,21 +2127,20 @@ impl ModuleHost { (self.on_panic)(); }); + let operation = self.operations.begin()?; match &*self.inner { ModuleHostInner::Js(js_host) => { - self.guard_closed()?; let on_panic = self.on_panic.clone(); js_host .main_instance - .with_instance(|inst| js(arg, inst, on_panic)) + .with_instance(|inst| js(arg, inst.with_operation(operation), on_panic)) .await; Ok(()) } ModuleHostInner::Wasm(wasm_host) => { - self.guard_closed()?; let timer_guard = self.start_call_timer(label); let on_panic = self.on_panic.clone(); - wasm(arg, wasm_host, on_panic, timer_guard) + wasm(arg, wasm_host, on_panic, timer_guard, operation) } } } @@ -2025,12 +2157,13 @@ impl ModuleHost { label, (cmd, metric), |(cmd, metric), inst, on_panic| async move { inst.enqueue_call_view(cmd, metric, on_panic).await }, - move |(cmd, metric), wasm_host, on_panic, timer_guard| { + move |(cmd, metric), wasm_host, on_panic, timer_guard, operation| { let info = wasm_host.module.info(); wasm_host.enqueue_with_main_instance( label, on_panic, timer_guard, + operation, (cmd, metric), move |(cmd, metric), inst| { let result = inst.call_view(cmd); @@ -2130,7 +2263,7 @@ impl ModuleHost { } /// Invokes the `client_disconnected` reducer, if present, - /// then deletes the client’s rows from `st_client` and `st_connection_credentials`. + /// then deletes the client's rows from `st_client`, `st_connection_credentials`, and `st_connection_auth`. /// If the reducer fails, the rows are still deleted. /// Calling this on an already-disconnected client is a no-op. pub fn call_identity_disconnected_inner( @@ -2196,22 +2329,35 @@ impl ModuleHost { // The module defined a lifecycle reducer to handle disconnects. Call it. // If it succeeds, `WasmModuleInstance::call_reducer_with_tx` has already ensured // that `st_client` is updated appropriately. + let flags = crate::db::deployment::connection_auth_flags(&mut_tx, caller_connection_id, caller_identity) + .map_err(|err| { + InvalidReducerArguments(InvalidFunctionArguments { + err: err.into(), + function_name: reducer_name.clone().into(), + }) + }); let tx = Some(mut_tx); - let result = Self::call_reducer_params( - &info.module_def, - caller_identity, - Some(caller_connection_id), - None, - None, - None, - reducer_id, - reducer_def, - FunctionArgs::Nullary, - ) - .map(|params| { - let (res, trapped) = call_reducer(tx, params); - *trapped_slot = trapped; - res + let result = flags.and_then(|flags| { + Self::call_reducer_params( + &info.module_def, + caller_identity, + Some(caller_connection_id), + None, + None, + None, + reducer_id, + reducer_def, + FunctionArgs::Nullary, + ) + .map(|mut params| { + // This host event retains the connection's captured authority, + // including after credential expiry, revocation, or host recovery. + // It must not carry a live hosted proof that could block cleanup. + params.call_auth_flags = flags; + let (res, trapped) = call_reducer(tx, params); + *trapped_slot = trapped; + res + }) }); // If it failed, we still need to update `st_client`: the client's not coming back. @@ -2304,6 +2450,7 @@ impl ModuleHost { caller_identity, caller_connection_id, call_auth_flags: 0, + hosted_auth: None, client, request_id, timer, @@ -2314,7 +2461,7 @@ impl ModuleHost { fn reducer_call_params<'a>( &'a self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2322,6 +2469,10 @@ impl ModuleHost { reducer_name: &str, args: FunctionArgs, ) -> Result<(&'a ReducerDef, CallReducerParams), ReducerCallError> { + let flags = caller + .flags_for(self.info.database_identity, &self.info.module_def) + .map_err(|_| ReducerCallError::NoSuchReducer)?; + let caller_identity = caller.identity; let (reducer_id, reducer_def, owning_def) = self .info .module_def @@ -2333,25 +2484,25 @@ impl ModuleHost { if !reducer_def .visibility - .allows_invocation(false, self.is_database_owner(caller_identity)) + .allows_invocation(flags & 1 != 0, self.is_database_owner(caller_identity)) { return Err(ReducerCallError::NoSuchReducer); } - Ok(( + let mut params = Self::call_reducer_params( + owning_def, + caller_identity, + caller_connection_id, + client, + request_id, + timer, + reducer_id, reducer_def, - Self::call_reducer_params( - owning_def, - caller_identity, - caller_connection_id, - client, - request_id, - timer, - reducer_id, - reducer_def, - args, - )?, - )) + args, + )?; + params.call_auth_flags = flags; + params.hosted_auth = caller.hosted; + Ok((reducer_def, params)) } async fn call_reducer_with_params( @@ -2379,7 +2530,7 @@ impl ModuleHost { async fn with_reducer_call( &self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2390,7 +2541,7 @@ impl ModuleHost { ) -> Result { let res = async { let (reducer_def, params) = self.reducer_call_params( - caller_identity, + caller, caller_connection_id, client, request_id, @@ -2415,7 +2566,7 @@ impl ModuleHost { pub async fn call_reducer( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2424,7 +2575,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result { self.with_reducer_call( - caller_identity, + caller.into(), caller_connection_id, client, request_id, @@ -2438,7 +2589,7 @@ impl ModuleHost { pub async fn enqueue_reducer( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2447,7 +2598,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result<(), ReducerCallError> { self.with_reducer_call( - caller_identity, + caller.into(), caller_connection_id, client, request_id, @@ -2461,11 +2612,12 @@ impl ModuleHost { reducer_name, call.params, |params, inst, on_panic| async move { inst.enqueue_reducer(params, on_panic).await }, - move |params, wasm_host, on_panic, timer_guard| { + move |params, wasm_host, on_panic, timer_guard, operation| { wasm_host.enqueue_with_main_instance( &reducer_label, on_panic, timer_guard, + operation, params, move |params, inst| { let _ = inst.call_reducer(params); @@ -2590,10 +2742,15 @@ impl ModuleHost { &self, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result { + InvocationCaller { + identity: auth.caller(), + hosted: auth.hosted.clone(), + } + .flags_for(self.info.database_identity, &self.info.module_def)?; let cmd = SqlCommand { db, sql_text, @@ -2609,18 +2766,15 @@ impl ModuleHost { pub async fn call_procedure( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> CallProcedureReturn { let res = async { - let call = - self.prepare_procedure_call(caller_identity, caller_connection_id, timer, procedure_name, args)?; - self.call_procedure_with_params(&call.name, call.params) - .await - .map_err(Into::into) + let call = self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args)?; + self.call_procedure_with_params(&call.name, call.params).await } .await; @@ -2633,7 +2787,7 @@ impl ModuleHost { pub(crate) async fn enqueue_procedure( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, timer: Option, procedure_name: &str, @@ -2641,7 +2795,7 @@ impl ModuleHost { target: ProcedureResultTarget, ) -> Result<(), BroadcastError> { let PreparedProcedureCall { name, params } = - match self.prepare_procedure_call(caller_identity, caller_connection_id, timer, procedure_name, args) { + match self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args) { Ok(value) => value, Err(err) => { return self.send_procedure_error(procedure_name, timer, target, err); @@ -2655,14 +2809,39 @@ impl ModuleHost { (self.on_panic)(); }); - if let Err(err) = self.guard_closed() { - return self.send_procedure_error(&procedure_name, timer, target, err.into()); - } + let operation = match self.operations.begin() { + Ok(operation) => operation, + Err(err) => return self.send_procedure_error(&procedure_name, timer, target, err.into()), + }; match &*self.inner { ModuleHostInner::Js(host) => { - let lease = host.procedure_instances.get_instance().await; - let call = lease.instance.enqueue_procedure(params).await; + let mut lease = match host.procedure_instances.get_instance(Some(operation.clone())).await { + Ok(lease) => lease, + Err(error) => { + return self.send_procedure_error( + &procedure_name, + timer, + target, + PooledCallError::Startup(error).into(), + ); + } + }; + lease + .instance + .set_operation(Some(operation.with_pool_slot(lease.slot.take()))); + let call = match lease.instance.enqueue_procedure(params).await { + Ok(call) => call, + Err(error) => { + self.return_js_procedure_instance(lease); + return self.send_procedure_error( + &procedure_name, + timer, + target, + ProcedureCallError::InternalError(error.to_string()), + ); + } + }; let module = self.clone(); tokio::spawn(async move { match call.receive().await { @@ -2672,6 +2851,16 @@ impl ModuleHost { log::warn!("failed to send procedure result: {err:#}"); } } + JsProcedureCallCompletion::StartupFailed(error) => { + if let Err(error) = module.send_procedure_error( + &procedure_name, + timer, + target, + ProcedureCallError::InternalError(error.to_string()), + ) { + log::warn!("failed to send procedure startup error: {error:#}"); + } + } JsProcedureCallCompletion::Panicked | JsProcedureCallCompletion::WorkerExited => { log::error!("detached JS procedure worker failed before returning a result"); (module.on_panic)(); @@ -2692,6 +2881,7 @@ impl ModuleHost { &procedure_name, on_panic, timer_guard, + operation, params, async move |params, inst| { let ret = inst.call_procedure(params).await; @@ -2711,10 +2901,11 @@ impl ModuleHost { } } - fn return_js_procedure_instance(&self, lease: ModuleInstanceLease) { + fn return_js_procedure_instance(&self, mut lease: ModuleInstanceLease) { let ModuleHostInner::Js(host) = &*self.inner else { return; }; + lease.instance.set_operation(None); host.procedure_instances.return_instance(lease); } @@ -2816,14 +3007,14 @@ impl ModuleHost { fn prepare_procedure_call( &self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> Result { let (procedure_def, params) = - self.procedure_call_params(caller_identity, caller_connection_id, timer, procedure_name, args)?; + self.procedure_call_params(caller, caller_connection_id, timer, procedure_name, args)?; Ok(PreparedProcedureCall { name: procedure_def.name.to_string(), params, @@ -2832,12 +3023,16 @@ impl ModuleHost { fn procedure_call_params<'a>( &'a self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> Result<(&'a ProcedureDef, CallProcedureParams), ProcedureCallError> { + let flags = caller + .flags_for(self.info.database_identity, &self.info.module_def) + .map_err(|_| ProcedureCallError::NoSuchProcedure)?; + let caller_identity = caller.identity; let (procedure_id, procedure_def, owning_def) = self .info .module_def @@ -2846,7 +3041,7 @@ impl ModuleHost { if !procedure_def .visibility - .allows_invocation(false, self.is_database_owner(caller_identity)) + .allows_invocation(flags & 1 != 0, self.is_database_owner(caller_identity)) { return Err(ProcedureCallError::NoSuchProcedure); } @@ -2862,7 +3057,8 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, - call_auth_flags: 0, + call_auth_flags: flags, + hosted_auth: caller.hosted, timer, procedure_id, args, @@ -2879,7 +3075,7 @@ impl ModuleHost { &self, name: &str, params: CallProcedureParams, - ) -> Result { + ) -> Result { call_pooled_instance!( self, name, @@ -2887,6 +3083,7 @@ impl ModuleHost { |params, inst| inst.call_procedure(params).await, |params, inst| inst.call_procedure(params).await, ) + .map_err(Into::into) } pub async fn call_http_handler( @@ -2937,10 +3134,13 @@ impl ModuleHost { self, "scheduled procedure", params, - |params, inst| inst.call_scheduled_procedure(params).await, - |params, inst| inst.call_scheduled_procedure(params).await, + |params, inst| Ok(inst.call_scheduled_procedure(params).await), + |params, inst| inst + .call_scheduled_procedure(params) + .await + .map_err(|error| CallScheduledFunctionError::InstanceStartup(error.into())), ) - .map_err(Into::into) + .map_err(CallScheduledFunctionError::from)? } /// Materializes the views return by the `view_collector`, if not already materialized, @@ -3193,14 +3393,49 @@ impl ModuleHost { } pub async fn init_database(&self, program: Program) -> Result { - call_instance!( + self.init_database_with_deployment(program, None).await + } + + /// Host-only initialization following an authorized, durable control intent. + /// A new database installs its bootstrap fence in the initialization + /// transaction. Successful deployment initialization waits for durability + /// before its scheduler or container can be activated. + pub async fn init_database_with_deployment( + &self, + program: Program, + deployment: Option, + ) -> Result { + let confirm_deployment = deployment.is_some(); + let result = call_instance!( self, "", - program, - |p, inst| inst.init_database(p), - |p, inst| inst.init_database(p).await, + (program, deployment), + |(p, d), inst| inst.init_database(p, d), + |(p, d), inst| inst.init_database(p, d).await, )? - .map_err(InitDatabaseError::Other) + .map_err(InitDatabaseError::Other)?; + if confirm_deployment + && result.reducer.as_ref().is_none_or(|result| result.is_ok()) + && let Some(mut durability) = self.relational_db().durable_tx_offset() + { + // A read barrier after init includes its receipt even when the + // module has no init reducer. Do not block the async worker on + // the datastore lock while capturing that barrier. + let db = self.relational_db().clone(); + let offset = tokio::task::spawn_blocking(move || { + let tx = db.begin_tx(Workload::Internal); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + offset + }) + .await + .map_err(|error| InitDatabaseError::Other(error.into()))?; + durability + .wait_for(offset) + .await + .map_err(|error| InitDatabaseError::Other(error.into()))?; + } + Ok(result) } pub async fn update_database( @@ -3208,25 +3443,40 @@ impl ModuleHost { program: Program, old_module_info: Arc, policy: MigrationPolicy, + ) -> Result { + self.update_database_with_deployment(program, old_module_info, policy, None) + .await + } + + /// Commit the prepared deployment in the same transaction as migration. + pub async fn update_database_with_deployment( + &self, + program: Program, + old_module_info: Arc, + policy: MigrationPolicy, + deployment: Option, ) -> Result { call_instance!( self, "", - (program, old_module_info, policy), - |(a, b, c), inst| inst.update_database(a, b, c), - |(a, b, c), inst| inst.update_database(a, b, c).await, + (program, old_module_info, policy, deployment), + |(a, b, c, d), inst| inst.update_database(a, b, c, d), + |(a, b, c, d), inst| inst.update_database(a, b, c, d).await, )? } pub async fn exit(&self) { - // As in `Self::marked_closed`, `Relaxed` is sufficient because we're not synchronizing any external state. - self.closed.store(true, std::sync::atomic::Ordering::Relaxed); + // Admission and closure share one lock. Already admitted work retains + // its lease inside the physical executor after cancellation of a caller. + self.operations.close(); self.scheduler().close(); self.exited().await; } pub async fn exited(&self) { self.scheduler().closed().await; + self.operations.close(); + self.operations.drained().await; } pub fn inject_logs(&self, log_level: LogLevel, function_name: &str, message: &str) { @@ -3299,11 +3549,12 @@ impl ModuleHost { label, request, |request, inst, on_panic| async move { inst.enqueue_one_off_query(request, on_panic).await }, - move |request, wasm_host, on_panic, timer_guard| { + move |request, wasm_host, on_panic, timer_guard, operation| { let executor = wasm_host.executor.clone(); let info = wasm_host.module.info(); let label = label.to_owned(); executor.enqueue_sync_job(move |_| { + let _operation = operation; scopeguard::defer_on_unwind!({ log::error!("websocket one-off query operation {label} panicked"); on_panic(); @@ -3456,8 +3707,10 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { - ws_v1::OneOffTable { table_name, rows } + let result = check_hosted_admission(&*tx, &db, client.auth.hosted.as_ref()).and_then(|()| { + Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { + ws_v1::OneOffTable { table_name, rows } + }) }); let total_host_execution_duration = timer.elapsed().into(); @@ -3535,10 +3788,11 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = + let result = check_hosted_admission(&*tx, &db, client.auth.hosted.as_ref()).and_then(|()| { Self::execute_one_off_query::(&db, &tx, &auth, &query, &rlb_pool, |table, rows| { ws_v2::SingleTableRows { table, rows } - }); + }) + }); let (message, metrics) = match result { Ok((rows, metrics)) => { @@ -3576,6 +3830,7 @@ impl ModuleHost { /// for tables without primary keys. It is only used in the benchmarks. /// Note: this doesn't drop the table, it just clears it! pub fn clear_table(&self, table_name: &str) -> Result<(), anyhow::Error> { + let _operation = self.operations.begin()?; let db = self.relational_db(); db.with_auto_commit(Workload::Internal, |tx| { @@ -3597,7 +3852,7 @@ impl ModuleHost { info: self.info.clone(), inner: Arc::downgrade(&self.inner), on_panic: Arc::downgrade(&self.on_panic), - closed: Arc::downgrade(&self.closed), + operations: Arc::downgrade(&self.operations), } } @@ -3636,12 +3891,12 @@ impl WeakModuleHost { pub fn upgrade(&self) -> Option { let inner = self.inner.upgrade()?; let on_panic = self.on_panic.upgrade()?; - let closed = self.closed.upgrade()?; + let operations = self.operations.upgrade()?; Some(ModuleHost { info: self.info.clone(), inner, on_panic, - closed, + operations, }) } } diff --git a/crates/core/src/host/module_host/drain_tests.rs b/crates/core/src/host/module_host/drain_tests.rs new file mode 100644 index 00000000000..2465d7cca36 --- /dev/null +++ b/crates/core/src/host/module_host/drain_tests.rs @@ -0,0 +1,397 @@ +//! Actual local database writers, Wasm/JS worker queues and cancelled callers. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::host_controller::{HostController, HostRuntimeConfig}; +use crate::util::jobs::JobCores; +use spacetimedb_auth::identity::SpacetimeIdentityClaims; +use spacetimedb_datastore::system_tables::ModuleKind; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; +use spacetimedb_paths::{server::ServerDataDir, FromPathUnchecked}; +use tokio::time::timeout; + +fn javascript() -> Program { + let mut definition = RawModuleDefV10Builder::new(); + definition.add_lifecycle_reducer( + Lifecycle::OnDisconnect, + "disconnected", + spacetimedb_sats::ProductType::unit(), + ); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(definition.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{register_hooks}} from "spacetime:sys@1.0"; + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), + __call_reducer__: () => ({{tag:"ok"}}) }}); + "# + ) + .into_bytes(), + ) +} + +fn fixture(id: u64, program: Program) -> (tempfile::TempDir, HostController, Database) { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let initial = program.clone(); + let controller = HostController::new( + data.clone(), + crate::db::Config { + storage: crate::db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }), + Arc::new(crate::energy::NullEnergyMonitor), + Arc::new(()), + Arc::new(LocalPersistenceProvider::new(data)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: if program.kind == ModuleKind::JS { + HostType::Js + } else { + HostType::Wasm + }, + initial_program: program.hash, + bootstrap_generation: 0, + }; + (directory, controller, database) +} + +async fn until(mut condition: impl FnMut() -> bool) { + timeout(Duration::from_secs(5), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +fn client_auth() -> ConnectionAuthCtx { + SpacetimeIdentityClaims { + identity: Identity::ONE, + subject: "local-test".into(), + issuer: "local-test".into(), + audience: Box::new([]), + iat: std::time::SystemTime::now(), + exp: None, + extra: None, + } + .try_into() + .unwrap() +} + +async fn queued_disconnect(js: bool, cancel: bool, id: u64) { + let program = if js { + javascript() + } else { + crate::host::empty_module::program(1).unwrap() + }; + let (_directory, controller, database) = fixture(id, program); + let module = controller + .get_or_launch_module_host(database.clone(), id) + .await + .unwrap(); + let client = ClientActorId::for_test(Identity::ONE); + module + .call_identity_connected(client_auth(), client.connection_id) + .await + .unwrap(); + let db = module.relational_db().clone(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + assert!(tx.st_client_row(client.identity, client.connection_id).is_some()); + let call = tokio::spawn({ + let module = module.clone(); + async move { module.disconnect_client(client).await } + }); + until(|| module.operations.active() == 1).await; + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + if cancel { + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + } else { + drop(call); + } + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + let _ = db.rollback_mut_tx(tx); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + assert!(tx.st_client_row(client.identity, client.connection_id).is_none()); + let _ = db.rollback_mut_tx(tx); + assert!(module.clear_all_clients().await.is_err()); + let result = crate::sql::execute::run( + db.clone(), + "SELECT * FROM st_client".into(), + AuthCtx::new(db.owner_identity(), db.owner_identity()), + None, + None, + &mut Vec::new(), + ) + .await; + assert!(matches!(result, Err(DBError::DatabaseClosed))); + assert!(controller.get_module_host(id).await.is_err()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_wasm_queued_disconnect_survives_cancellation() { + queued_disconnect(false, true, 0xd100).await; + queued_disconnect(false, false, 0xd101).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_js_queued_disconnect_survives_cancellation() { + queued_disconnect(true, true, 0xd102).await; + queued_disconnect(true, false, 0xd103).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_wasm_procedure_owns_external_wait_and_late_commit() { + let id = 0xd104; + let (_directory, controller, database) = fixture(id, crate::host::empty_module::program(1).unwrap()); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let db = module.relational_db().clone(); + let started = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + let call = tokio::spawn({ + let module = module.clone(); + let db = db.clone(); + let started = started.clone(); + let release = release.clone(); + async move { + module + .call_pooled( + "external-wait-test", + (), + async move |_, _| { + started.add_permits(1); + release.acquire().await.unwrap().forget(); + db.with_auto_commit(Workload::Internal, |tx| { + crate::db::environment::set(&db, tx, "AFTER_IO", "committed").map_err(anyhow::Error::from) + }) + .unwrap(); + }, + async |_, _| unreachable!(), + ) + .await + .unwrap(); + } + }); + timeout(Duration::from_secs(5), started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + release.add_permits(1); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let tx = db.begin_tx(Workload::Internal); + assert_eq!( + crate::db::environment::get(&tx, "AFTER_IO").unwrap().as_deref(), + Some("committed") + ); + let _ = db.release_tx(tx); +} + +/// Opt in only for the existing loopback test feature. Production IP filtering +/// stays enabled. Run with proxy variables cleared, as asserted below. +#[cfg(feature = "allow_loopback_http_for_tests")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_js_procedure_owns_actual_http_until_late_commit() { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for variable in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + assert!( + std::env::var_os(variable).is_none(), + "clear {variable} for this disposable loopback test" + ); + } + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let address = listener.local_addr().unwrap(); + assert!(address.ip().is_loopback()); + let started = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + let server = tokio::spawn({ + let started = started.clone(); + let release = release.clone(); + async move { + let (mut socket, peer) = listener.accept().await.unwrap(); + assert!(peer.ip().is_loopback()); + let mut request = [0u8; 4096]; + assert!(socket.read(&mut request).await.unwrap() > 0); + started.add_permits(1); + release.acquire().await.unwrap().forget(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + } + }); + let request = bsatn::to_vec(&spacetimedb_lib::http::Request { + method: spacetimedb_lib::http::Method::Get, + headers: std::iter::empty().collect(), + timeout: None, + uri: format!("http://{address}/owned-test"), + version: spacetimedb_lib::http::Version::Http11, + }) + .unwrap(); + let mut schema = RawModuleDefV10Builder::new(); + schema + .build_table_with_new_type("rows", [("value", AlgebraicType::U64)], true) + .finish(); + schema.add_procedure("task", spacetimedb_sats::ProductType::unit(), AlgebraicType::U64); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + let program = Program::from_bytes(ModuleKind::JS, format!(r#" + import {{register_hooks,table_id_from_name,datastore_insert_bsatn}} from "spacetime:sys@1.0"; + import {{register_hooks as procedures,procedure_http_request,procedure_start_mut_tx,procedure_commit_mut_tx}} from "spacetime:sys@1.2"; + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), __call_reducer__: () => ({{tag:"ok"}})}}); + procedures({{__call_procedure__: () => {{ + procedure_http_request(new Uint8Array({request:?}), ""); + procedure_start_mut_tx(); + datastore_insert_bsatn(table_id_from_name("rows"), new Uint8Array([42,0,0,0,0,0,0,0])); + procedure_commit_mut_tx(); + return new Uint8Array(8); + }} }}); + "#).into_bytes()); + let id = 0xd105; + let (_directory, controller, database) = fixture(id, program); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let db = module.relational_db().clone(); + let call = tokio::spawn({ + let module = module.clone(); + async move { + module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + } + }); + timeout(Duration::from_secs(10), started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(10)).await } + }); + until(|| module.operations.is_closed()).await; + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + release.add_permits(1); + timeout(Duration::from_secs(10), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + server.await.unwrap(); + let tx = db.begin_tx(Workload::Internal); + let table = db.table_id_from_name(&tx, "rows").unwrap().unwrap(); + assert_eq!(tx.table_row_count(table), Some(1)); + let _ = db.release_tx(tx); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_cancelled_js_procedure_startup_retains_physical_slot() { + use futures::StreamExt; + let mut schema = RawModuleDefV10Builder::new(); + schema.add_procedure("task", spacetimedb_sats::ProductType::unit(), AlgebraicType::U64); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + let program = Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{register_hooks,console_log}} from "spacetime:sys@1.0"; + import {{register_hooks as procedures}} from "spacetime:sys@1.2"; + function startup() {{ console_log(2, "physical-startup-entered"); }} + startup(); + const until = Date.now() + 2000; + while (Date.now() < until) {{}} + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), __call_reducer__: () => ({{tag:"ok"}})}}); + procedures({{__call_procedure__: () => new Uint8Array(8)}}); + "# + ) + .into_bytes(), + ); + let id = 0xd106; + let (_directory, controller, database) = fixture(id, program); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let ModuleHostInner::Js(host) = &*module.inner else { + unreachable!() + }; + let slots = host.procedure_instances.instance_slots.as_ref().unwrap(); + let maximum = slots.available_permits(); + let mut logs = module.database_logger().tail(Some(0), true).await.unwrap(); + let call = tokio::spawn({ + let module = module.clone(); + async move { + module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + } + }); + timeout(Duration::from_secs(5), async { + loop { + let log = logs.next().await.unwrap().unwrap(); + if String::from_utf8_lossy(&log).contains("physical-startup-entered") { + break; + } + } + }) + .await + .unwrap(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + assert_eq!(module.operations.active(), 1); + assert_eq!(slots.available_permits(), maximum - 1); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + assert!(!closing.is_finished()); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(slots.available_permits(), maximum); +} diff --git a/crates/core/src/host/module_host/operations.rs b/crates/core/src/host/module_host/operations.rs new file mode 100644 index 00000000000..4906f304a5c --- /dev/null +++ b/crates/core/src/host/module_host/operations.rs @@ -0,0 +1,126 @@ +//! Physical operation ownership, independent of an HTTP/WebSocket waiter's lifetime. +use super::NoSuchModule; +use parking_lot::Mutex; +use std::sync::Arc; +use tokio::sync::{Notify, OwnedSemaphorePermit}; + +#[derive(Default)] +pub(super) struct ModuleOperations { + state: Mutex, + drained: Notify, +} + +#[derive(Default)] +struct State { + closed: bool, + active: usize, +} + +impl ModuleOperations { + pub(super) fn begin(self: &Arc) -> Result { + let mut state = self.state.lock(); + if state.closed { + return Err(NoSuchModule); + } + state.active = state.active.checked_add(1).ok_or(NoSuchModule)?; + Ok(OperationLease { + _token: Arc::new(ActiveOperation(self.clone())), + pool_slot: None, + }) + } + + #[cfg(test)] + pub(super) fn active(&self) -> usize { + self.state.lock().active + } + + #[cfg(test)] + pub(super) fn is_closed(&self) -> bool { + self.state.lock().closed + } + + pub(super) fn close(&self) { + self.state.lock().closed = true; + } + + pub(super) async fn drained(&self) { + loop { + let notified = self.drained.notified(); + tokio::pin!(notified); + // Register before reading the counter, including on a multi-threaded runtime. + notified.as_mut().enable(); + if self.state.lock().active == 0 { + return; + } + notified.await; + } + } +} + +/// Clones describe one admitted operation, not additional admissions. A clone +/// moves into the physical job/request before it is enqueued. Cancellation of +/// the caller therefore cannot release admission or a pooled instance's slot. +#[derive(Clone)] +pub(in crate::host) struct OperationLease { + _token: Arc, + pool_slot: Option>, +} + +impl OperationLease { + pub(super) fn with_pool_slot(mut self, slot: Option>) -> Self { + self.pool_slot = slot; + self + } +} + +struct ActiveOperation(Arc); + +impl Drop for ActiveOperation { + fn drop(&mut self) { + let mut state = self.0.state.lock(); + state.active -= 1; + if state.active == 0 { + self.0.drained.notify_waiters(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn close_counts_previously_admitted_work_even_before_enqueue() { + let operations = Arc::new(ModuleOperations::default()); + let operation = operations.begin().unwrap(); + operations.close(); + assert!(operations.begin().is_err()); + let waiter = tokio::spawn({ + let operations = operations.clone(); + async move { operations.drained().await } + }); + tokio::task::yield_now().await; + assert!(!waiter.is_finished()); + drop(operation); + waiter.await.unwrap(); + operations.drained().await; + } + + #[tokio::test] + async fn cancelled_waiter_does_not_release_physical_job_or_pool_slot() { + let operations = Arc::new(ModuleOperations::default()); + let slots = Arc::new(tokio::sync::Semaphore::new(1)); + let caller = operations + .begin() + .unwrap() + .with_pool_slot(Some(Arc::new(slots.clone().acquire_owned().await.unwrap()))); + let physical_job = caller.clone(); + drop(caller); + operations.close(); + assert_eq!(slots.available_permits(), 0); + assert_eq!(operations.state.lock().active, 1); + drop(physical_job); + operations.drained().await; + assert_eq!(slots.available_permits(), 1); + } +} diff --git a/crates/core/src/host/scheduler.rs b/crates/core/src/host/scheduler.rs index a07dd43597e..4a8c4005a81 100644 --- a/crates/core/src/host/scheduler.rs +++ b/crates/core/src/host/scheduler.rs @@ -321,6 +321,8 @@ impl ScheduledFunctionParams { pub(crate) enum CallScheduledFunctionError { #[error(transparent)] NoSuchModule(#[from] NoSuchModule), + #[error("module instance startup failed: {0}")] + InstanceStartup(anyhow::Error), } #[cfg(target_pointer_width = "64")] @@ -413,6 +415,17 @@ impl SchedulerActor { // If the module already exited, leave the `ScheduledFunction` in // the database for when the module restarts. Err(CallScheduledFunctionError::NoSuchModule(_)) => {} + Err(CallScheduledFunctionError::InstanceStartup(error)) => { + // No transaction or procedure ran. Keep the schedule available + // and retry with a delay, including under shared deadline pressure. + log::warn!("scheduled procedure instance startup failed: {error:#}"); + if startup_retry_is_needed(module_host.info().relational_db(), &item) { + let key = self.queue.insert(item, Duration::from_secs(1)); + if let Some(id) = id { + self.key_map.insert(id, key); + } + } + } Ok(CallScheduledFunctionResult { reschedule: None }) => { // nothing to do } @@ -436,6 +449,26 @@ impl SchedulerActor { } } +/// A cancelled table-backed schedule must not retry forever when every new +/// procedure instance fails before reaching the normal schedule-row lookup. +fn startup_retry_is_needed(db: &RelationalDB, item: &QueueItem) -> bool { + let QueueItem::Id { id, .. } = item else { + return true; + }; + let exists = db.with_read_only(Workload::Internal, |tx| { + db.iter_by_col_eq(tx, id.table_id, id.id_column, &id.schedule_id.into()) + .map(|mut rows| rows.next().is_some()) + }); + match exists { + Ok(exists) => exists, + Err(error) => { + // A failed read is not proof that the user cancelled the schedule. + log::warn!("could not check scheduled procedure startup retry: {error:#}"); + true + } + } +} + #[derive(Debug)] pub(crate) struct CallScheduledFunctionResult { reschedule: Option, @@ -986,3 +1019,61 @@ mod tests { assert_eq!(next, ts(1_400)); } } + +#[cfg(test)] +mod startup_retry_tests { + use super::*; + use crate::db::relational_db::tests_utils::{insert, with_auto_commit, TestDB}; + use spacetimedb_sats::{product, AlgebraicType}; + + fn item(table_id: TableId, schedule_id: u64) -> QueueItem { + QueueItem::Id { + id: ScheduledFunctionId { + table_id, + schedule_id, + id_column: 0.into(), + at_column: 0.into(), + }, + function_name: "task".into(), + at: Timestamp::now(), + } + } + + #[test] + fn execution_deadline_startup_retry_stops_after_schedule_deletion() -> anyhow::Result<()> { + let db = TestDB::in_memory()?; + let table = db.create_table_for_test("pending", &[("id", AlgebraicType::U64)], &[0.into()])?; + with_auto_commit(&db, |tx| { + insert(&db, tx, table, &(7u64,))?; + insert(&db, tx, table, &(8u64,))?; + Ok::<_, anyhow::Error>(()) + })?; + + let cancelled = item(table, 7); + assert!(startup_retry_is_needed(&db, &cancelled)); + with_auto_commit(&db, |tx| { + assert_eq!(db.delete_by_rel(tx, table, [product!(7u64)]), 1); + Ok::<_, anyhow::Error>(()) + })?; + + // An unrelated remaining schedule cannot keep the cancelled ID alive. + assert!(!startup_retry_is_needed(&db, &cancelled)); + assert!(startup_retry_is_needed(&db, &item(table, 8))); + Ok(()) + } + + #[test] + fn execution_deadline_startup_retry_preserves_read_errors_and_volatile_calls() -> anyhow::Result<()> { + let db = TestDB::in_memory()?; + // The point read fails rather than positively observing a missing row. + assert!(startup_retry_is_needed(&db, &item(u32::MAX.into(), 7))); + assert!(startup_retry_is_needed( + &db, + &QueueItem::VolatileNonatomicImmediate { + function_name: "task".into(), + args: FunctionArgs::Nullary, + }, + )); + Ok(()) + } +} diff --git a/crates/core/src/host/v8/execution_deadline.rs b/crates/core/src/host/v8/execution_deadline.rs new file mode 100644 index 00000000000..ac6b1f3d1c9 --- /dev/null +++ b/crates/core/src/host/v8/execution_deadline.rs @@ -0,0 +1,272 @@ +//! Direct cross-thread termination, without executing a V8 interrupt callback. +//! +//! Pinned v8 145's IsolateHandle is Send+Sync and protects termination with its +//! annex mutex. No scope, raw isolate pointer, Rc or Cell leaves the isolate +//! thread. One persistent timer serves bounded active registrations. Finishing +//! or dropping a guard waits out any in-flight termination before the isolate +//! may reset termination or begin another invocation. + +use std::{ + collections::BTreeMap, + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Condvar, Mutex, OnceLock, + }, + time::{Duration, Instant}, +}; + +#[derive(Debug, thiserror::Error)] +#[error("JavaScript execution exceeded its wall-clock limit")] +pub(super) struct ExecutionTimedOut; + +const MAX_ACTIVE_DEADLINES: usize = 16_384; +static ACTIVE_DEADLINES: AtomicUsize = AtomicUsize::new(0); +type Key = (Instant, u64); +static SERVICE: OnceLock, String>> = OnceLock::new(); + +struct Capacity; +impl Capacity { + fn acquire() -> io::Result { + ACTIVE_DEADLINES + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { + (count < MAX_ACTIVE_DEADLINES).then_some(count + 1) + }) + .map_err(|_| io::Error::other("JavaScript execution deadline capacity exhausted"))?; + Ok(Self) + } +} +impl Drop for Capacity { + fn drop(&mut self) { + ACTIVE_DEADLINES.fetch_sub(1, Ordering::Relaxed); + } +} + +struct Registration { + expires: Instant, + state: Mutex, +} +struct State { + finished: bool, + expired: bool, + handle: v8::IsolateHandle, +} +#[derive(Default)] +struct Queue { + sequence: u64, + entries: BTreeMap>, +} +struct DeadlineService { + queue: Mutex, + wake: Condvar, +} + +impl DeadlineService { + fn start() -> io::Result> { + let service = Arc::new(Self { + queue: Mutex::new(Queue::default()), + wake: Condvar::new(), + }); + std::thread::Builder::new().name("v8-deadlines".into()).spawn({ + let service = service.clone(); + move || service.run() + })?; + Ok(service) + } + + fn run(&self) { + loop { + let registration = { + let mut queue = self.queue.lock().unwrap_or_else(|error| error.into_inner()); + loop { + let Some((&(expires, _), _)) = queue.entries.first_key_value() else { + queue = self.wake.wait(queue).unwrap_or_else(|error| error.into_inner()); + continue; + }; + let remaining = expires.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break queue.entries.pop_first().unwrap().1; + } + queue = self + .wake + .wait_timeout(queue, remaining) + .unwrap_or_else(|error| error.into_inner()) + .0; + } + }; + // Never hold the queue lock while acquiring a registration lock. + // Cancellation may hold this lock while removing its queued entry. + let mut state = registration.state.lock().unwrap_or_else(|error| error.into_inner()); + if !state.finished { + state.expired = true; + // finish() must acquire this same lock before resetting V8. + state.handle.terminate_execution(); + } + } + } +} + +pub(super) struct ExecutionDeadline { + service: Arc, + registration: Arc, + key: Key, + finished: bool, + // Includes expired calls still returning from a native syscall. The timer + // popping its queue entry never releases this active-call reservation. + _capacity: Capacity, +} + +impl ExecutionDeadline { + pub fn start(handle: v8::IsolateHandle, timeout: Duration) -> io::Result { + let capacity = Capacity::acquire()?; + let service = SERVICE + .get_or_init(|| DeadlineService::start().map_err(|error| error.to_string())) + .as_ref() + .map_err(|error| io::Error::other(error.clone()))? + .clone(); + let expires = Instant::now() + timeout; + let registration = Arc::new(Registration { + expires, + state: Mutex::new(State { + finished: false, + expired: false, + handle, + }), + }); + let mut queue = service.queue.lock().unwrap_or_else(|error| error.into_inner()); + queue.sequence = queue + .sequence + .checked_add(1) + .ok_or_else(|| io::Error::other("JavaScript execution deadline sequence exhausted"))?; + let key = (expires, queue.sequence); + queue.entries.insert(key, registration.clone()); + drop(queue); + service.wake.notify_one(); + Ok(Self { + service, + registration, + key, + finished: false, + _capacity: capacity, + }) + } + + pub fn finish(mut self) -> bool { + let expired = self.cancel(); + self.finished = true; + expired + } + + fn cancel(&self) -> bool { + let mut state = self + .registration + .state + .lock() + .unwrap_or_else(|error| error.into_inner()); + if !state.finished { + // A timer delayed by OS scheduling must still make a late return + // roll back, even if it has not issued its termination request yet. + state.expired |= Instant::now() >= self.registration.expires; + state.finished = true; + } + let expired = state.expired; + // The timer releases the queue lock before taking this lock, so this + // order cannot invert its locks. Remove completed calls immediately, + // rather than accumulating their registrations for 120 seconds. + self.service + .queue + .lock() + .unwrap_or_else(|error| error.into_inner()) + .entries + .remove(&self.key); + drop(state); + self.service.wake.notify_one(); + expired + } +} + +impl Drop for ExecutionDeadline { + fn drop(&mut self) { + if !self.finished { + self.cancel(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::host::v8::to_value::test::with_scope; + + fn run(scope: &mut v8::PinScope<'_, '_>, source: &str) -> bool { + let source = v8::String::new(scope, source).unwrap(); + v8::Script::compile(scope, source, None).unwrap().run(scope).is_some() + } + + #[test] + fn deadline_terminates_and_isolate_can_run_again() { + with_scope(|scope| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_millis(40)).unwrap(); + assert!(!run(scope, "for (;;) {}")); + assert!(deadline.finish()); + scope.cancel_terminate_execution(); + assert!(run(scope, "1 + 1")); + }); + } + + #[test] + fn finished_and_dropped_registrations_cannot_terminate_later_execution() { + with_scope(|scope| { + for drop_guard in [false, true] { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_millis(40)).unwrap(); + let service = deadline.service.clone(); + let key = deadline.key; + if drop_guard { + drop(deadline); + } else { + assert!(!deadline.finish()); + } + assert!(!service.queue.lock().unwrap().entries.contains_key(&key)); + assert!(run( + scope, + "{ const end = Date.now() + 80; while (Date.now() < end) {} }" + )); + } + for _ in 0..1_000 { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_secs(1)).unwrap(); + let service = deadline.service.clone(); + let key = deadline.key; + assert!(run(scope, "1 + 1")); + assert!(!deadline.finish()); + assert!(!service.queue.lock().unwrap().entries.contains_key(&key)); + } + }); + } + + #[test] + fn late_return_is_expired_even_before_timer_observes_it() { + with_scope(|scope| { + // A service without a timer deterministically models OS delay. + let expires = Instant::now(); + let deadline = ExecutionDeadline { + service: Arc::new(DeadlineService { + queue: Mutex::new(Queue::default()), + wake: Condvar::new(), + }), + registration: Arc::new(Registration { + expires, + state: Mutex::new(State { + finished: false, + expired: false, + handle: scope.thread_safe_handle(), + }), + }), + key: (expires, 1), + finished: false, + _capacity: Capacity::acquire().unwrap(), + }; + assert!(deadline.finish()); + assert!(run(scope, "1 + 1")); + }); + } +} diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 564e7db862d..8593201a430 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -57,6 +57,7 @@ use self::error::{ catch_exception, exception_already_thrown, log_traceback, ErrorOrException, ExcResult, ExceptionThrown, PinTryCatch, Throwable, }; +use self::execution_deadline::{ExecutionDeadline, ExecutionTimedOut}; use self::ser::serialize_to_js; use self::string::{str_from_ident, IntoJsString}; use self::syscall::{ @@ -73,6 +74,7 @@ use crate::client::{ClientActorId, MeteredUnboundedReceiver, MeteredUnboundedSen use crate::config::{V8Config, V8HeapPolicyConfig}; use crate::host::host_controller::CallProcedureReturn; use crate::host::instance_env::{ChunkPool, InstanceEnv, TxSlot}; +use crate::host::module_host::OperationLease; use crate::host::module_host::{ call_identity_connected, init_database, ClientConnectedError, HttpHandlerCallError, OneOffQueryRequest, SqlCommand, SqlCommandResult, ViewCommand, ViewCommandMetric, ViewCommandResult, @@ -85,6 +87,7 @@ use crate::host::wasm_common::module_host_actor::{ ReducerExecuteResult, ReducerOp, ViewExecuteResult, ViewOp, WasmInstance, }; use crate::host::wasm_common::{RowIters, TimingSpanSet}; +use crate::host::ProcedureCallError; use crate::host::{InitDatabaseResult, ModuleHost, ReducerCallError, ReducerCallResult, Scheduler}; use crate::messages::control_db::HostType; use crate::module_host_context::ModuleCreationContext; @@ -110,8 +113,8 @@ use std::cell::Cell; use std::num::NonZeroUsize; use std::os::raw::c_void; use std::panic::{self, AssertUnwindSafe}; -use std::sync::{Arc, LazyLock}; -use std::time::Instant; +use std::sync::{Arc, LazyLock, OnceLock}; +use std::time::{Duration, Instant}; use tokio::sync::{mpsc, oneshot}; use v8::script_compiler::{compile_module, Source}; use v8::{ @@ -123,6 +126,7 @@ mod budget; mod builtins; mod de; mod error; +mod execution_deadline; mod from_value; mod ser; mod string; @@ -154,6 +158,7 @@ impl V8Runtime { program_bytes: &[u8], core: AllocatedJobCore, ) -> anyhow::Result { + self.config.validate_execution_timeout()?; V8_RUNTIME_GLOBAL .make_actor(mcc, program_bytes, core, self.config) .await @@ -256,6 +261,7 @@ impl V8RuntimeInner { load_balance_guard.clone(), core_pinner.clone(), heap_policy, + config.execution_timeout, metrics.clone(), ) .await?; @@ -266,6 +272,7 @@ impl V8RuntimeInner { core_pinner, procedure_instance_pool_size: config.procedure_instance_pool_size, heap_policy: config.heap_policy, + execution_timeout: config.execution_timeout, metrics, }; @@ -281,6 +288,7 @@ pub struct JsModule { core_pinner: CorePinner, procedure_instance_pool_size: NonZeroUsize, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, } @@ -305,7 +313,10 @@ impl JsModule { self.procedure_instance_pool_size } - async fn create_procedure_instance(&self) -> JsProcedureInstance { + async fn create_procedure_instance( + &self, + operation: Option, + ) -> anyhow::Result { let program = self.program.clone(); let common = self.common.clone(); let load_balance_guard = self.load_balance_guard.clone(); @@ -320,15 +331,23 @@ impl JsModule { load_balance_guard, core_pinner, heap_policy, + self.execution_timeout, metrics, + operation, ) - .await - .expect("`spawn_procedure_instance_worker` should succeed when passed `ModuleCommon`"); - instance + .await?; + Ok(instance) } - pub async fn create_instance(&self) -> JsProcedureInstance { - self.create_procedure_instance().await + pub(in crate::host) async fn create_instance_for_operation( + &self, + operation: Option, + ) -> anyhow::Result { + self.create_procedure_instance(operation).await + } + + pub async fn create_instance(&self) -> anyhow::Result { + self.create_procedure_instance(None).await } } @@ -448,7 +467,8 @@ impl JsInstanceEnv { /// and friends. #[derive(Clone)] pub struct JsMainInstance { - tx: MeteredUnboundedSender, + tx: MeteredUnboundedSender>, + operation: Option, } /// A procedure instance for a [`JsModule`]. @@ -456,16 +476,48 @@ pub struct JsMainInstance { /// Procedure instances are checked out exclusively from the procedure pool and /// only execute procedure-style requests. pub struct JsProcedureInstance { - tx: mpsc::Sender, + tx: mpsc::Sender>, + operation: Option, + startup_failure: ProcedureStartupStatus, +} + +// Set only when a replacement generation fails before receiving another +// request. Publishing before closing the receiver proves queued calls did not +// execute. Unexplained worker exits and actual Rust panics remain fatal. +type ProcedureStartupStatus = Arc>; + +#[derive(Clone, Debug, thiserror::Error)] +#[error("procedure isolate startup failed: {0}")] +pub(in crate::host) struct JsProcedureStartupError(Arc); + +struct PhysicalRequest { + request: R, + operation: Option, } impl JsMainInstance { + pub(in crate::host) fn with_operation(mut self, operation: OperationLease) -> Self { + self.operation = Some(operation); + self + } + async fn request(&self, request: R) -> R::Response { - send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| request.into_worker_request(reply_tx)).await + send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| PhysicalRequest { + request: request.into_worker_request(reply_tx), + operation: self.operation.clone(), + }) + .await } async fn send_detached_request(&self, ctx: &'static str, request: JsMainWorkerRequest) { - if self.tx.send(request).is_err() { + if self + .tx + .send(PhysicalRequest { + request, + operation: self.operation.clone(), + }) + .is_err() + { panic!("JS worker exited before accepting `{ctx}`"); } } @@ -475,11 +527,13 @@ impl JsMainInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { self.request(UpdateDatabaseRequest { program, old_module_info, policy, + deployment, }) .await } @@ -535,8 +589,12 @@ impl JsMainInstance { self.request(DisconnectClientRequest { client_id }).await } - pub async fn init_database(&self, program: Program) -> anyhow::Result { - self.request(InitDatabaseRequest { program }).await + pub async fn init_database( + &self, + program: Program, + deployment: Option, + ) -> anyhow::Result { + self.request(InitDatabaseRequest { program, deployment }).await } pub async fn call_view(&self, cmd: ViewCommand) -> ViewCommandResult { @@ -620,6 +678,7 @@ js_main_request! { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, } => "update_database", anyhow::Result, UpdateDatabase } @@ -662,6 +721,7 @@ js_main_request! { js_main_request! { InitDatabaseRequest { program: Program, + deployment: Option, } => "init_database", anyhow::Result, InitDatabase } @@ -678,16 +738,24 @@ js_main_request! { } impl JsProcedureInstance { + pub(in crate::host) fn set_operation(&mut self, operation: Option) { + self.operation = operation; + } + pub(in crate::host) fn is_closed(&self) -> bool { - self.tx.is_closed() + self.startup_failure.get().is_some() || self.tx.is_closed() } async fn send_request( &self, ctx: &'static str, request: impl FnOnce(JsReplyTx) -> JsProcedureWorkerRequest, - ) -> T { - send_js_request(ctx, &self.tx, request).await + ) -> Result { + send_js_request(ctx, &self.tx, &self.startup_failure, |reply_tx| PhysicalRequest { + request: request(reply_tx), + operation: self.operation.clone(), + }) + .await } pub async fn call_procedure(&self, params: CallProcedureParams) -> CallProcedureReturn { @@ -696,6 +764,10 @@ impl JsProcedureInstance { params, }) .await + .unwrap_or_else(|error| CallProcedureReturn { + result: Err(ProcedureCallError::InternalError(error.to_string())), + tx_offset: None, + }) } pub async fn call_http_handler( @@ -706,25 +778,38 @@ impl JsProcedureInstance { JsProcedureWorkerRequest::CallHttpHandler { reply_tx, params } }) .await + .map_err(|error| HttpHandlerCallError::InternalError(error.to_string()))? } - pub(in crate::host) async fn enqueue_procedure(&self, params: CallProcedureParams) -> JsProcedureCall { + pub(in crate::host) async fn enqueue_procedure( + &self, + params: CallProcedureParams, + ) -> Result { let (reply_tx, reply_rx) = oneshot::channel(); if self .tx - .send(JsProcedureWorkerRequest::CallProcedure { reply_tx, params }) + .send(PhysicalRequest { + request: JsProcedureWorkerRequest::CallProcedure { reply_tx, params }, + operation: self.operation.clone(), + }) .await .is_err() { + if let Some(error) = self.startup_failure.get() { + return Err(error.clone()); + } panic!("JS worker exited before accepting `call_procedure`"); } - JsProcedureCall { reply_rx } + Ok(JsProcedureCall { + reply_rx, + startup_failure: self.startup_failure.clone(), + }) } pub(in crate::host) async fn call_scheduled_procedure( &self, params: ScheduledFunctionParams, - ) -> CallScheduledFunctionResult { + ) -> Result { self.send_request("scheduled_procedure", |reply_tx| { JsProcedureWorkerRequest::ScheduledProcedure { reply_tx, params } }) @@ -735,26 +820,33 @@ impl JsProcedureInstance { async fn send_js_request( ctx: &'static str, tx: &mpsc::Sender, + startup_failure: &ProcedureStartupStatus, request: impl FnOnce(JsReplyTx) -> Req, -) -> T +) -> Result where Req: Send + 'static, { let (reply_tx, reply_rx) = oneshot::channel(); if tx.send(request(reply_tx)).await.is_err() { + if let Some(error) = startup_failure.get() { + return Err(error.clone()); + } panic!("JS worker exited before accepting `{ctx}`"); } match reply_rx.await { - Ok(Ok(value)) => value, + Ok(Ok(value)) => Ok(value), Ok(Err(panic)) => panic::resume_unwind(panic), - Err(_) => panic!("JS worker exited before replying to `{ctx}`"), + Err(_) => match startup_failure.get() { + Some(error) => Err(error.clone()), + None => panic!("JS worker exited before replying to `{ctx}`"), + }, } } -async fn send_js_unbounded_request( +async fn send_js_unbounded_request( ctx: &'static str, - tx: &MeteredUnboundedSender, - request: impl FnOnce(JsReplyTx) -> JsMainWorkerRequest, + tx: &MeteredUnboundedSender, + request: impl FnOnce(JsReplyTx) -> Req, ) -> T { let (reply_tx, reply_rx) = oneshot::channel(); if tx.send(request(reply_tx)).is_err() { @@ -774,11 +866,13 @@ pub(in crate::host) type JsFatalHook = Arc; pub(in crate::host) struct JsProcedureCall { reply_rx: oneshot::Receiver>, + startup_failure: ProcedureStartupStatus, } pub(in crate::host) enum JsProcedureCallCompletion { Completed(CallProcedureReturn), Panicked, + StartupFailed(JsProcedureStartupError), WorkerExited, } @@ -787,7 +881,10 @@ impl JsProcedureCall { match self.reply_rx.await { Ok(Ok(ret)) => JsProcedureCallCompletion::Completed(ret), Ok(Err(_panic)) => JsProcedureCallCompletion::Panicked, - Err(_) => JsProcedureCallCompletion::WorkerExited, + Err(_) => match self.startup_failure.get() { + Some(error) => JsProcedureCallCompletion::StartupFailed(error.clone()), + None => JsProcedureCallCompletion::WorkerExited, + }, } } } @@ -804,6 +901,7 @@ enum JsMainWorkerRequest { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, }, /// See [`JsMainInstance::call_reducer`]. CallReducer { @@ -864,6 +962,7 @@ enum JsMainWorkerRequest { InitDatabase { reply_tx: JsReplyTx>, program: Program, + deployment: Option, }, } @@ -886,7 +985,7 @@ enum JsProcedureWorkerRequest { }, } -static_assert_size!(CallReducerParams, 192); +static_assert_size!(CallReducerParams, 208); fn send_worker_reply(ctx: &str, reply_tx: JsReplyTx, value: T) { if reply_tx.send(Ok(value)).is_err() { @@ -1177,7 +1276,9 @@ fn startup_instance_worker<'scope>( scope: &mut PinScope<'scope, '_>, program: Arc, module_or_mcc: Either, + execution_timeout: Duration, ) -> anyhow::Result<(HookFunctions<'scope>, ModuleCommon)> { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), execution_timeout)?; let hook_functions = catch_exception(scope, |scope| { // Start-up the user's module. let exports_obj = eval_user_module(scope, &program)?; @@ -1186,13 +1287,19 @@ fn startup_instance_worker<'scope>( let hooks = get_hooks(scope, exports_obj)?.ok_or_else(|| anyhow::anyhow!("must export schema as default export"))?; Ok(hooks) - })?; + }); + let expired = deadline.finish(); + scope.cancel_terminate_execution(); + if expired { + return Err(ExecutionTimedOut.into()); + } + let hook_functions = hook_functions?; // If we don't have a module, make one. let module_common = match module_or_mcc { Either::Left(module_common) => module_common, Either::Right(mcc) => { - let def = extract_description(scope, &hook_functions, &mcc.replica_ctx)?; + let def = extract_description(scope, &hook_functions, &mcc.replica_ctx, execution_timeout)?; // Validate and create a common module from the raw definition. build_common_module_from_raw(mcc, def)? @@ -1255,6 +1362,7 @@ async fn spawn_main_instance_worker( load_balance_guard: Arc, core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, ) -> anyhow::Result<(ModuleCommon, JsMainInstance)> { spawn_instance_worker::( @@ -1263,18 +1371,23 @@ async fn spawn_main_instance_worker( load_balance_guard, core_pinner, heap_policy, + execution_timeout, metrics, + None, ) .await } +#[allow(clippy::too_many_arguments)] async fn spawn_procedure_instance_worker( program: Arc, module_or_mcc: Either, load_balance_guard: Arc, core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, + operation: Option, ) -> anyhow::Result<(ModuleCommon, JsProcedureInstance)> { spawn_instance_worker::( program, @@ -1282,7 +1395,9 @@ async fn spawn_procedure_instance_worker( load_balance_guard, core_pinner, heap_policy, + execution_timeout, metrics, + operation, ) .await } @@ -1301,9 +1416,9 @@ trait JsWorkerSpec { fn channel(database_identity: &Identity) -> (Self::Sender, Self::Receiver); - fn make_instance(tx: Self::Sender) -> Self::Instance; + fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance; - fn blocking_recv(rx: &mut Self::Receiver) -> Option; + fn blocking_recv(rx: &mut Self::Receiver) -> Option>; fn handle_request( request: Self::Request, @@ -1317,8 +1432,8 @@ trait JsWorkerSpec { impl JsWorkerSpec for MainJsWorker { type Request = JsMainWorkerRequest; type Instance = JsMainInstance; - type Sender = MeteredUnboundedSender; - type Receiver = MeteredUnboundedReceiver; + type Sender = MeteredUnboundedSender>; + type Receiver = MeteredUnboundedReceiver>; const KIND: JsWorkerKind = JsWorkerKind::Main; @@ -1333,11 +1448,11 @@ impl JsWorkerSpec for MainJsWorker { ) } - fn make_instance(tx: Self::Sender) -> Self::Instance { - JsMainInstance { tx } + fn make_instance(tx: Self::Sender, _startup_failure: ProcedureStartupStatus) -> Self::Instance { + JsMainInstance { tx, operation: None } } - fn blocking_recv(rx: &mut Self::Receiver) -> Option { + fn blocking_recv(rx: &mut Self::Receiver) -> Option> { rx.blocking_recv() } @@ -1355,8 +1470,8 @@ impl JsWorkerSpec for MainJsWorker { impl JsWorkerSpec for ProcedureJsWorker { type Request = JsProcedureWorkerRequest; type Instance = JsProcedureInstance; - type Sender = mpsc::Sender; - type Receiver = mpsc::Receiver; + type Sender = mpsc::Sender>; + type Receiver = mpsc::Receiver>; const KIND: JsWorkerKind = JsWorkerKind::Procedure; @@ -1364,11 +1479,15 @@ impl JsWorkerSpec for ProcedureJsWorker { mpsc::channel(JS_PROCEDURE_INSTANCE_QUEUE_CAPACITY) } - fn make_instance(tx: Self::Sender) -> Self::Instance { - JsProcedureInstance { tx } + fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance { + JsProcedureInstance { + tx, + startup_failure, + operation: None, + } } - fn blocking_recv(rx: &mut Self::Receiver) -> Option { + fn blocking_recv(rx: &mut Self::Receiver) -> Option> { rx.blocking_recv() } @@ -1398,8 +1517,9 @@ fn handle_main_worker_request( program, old_module_info, policy, + deployment, } => handle_worker_request("update_database", reply_tx, || { - let res = instance_common.update_database(program, old_module_info, policy, inst); + let res = instance_common.update_database(program, old_module_info, policy, deployment, inst); (res, false) }), JsMainWorkerRequest::CallReducer { reply_tx, params } => { @@ -1496,14 +1616,22 @@ fn handle_main_worker_request( (res, trapped) }) } - JsMainWorkerRequest::InitDatabase { reply_tx, program } => { - handle_worker_request("init_database", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); - let (res, trapped): (Result, bool) = - init_database(replica_ctx, &info.module_def, program, call_reducer); - (res, trapped) - }) - } + JsMainWorkerRequest::InitDatabase { + reply_tx, + program, + deployment, + } => handle_worker_request("init_database", reply_tx, || { + let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let (res, trapped): (Result, bool) = init_database( + replica_ctx, + &info.module_def, + info.module_hash, + program, + deployment, + call_reducer, + ); + (res, trapped) + }), } } @@ -1569,21 +1697,24 @@ fn spawn_v8_worker_thread(worker_kind: JsWorkerKind, database_identity: Identity /// Spawns an instance worker for `program` and returns on success the /// corresponding instance handle that talks to the worker. /// -/// When [`ModuleCommon`] is passed, it's assumed that this program has already -/// been validated. In that case, `Ok(_)` should be returned. +/// When [`ModuleCommon`] is passed, the program has already been validated. +/// Starting another isolate can still fail, including its execution deadline. /// /// Otherwise, when [`ModuleCreationContext`] is passed, this is the first time /// both the module and instance are created. /// /// `load_balance_guard` and `core_pinner` should both be from the same /// [`AllocatedJobCore`], and are used to manage the core pinning of this thread. +#[allow(clippy::too_many_arguments)] async fn spawn_instance_worker( program: Arc, module_or_mcc: Either, load_balance_guard: Arc, mut core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, instance_metrics: InstanceManagerMetrics, + operation: Option, ) -> anyhow::Result<(ModuleCommon, W::Instance)> where W: JsWorkerSpec + 'static, @@ -1596,6 +1727,8 @@ where Either::Right(mcc) => mcc.replica_ctx.database_identity, }; let (request_tx, mut request_rx) = W::channel(&database_identity); + let startup_failure = ProcedureStartupStatus::default(); + let worker_startup_failure = startup_failure.clone(); let rt = tokio::runtime::Handle::current(); @@ -1608,6 +1741,7 @@ where let mut startup_result_tx = Some(result_tx); let mut module_common_for_recreate = None::; + let mut physical_operation = operation; 'worker: loop { let replacing_instance = module_common_for_recreate.is_some(); let generation_start_time = replacing_instance.then(Instant::now); @@ -1640,7 +1774,7 @@ where .expect("our builtin code shouldn't error"); // Setup the JS module, find call_reducer, and maybe build the module. - startup_instance_worker(scope, program.clone(), generation_module_or_mcc) + startup_instance_worker(scope, program.clone(), generation_module_or_mcc, execution_timeout) })); let (hooks, module_common) = match startup_result { @@ -1663,6 +1797,7 @@ where log::warn!("startup result receiver disconnected"); } } else { + let _ = worker_startup_failure.set(JsProcedureStartupError(format!("{err:#}").into())); log::error!("failed to restart JS worker: {err:#}"); } return; @@ -1711,6 +1846,7 @@ where .v8_heap_limit_hit .with_label_values(&info.database_identity), initial_heap_limit: heap_policy.heap_limit_bytes, + execution_timeout, }; let _initial_heap_stats = record_heap_stats(inst.scope, &mut heap_metrics); @@ -1718,9 +1854,11 @@ where // // The loop is terminated when the last worker instance handle is dropped. // This will cause channels, scopes, and the isolate to be cleaned up. + physical_operation.take(); let mut requests_since_heap_check = 0u64; let mut last_heap_check_at = Instant::now(); - while let Some(request) = W::blocking_recv(&mut request_rx) { + while let Some(PhysicalRequest { request, operation }) = W::blocking_recv(&mut request_rx) { + physical_operation = operation; core_pinner.pin_if_changed(); let mut outcome = @@ -1751,7 +1889,9 @@ where } match outcome { - WorkerRequestOutcome::Continue => {} + WorkerRequestOutcome::Continue => { + physical_operation.take(); + } WorkerRequestOutcome::RecreateInstance => { instance_metrics.track_instance_removed(); continue 'worker; @@ -1767,7 +1907,7 @@ where // Get the module, if any, and get any setup errors from the worker. let res: Result = result_rx.await.expect("should have a sender"); res.map(|opt_mc| { - let inst = W::make_instance(request_tx); + let inst = W::make_instance(request_tx, startup_failure); (opt_mc, inst) }) } @@ -1857,11 +1997,12 @@ struct V8Instance<'a, 'scope, 'isolate> { /// Metric for the number of times the v8 heap limit has been hit. heap_limit_hit_metric: &'a IntCounter, initial_heap_limit: usize, + execution_timeout: Duration, } impl WasmInstance for V8Instance<'_, '_, '_> { fn extract_descriptions(&mut self) -> Result { - extract_description(self.scope, self.hooks, self.replica_ctx) + extract_description(self.scope, self.hooks, self.replica_ctx, self.execution_timeout) } fn replica_ctx(&self) -> &Arc { @@ -1982,13 +2123,14 @@ where // are released when the reducer/view/procedure returns. v8::scope!(let scope, scope); - // TODO(v8): Start the budget timeout and long-running logger. let env = env_on_isolate_unwrap(scope); // Start the timer. // We'd like this tightly around `call`. env.start_funcall(op.name().clone(), op.timestamp(), op.call_type()); env.instance_env.set_call_auth_flags(op.call_auth_flags()); + env.instance_env + .set_hosted_auth(op.hosted_auth().map(|proof| std::sync::Arc::new(proof.clone()))); // Wrap the call in `TryCatch`. // @@ -1997,7 +2139,12 @@ where // opened by the caller before entering `common_call`. v8::tc_scope!(let scope, scope); - let call_result = call(scope, inst.hooks, op).map_err(|mut e| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), inst.execution_timeout); + let mut call_result = match &deadline { + Ok(_) => call(scope, inst.hooks, op), + Err(error) => Err(anyhow::anyhow!("cannot start JavaScript execution deadline: {error}").into()), + } + .map_err(|mut e| { if let ErrorOrException::Exception(_) = e { // If we're terminating execution, don't try to check `instanceof`. if scope.can_continue() @@ -2017,10 +2164,7 @@ where // We can continue. ExecutionError::Recoverable(e.unwrap_or_else(Into::into)) } else if scope.has_terminated() { - // We can continue if we do `Isolate::cancel_terminate_execution`. - // Must be called *after* we check `has_terminated()`, or else it will - // cause it to return `false`. - scope.cancel_terminate_execution(); + // Reset only after synchronizing with this invocation's timer. let e = e.unwrap_or_else(|unknown| termination_error.unwrap_or_else(|| unknown.into())); ExecutionError::Recoverable(e) } else { @@ -2029,6 +2173,14 @@ where } }); + // The timer also covers exception inspection, which can execute user + // getters or Symbol.hasInstance. Synchronizing with it before resetting prevents a + // delayed timer from terminating the next invocation of this isolate. + if deadline.is_ok_and(ExecutionDeadline::finish) { + // Even a call that returned at the exact boundary must roll back. + call_result = Err(ExecutionError::Recoverable(ExecutionTimedOut.into())); + } + // Ensure there's no lingering termination request. termination_flag.clear(); scope.cancel_terminate_execution(); @@ -2065,14 +2217,22 @@ fn extract_description<'scope>( scope: &mut PinScope<'scope, '_>, hooks: &HookFunctions<'_>, replica_ctx: &ReplicaContext, + execution_timeout: Duration, ) -> Result { run_describer( |a, b, c| log_traceback(replica_ctx, a, b, c), || { - Ok(catch_exception(scope, |scope| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), execution_timeout)?; + let result = catch_exception(scope, |scope| { let def = call_describe_module(scope, hooks)?; Ok(def) - })?) + }); + let expired = deadline.finish(); + scope.cancel_terminate_execution(); + if expired { + return Err(ExecutionTimedOut.into()); + } + Ok(result?) }, ) } @@ -2115,6 +2275,7 @@ mod test { caller_identity: &Identity::ONE, caller_connection_id: &ConnectionId::ZERO, call_auth_flags: 0, + hosted_auth: None, timestamp: Timestamp::from_micros_since_unix_epoch(24), args: &ArgsTuple::nullary(), }; diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index 29aedea9b7b..1351d02d8b7 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -45,6 +45,7 @@ pub fn call_call_procedure( caller_identity: sender, caller_connection_id: connection_id, call_auth_flags: _, + hosted_auth: _, timestamp, arg_bytes: procedure_args, } = op; @@ -450,13 +451,19 @@ pub fn console_log<'scope>( let mut buf = scratch_buf::<128>(); let msg = msg.to_rust_cow_lossy(scope, &mut buf); - let frame: Local<'_, v8::StackFrame> = v8::StackTrace::current_stack_trace(scope, 2) - .ok_or_else(exception_already_thrown)? - .get_frame(scope, 1) - .ok_or_else(exception_already_thrown)?; + let trace = v8::StackTrace::current_stack_trace(scope, 2).ok_or_else(exception_already_thrown)?; + // The normal bindings add a logging wrapper, but modules may call this + // syscall directly, including from top-level code. V8's GetFrame does not + // check the index in release builds, so never request an absent frame. + let frame = match trace.get_frame_count() { + 0 => None, + 1 => trace.get_frame(scope, 0), + _ => trace.get_frame(scope, 1), + }; + let line_number = frame.map(|frame| frame.get_line_number() as u32); let mut buf = scratch_buf::<32>(); let filename = frame - .get_script_name(scope) + .and_then(|frame| frame.get_script_name(scope)) .map(|s| s.to_rust_cow_lossy(scope, &mut buf)); let level = (level as u8).into(); @@ -470,7 +477,7 @@ pub fn console_log<'scope>( tracing::error!( "`JsInstanceEnv` unavailable while processing guest log at {}:{}: {msg}", filename.as_deref().unwrap_or("unknown"), - frame.get_line_number() + line_number.unwrap_or_default() ); })?; @@ -480,7 +487,7 @@ pub fn console_log<'scope>( ts: InstanceEnv::now_for_logging(), target: None, filename: filename.as_deref(), - line_number: Some(frame.get_line_number() as u32), + line_number, function, message: &msg, }; diff --git a/crates/core/src/host/v8/syscall/v1.rs b/crates/core/src/host/v8/syscall/v1.rs index 4d6783465df..ab852eda401 100644 --- a/crates/core/src/host/v8/syscall/v1.rs +++ b/crates/core/src/host/v8/syscall/v1.rs @@ -496,6 +496,7 @@ pub(super) fn call_call_reducer( caller_identity: sender, caller_connection_id: conn_id, call_auth_flags: _, + hosted_auth: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index 95783ddddb6..27766216b55 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -480,6 +480,7 @@ pub(super) fn call_call_reducer<'scope>( caller_identity: sender, caller_connection_id: conn_id, call_auth_flags: _, + hosted_auth: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 95df05f3616..ab4f77b9fe3 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -1,7 +1,10 @@ use super::instrumentation::CallTimes; use super::*; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::check_hosted_admission; use crate::client::ClientActorId; use crate::database_logger; +use crate::db::deployment::{self, CommitAdmission, DeploymentCommit}; use crate::db::sql::ast::SchemaViewer; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; @@ -347,7 +350,7 @@ pub enum InitializationError { #[error(transparent)] Validation(#[from] ValidationError), #[error(transparent)] - ModuleValidation(#[from] spacetimedb_schema::error::ValidationErrors), + ModuleValidation(#[from] Box), #[error("setup function returned an error: {0}")] Setup(Box), #[error("wasm trap while calling {func:?}")] @@ -489,9 +492,10 @@ impl WasmModuleInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { self.common - .update_database(program, old_module_info, policy, &mut self.instance) + .update_database(program, old_module_info, policy, deployment, &mut self.instance) } pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { @@ -545,11 +549,23 @@ impl WasmModuleInstance { res } - pub fn init_database(&mut self, program: Program) -> anyhow::Result { - let module_def = &self.common.info.clone().module_def; + pub fn init_database( + &mut self, + program: Program, + deployment: Option, + ) -> anyhow::Result { + let info = self.common.info.clone(); + let module_def = &info.module_def; let replica_ctx = &self.instance.replica_ctx().clone(); let call_reducer = |tx, params| self.call_reducer_with_tx_offset(tx, params); - let (res, trapped) = init_database(replica_ctx, module_def, program, call_reducer); + let (res, trapped) = init_database( + replica_ctx, + module_def, + info.module_hash, + program, + deployment, + call_reducer, + ); self.trapped = trapped; res } @@ -649,12 +665,82 @@ impl InstanceCommon { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, inst: &mut I, ) -> Result { let replica_ctx = inst.replica_ctx().clone(); let system_logger = replica_ctx.logger.system_logger(); let stdb = &replica_ctx.relational_db(); + let timestamp = Timestamp::now(); + let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, (admission, unchanged)) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<_> { + ensure!( + self.info.module_hash == program.hash, + "program does not match the instantiated module" + ); + let admission = if let Some(request) = &deployment { + deployment::validate_deployment_program(request, &program, &self.info.module_def)?; + deployment::check_deployment_commit(tx, request, timestamp, &Default::default())? + } else { + deployment::require_unmanaged_publication(tx)?; + CommitAdmission::Ready + }; + if matches!(admission, CommitAdmission::AlreadyCommitted(_)) { + return Ok((admission, false)); + } + deployment::validate_active_hosted_grants(tx, &self.info.module_def)?; + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use spacetimedb_datastore::system_tables::{read_hash_from_col, StModuleFields, ST_MODULE_ID}; + let row = tx.iter(ST_MODULE_ID)?.next().context("database is not initialized")?; + let stored_hash = read_hash_from_col(row, StModuleFields::ProgramHash)?; + ensure!( + stored_hash == old_module_info.module_hash, + "module changed before publication admission" + ); + Ok((admission, stored_hash == program.hash)) + })?; + if let CommitAdmission::AlreadyCommitted(result) = admission { + let (offset, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + let (sender, tx_offset) = tokio::sync::oneshot::channel(); + let _ = sender.send(offset); + return Ok(UpdateDatabaseResult::DeploymentAlreadyCommitted { + result, + tx_offset, + durable_offset: stdb.durable_tx_offset(), + }); + } + if unchanged { + let Some(request) = &deployment else { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + return Ok(UpdateDatabaseResult::NoUpdateNeeded); + }; + let (tx, _) = stdb.with_auto_rollback(tx, |tx| { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default()) + })?; + let event = ModuleEvent { + timestamp, + caller_identity: request.publisher, + caller_connection_id: None, + function_call: ModuleFunctionCall::update(), + status: EventStatus::Committed(DatabaseUpdate::default()), + reducer_return_value: None, + execution_budget_used: FunctionBudget::ZERO, + host_execution_duration: Duration::ZERO, + request_id: None, + timer: None, + }; + let durable_offset = stdb.durable_tx_offset(); + let CommitAndBroadcastEventSuccess { tx_offset, .. } = + commit_and_broadcast_event(&self.info.subscriptions, None, event, tx); + return Ok(UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + }); + } + let plan: MigratePlan = match policy.try_migrate( self.info.database_identity, old_module_info.module_hash, @@ -664,17 +750,24 @@ impl InstanceCommon { ) { Ok(plan) => plan, Err(e) => { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); return match e { MigrationPolicyError::AutoMigrateFailure(e) => Ok(UpdateDatabaseResult::AutoMigrateError(e.into())), _ => Ok(UpdateDatabaseResult::ErrorExecutingMigration(e.into())), - } + }; } }; let program_hash = program.hash; let host_type = HostType::from(program.kind); - let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| stdb.update_program(tx, program))?; + let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + stdb.update_program(tx, program)?; + if let Some(request) = &deployment { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default())?; + } + Ok(()) + })?; system_logger.info(&format!("Updated program to {program_hash}")); let auth_ctx = AuthCtx::for_current(replica_ctx.database.owner_identity); @@ -769,7 +862,9 @@ impl InstanceCommon { tx: MutTxId, inst: &mut I, ) -> Result<(ViewCallResult, u32, bool), anyhow::Error> { - let view_calls = collect_subscribed_view_calls(&tx, &self.info.module_def, self.info.owner_identity)?; + let (tx, view_calls) = inst.replica_ctx().relational_db().with_auto_rollback(tx, |tx| { + collect_subscribed_view_calls(tx, &self.info.module_def, self.info.owner_identity) + })?; Ok(self.execute_view_calls(tx, view_calls, inst)) } @@ -783,11 +878,28 @@ impl InstanceCommon { caller_identity, caller_connection_id, call_auth_flags, + hosted_auth, timer, procedure_id, args, } = params; + let admission = inst + .replica_ctx() + .relational_db() + .with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, inst.replica_ctx().relational_db(), hosted_auth.as_deref()) + }); + if let Err(err) = admission { + return ( + CallProcedureReturn { + result: Err(ProcedureCallError::InternalError(err.to_string())), + tx_offset: None, + }, + false, + ); + } + // We've already validated by this point that the procedure exists, // so it's fine to use the panicking `procedure_by_id`. let procedure_def = self.info.module_def.procedure_by_id(procedure_id); @@ -803,6 +915,7 @@ impl InstanceCommon { caller_identity, caller_connection_id, call_auth_flags, + hosted_auth, timestamp, arg_bytes: args.get_bsatn().clone(), }; @@ -968,6 +1081,7 @@ impl InstanceCommon { caller_identity, caller_connection_id, call_auth_flags, + hosted_auth, client, request_id, reducer_id, @@ -992,12 +1106,44 @@ impl InstanceCommon { caller_identity: &caller_identity, caller_connection_id: &caller_connection_id, call_auth_flags, + hosted_auth, timestamp, args: &args, }; let workload = Workload::Reducer(ReducerContext::from(op.clone())); let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable, workload)); + if let Err(err) = check_hosted_admission(&tx, stdb, op.hosted_auth.as_deref()) { + let event = ModuleEvent { + timestamp, + caller_identity, + caller_connection_id: caller_connection_id_opt, + function_call: ModuleFunctionCall { + reducer: Some(reducer_name.clone()), + reducer_id, + args, + }, + status: EventStatus::FailedInternal(err.to_string()), + reducer_return_value: None, + execution_budget_used: FunctionBudget::ZERO, + host_execution_duration: Default::default(), + request_id, + timer, + }; + let CommitAndBroadcastEventSuccess { event, tx_offset, .. } = + commit_and_broadcast_event(&info.subscriptions, client, event, tx); + return ( + ReducerCallResultWithTxOffset { + result: ReducerCallResult { + outcome: ReducerOutcome::from(&event.status), + execution_budget_used: FunctionBudget::ZERO, + execution_duration: Default::default(), + }, + tx_offset, + }, + false, + ); + } let mut tx_slot = inst.tx_slot(); let vm_metrics = self.vm_metrics.get_for_reducer_id(reducer_id); @@ -1860,6 +2006,9 @@ pub trait InstanceOp { fn call_auth_flags(&self) -> u32 { 0 } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + None + } } /// Describes a view call in a cheaply shareable way. @@ -1921,12 +2070,19 @@ pub struct ReducerOp<'a> { pub caller_identity: &'a Identity, pub caller_connection_id: &'a ConnectionId, pub call_auth_flags: u32, + pub hosted_auth: Option>, pub timestamp: Timestamp, /// The arguments passed to the reducer. pub args: &'a ArgsTuple, } impl InstanceOp for ReducerOp<'_> { + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + self.hosted_auth.as_deref() + } fn name(&self) -> &NamespacedIdentifier { self.name.as_namespaced() } @@ -1936,9 +2092,6 @@ impl InstanceOp for ReducerOp<'_> { fn call_type(&self) -> FuncCallType { FuncCallType::Reducer } - fn call_auth_flags(&self) -> u32 { - self.call_auth_flags - } } impl From> for execution_context::ReducerContext { @@ -1949,6 +2102,7 @@ impl From> for execution_context::ReducerContext { caller_identity, caller_connection_id, call_auth_flags: _, + hosted_auth: _, timestamp, args, }: ReducerOp<'_>, @@ -1971,11 +2125,18 @@ pub struct ProcedureOp { pub caller_identity: Identity, pub caller_connection_id: ConnectionId, pub call_auth_flags: u32, + pub hosted_auth: Option>, pub timestamp: Timestamp, pub arg_bytes: Bytes, } impl InstanceOp for ProcedureOp { + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + self.hosted_auth.as_deref() + } fn name(&self) -> &NamespacedIdentifier { &self.name } @@ -1985,9 +2146,6 @@ impl InstanceOp for ProcedureOp { fn call_type(&self) -> FuncCallType { FuncCallType::Procedure } - fn call_auth_flags(&self) -> u32 { - self.call_auth_flags - } } /// Describes an HTTP handler call in a cheaply shareable way. diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index c35514864da..bf6e24eaac0 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -338,14 +338,6 @@ impl WasmInstanceEnv { self.bytes_sinks.remove(&sink).unwrap_or_default() } - pub fn get_call_auth_flags(caller: Caller<'_, Self>) -> u32 { - caller.data().instance_env.get_call_auth_flags() - } - - pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { - self.instance_env.set_call_auth_flags(flags); - } - /// Signal to this `WasmInstanceEnv` that a reducer or procedure call is beginning. /// /// Returns the handle used by reducers and procedures to read from `args` @@ -1591,34 +1583,6 @@ impl WasmInstanceEnv { }) } - /// Read an environment value as a nullable BytesSource. Zero means missing; - /// a present empty string always receives a nonzero, consumable source. - pub fn env_get( - caller: Caller<'_, Self>, - key: WasmPtr, - key_len: u32, - target_ptr: WasmPtr, - ) -> RtResult { - Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { - if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { - return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); - } - let (mem, env) = Self::mem_env(caller); - let key = mem.deref_str(key, key_len)?; - match env.instance_env.env_get(key)? { - None => Ok(0), - Some(value) => { - // These buffers live on the host heap until consumed or the - // invocation ends. Bound retained reads from hand-written Wasm. - if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { - return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); - } - Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) - } - } - }) - } - /// Finds the JWT payload associated with `connection_id`. /// A `[ByteSourceId]` for the payload will be written to `target_ptr`. /// If nothing is found for the connection, `[ByteSourceId::INVALID]` (zero) is written to `target_ptr`. @@ -1661,6 +1625,51 @@ impl WasmInstanceEnv { }) } + /// Read an environment value as a nullable BytesSource. Zero means missing; + /// a present empty string always receives a nonzero, consumable source. + pub fn env_get( + caller: Caller<'_, Self>, + key: WasmPtr, + key_len: u32, + target_ptr: WasmPtr, + ) -> RtResult { + Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { + if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { + return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); + } + let (mem, env) = Self::mem_env(caller); + let key = mem.deref_str(key, key_len)?; + match env.instance_env.env_get(key)? { + None => Ok(0), + Some(value) => { + // These buffers live on the host heap until consumed or the + // invocation ends. Bound retained reads from hand-written Wasm. + if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { + return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); + } + Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) + } + } + }) + } + + /// Returns host-verified invocation flags. Bit 0 is internal authority. + /// This does not read tables and is available outside transactions. + pub fn get_call_auth_flags(caller: Caller<'_, Self>) -> u32 { + caller.data().instance_env.get_call_auth_flags() + } + + pub(crate) fn set_hosted_auth( + &mut self, + auth: Option>, + ) { + self.instance_env.set_hosted_auth(auth); + } + + pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { + self.instance_env.set_call_auth_flags(flags); + } + /// Writes the identity of the module into `out = out_ptr[..32]`. /// /// # Traps diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 57ed3416ffa..54e4c5f1717 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -648,6 +648,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { .data_mut() .start_funcall(reducer_name, args_bytes, op.timestamp, op.call_type()); store.data_mut().set_call_auth_flags(op.call_auth_flags); + store.data_mut().set_hosted_auth(op.hosted_auth.clone()); let call_result = call_sync_typed_func( &self.call_reducer, @@ -772,6 +773,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { .data_mut() .start_funcall(op.name().clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); store.data_mut().set_call_auth_flags(op.call_auth_flags); + store.data_mut().set_hosted_auth(op.hosted_auth.clone()); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index e927921c886..29731bb6114 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1,3 +1,4 @@ +use crate::auth::invocation::{check_hosted_admission, SqlCallAuth}; use std::sync::Arc; use std::time::Duration; @@ -10,7 +11,7 @@ use crate::host::module_host::{ WasmInstance, }; use crate::host::{ArgsTuple, ModuleHost}; -use crate::subscription::module_subscription_actor::{commit_and_broadcast_event, ModuleSubscriptions}; +use crate::subscription::module_subscription_actor::ModuleSubscriptions; use crate::subscription::module_subscription_manager::TransactionOffset; use crate::subscription::tx::DeltaTx; use anyhow::anyhow; @@ -18,6 +19,7 @@ use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_engine::relational_db::RelationalDB; use spacetimedb_expr::statement::Statement; +#[cfg(test)] use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_lib::Timestamp; @@ -52,11 +54,15 @@ pub struct SqlResult { pub async fn run( db: Arc, sql_text: String, - auth: AuthCtx, + auth: impl Into, subs: Option, module: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result { + let auth = auth.into(); + if auth.hosted.is_some() && module.is_none() { + return Err(anyhow!("hosted SQL requires a module with hosted authentication capability").into()); + } match module { Some(module) => module.call_sql(db, sql_text, auth, subs, head).await, None => run_inner::(None, db, sql_text, auth, subs, head).map(|x| x.0), @@ -70,7 +76,7 @@ pub(crate) fn run_with_instance( instance: &mut RefInstance, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result<(SqlResult, bool), DBError> { @@ -81,13 +87,14 @@ fn run_inner( instance: Option<&mut RefInstance>, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result<(SqlResult, bool), DBError> { // We parse the sql statement in a mutable transaction. // If it turns out to be a query, we downgrade the tx. let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { + check_hosted_admission(tx, &db, auth.hosted.as_deref())?; let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; // Check mutation authority while the automatic rollback guard owns // the transaction, including rejected administrative statements. @@ -104,6 +111,13 @@ fn run_inner( "Use SET env.KEY or DELETE env.KEY to modify database environment variables" )); } + if let Statement::DML(dml) = &stmt + && spacetimedb_datastore::system_tables::is_host_managed_deployment_table(dml.table_id()) + { + return Err(anyhow!( + "Deployment and container authorization metadata may only be changed by the host" + )); + } Ok(stmt) })?; @@ -117,7 +131,7 @@ fn run_inner( None => (tx, false), }; - let (tx_data, tx_metrics_mut, tx) = db.commit_tx_downgrade(tx, Workload::Sql); + let (tx_data, tx_metrics_mut, tx) = db.commit_tx_downgrade(tx, Workload::Sql)?; let (tx_offset_send, tx_offset) = oneshot::channel(); // Release the tx on drop, so that we record metrics @@ -232,7 +246,10 @@ fn run_inner( request_id: None, timer: None, }; - let res = commit_and_broadcast_event(&subs.unwrap(), None, event, tx); + let res = subs + .unwrap() + .commit_and_broadcast_event(None, event, tx)? + .map_err(|_| anyhow!("SQL transaction write conflict"))?; Ok(( SqlResult { tx_offset: res.tx_offset, @@ -480,6 +497,25 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn owner_sql_cannot_rewrite_container_authority_or_deployment() -> ResultTest<()> { + let db = TestDB::in_memory()?; + for table in [ + "st_deployment", + "st_publish_fence", + "st_deployment_operation", + "st_container_fence", + "st_connection_auth", + ] { + // This uses the owner test context. Permission to mutate application + // tables does not grant permission to replace host operational state. + let error = run_for_testing(&db, &format!("DELETE FROM {table}")).unwrap_err(); + assert!(error.to_string().contains("may only be changed by the host"), "{error}"); + assert!(run_for_testing(&db, &format!("SELECT * FROM {table}"))?.is_empty()); + } + Ok(()) + } + #[test] fn test_limit() -> ResultTest<()> { let (db, _) = create_data(5)?; diff --git a/crates/core/src/subscription/module_subscription_actor.rs b/crates/core/src/subscription/module_subscription_actor.rs index c88da72e8d2..11a2913f0ce 100644 --- a/crates/core/src/subscription/module_subscription_actor.rs +++ b/crates/core/src/subscription/module_subscription_actor.rs @@ -7,6 +7,7 @@ use super::module_subscription_manager::{ use super::query::{compile_query_with_hashes, CompiledQuery}; use super::tx::DeltaTx; use super::TableUpdateType; +use crate::auth::invocation::check_hosted_admission; use crate::client::messages::{ ProcedureResultMessage, SerializableMessage, SubscriptionData, SubscriptionError, SubscriptionMessage, SubscriptionResult, SubscriptionRows, SubscriptionUpdateMessage, TransactionUpdateMessage, @@ -31,6 +32,7 @@ use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet} use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; use spacetimedb_datastore::locking_tx_datastore::datastore::TxMetrics; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, TxId, ViewCallInfo}; use spacetimedb_datastore::traits::{IsolationLevel, TxData}; use spacetimedb_durability::TxOffset; @@ -66,6 +68,7 @@ pub struct ModuleSubscriptions { stats: Arc, metrics: Arc, module_def_version: Arc, + hosted_connections: Arc>>>, } #[derive(Debug, Clone)] @@ -341,6 +344,7 @@ impl ModuleSubscriptions { Self { relational_db, subscriptions, + hosted_connections: Arc::new(RwLock::new(Vec::new())), broadcast_queue, stats, metrics, @@ -351,6 +355,46 @@ impl ModuleSubscriptions { } } + /// Register every hosted socket, even if it has no subscriptions. Taking the + /// database transaction before the registry lock serializes with fencing. + pub(crate) fn register_hosted_connection(&self, sender: &Arc) -> anyhow::Result<()> { + self.relational_db.with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, &self.relational_db, sender.auth.hosted.as_ref())?; + let mut connections = self.hosted_connections.write(); + connections.retain(|connection| connection.strong_count() != 0); + connections.push(Arc::downgrade(sender)); + Ok(()) + }) + } + + pub(crate) fn unregister_hosted_connection(&self, sender: &std::sync::Weak) { + self.hosted_connections + .write() + .retain(|connection| connection.strong_count() != 0 && !connection.ptr_eq(sender)); + } + + #[cfg(test)] + pub(crate) fn hosted_connection_count(&self) -> usize { + self.hosted_connections.read().len() + } + + /// Call with the transaction containing the new target fence. After its + /// durable commit, await every returned actor's completion before acking the + /// fence barrier. This also cancels socket batches already dequeued for I/O. + pub fn cancel_invalid_hosted_connections(&self, tx: &S) -> Vec { + let mut cancelled = Vec::new(); + self.hosted_connections.write().retain(|connection| { + let Some(connection) = connection.upgrade() else { + return false; + }; + if check_hosted_admission(tx, &self.relational_db, connection.auth.hosted.as_ref()).is_err() { + cancelled.push(connection.cancel_hosted_connection()); + } + true + }); + cancelled + } + pub fn set_module_def_version(&self, version: RawModuleDefVersion) { self.module_def_version .store(Self::encode_module_def_version(version), Ordering::Release); @@ -370,7 +414,8 @@ impl ModuleSubscriptions { fn decode_module_def_version(version: u8) -> RawModuleDefVersion { match version { 1 => RawModuleDefVersion::V10, - _ => RawModuleDefVersion::V9OrEarlier, + 0 => RawModuleDefVersion::V9OrEarlier, + _ => unreachable!("invalid stored module definition version"), } } @@ -685,6 +730,7 @@ impl ModuleSubscriptions { let hash_with_param = QueryHash::from_string(&sql, auth.caller(), true); let (mut_tx, _) = self.begin_mut_tx(Workload::Subscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let existing_query = { let guard = self.subscriptions.read(); @@ -794,6 +840,8 @@ impl ModuleSubscriptions { ) }; + let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let queries = { let mut subscriptions = self.subscriptions.write(); return_on_err!( @@ -812,7 +860,8 @@ impl ModuleSubscriptions { return Ok(None); }; - let (mut tx, tx_offset) = self.unsubscribe_views(query, auth.caller())?; + let mut_tx = ScopeGuard::::into_inner(mut_tx); + let (mut tx, tx_offset) = self.unsubscribe_views_and_downgrade_tx(mut_tx, query, auth.caller())?; let (table_rows, metrics) = return_on_err_with_sql!( self.evaluate_initial_subscription(sender.clone(), query.clone(), &tx, TableUpdateType::Unsubscribe), @@ -875,6 +924,7 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -990,6 +1040,7 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -1063,7 +1114,7 @@ impl ModuleSubscriptions { /// If either one is currently tracked, we can avoid recompilation. fn compile_queries( &self, - sender: Identity, + sender: &ClientConnectionSender, auth: AuthCtx, queries: &[Box], num_queries: usize, @@ -1079,13 +1130,14 @@ impl ModuleSubscriptions { subscribe_to_all_tables = true; continue; } - let hash = QueryHash::from_string(sql, sender, false); - let hash_with_param = QueryHash::from_string(sql, sender, true); + let hash = QueryHash::from_string(sql, sender.id.identity, false); + let hash_with_param = QueryHash::from_string(sql, sender.id.identity, true); query_hashes.push((sql, hash, hash_with_param)); } // We always get the db lock before the subscription lock to avoid deadlocks. let (mut_tx, _tx_offset) = self.begin_mut_tx(Workload::Subscribe); + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let compile_timer = metrics.compilation_time.start_timer(); @@ -1355,13 +1407,7 @@ impl ModuleSubscriptions { mut_tx, compile_timer: _compile_timer, } = return_on_err!( - self.compile_queries( - sender.id.identity, - auth, - &request.query_strings, - num_queries, - subscription_metrics - ), + self.compile_queries(&sender, auth, &request.query_strings, num_queries, subscription_metrics), send_err_msg, (None, false) ); @@ -1463,13 +1509,7 @@ impl ModuleSubscriptions { mut_tx, compile_timer, } = return_on_err!( - self.compile_queries( - sender.id.identity, - auth, - &request.query_strings, - num_queries, - subscription_metrics - ), + self.compile_queries(&sender, auth, &request.query_strings, num_queries, subscription_metrics), send_err_msg, (None, false) ); @@ -1627,7 +1667,7 @@ impl ModuleSubscriptions { mut_tx, compile_timer, } = self.compile_queries( - sender.id.identity, + &sender, auth, &subscription.query_strings, num_queries, @@ -1757,7 +1797,7 @@ impl ModuleSubscriptions { // We'll later ensure tx is released/cleaned up once out of scope. let (read_tx, tx_data, tx_metrics_mut) = match &mut event.status { EventStatus::Committed(db_update) => { - let (tx_data, tx_metrics, read_tx) = stdb.commit_tx_downgrade(tx, Workload::Update); + let (tx_data, tx_metrics, read_tx) = stdb.commit_tx_downgrade(tx, Workload::Update)?; *db_update = DatabaseUpdate::from_writes(&tx_data); (read_tx, tx_data, tx_metrics) } @@ -1865,7 +1905,7 @@ impl ModuleSubscriptions { sender: Identity, ) -> Result<(TxGuard, TransactionOffset), DBError> { Self::_unsubscribe_views(&mut tx, view_collector, sender)?; - let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Unsubscribe); + let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Unsubscribe)?; let opts = GuardTxOptions::from_mut(tx_data, tx_metrics_mut); Ok(self.guard_tx(tx, opts)) } @@ -1906,7 +1946,7 @@ impl ModuleSubscriptions { (tx, trapped) = ModuleHost::materialize_views(tx, instance, view_collector, sender, Workload::Subscribe)?; }; - let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Subscribe); + let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Subscribe)?; let opts = GuardTxOptions::from_mut(tx_data, tx_metrics_mut); let (a, b) = self.guard_tx(tx, opts); diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index d21706074e1..6509fba550c 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -1775,7 +1775,13 @@ pub struct BroadcastQueue(SenderWithGauge); #[derive(thiserror::Error, Debug)] #[error(transparent)] -pub struct BroadcastError(#[from] mpsc::error::SendError); +pub struct BroadcastError(Box>); + +impl From> for BroadcastError { + fn from(error: mpsc::error::SendError) -> Self { + Self(Box::new(error)) + } +} impl BroadcastQueue { fn send(&self, message: SendWorkerMessage) -> Result<(), BroadcastError> { diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 9169a96b45c..b00d36a448e 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -359,6 +359,9 @@ impl CommittedState { self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); let env = crate::system_tables::st_env_schema(); self.create_table(env.table_id, env.into()); + for schema in crate::system_tables::deployment_system_schemas() { + self.create_table(schema.table_id, schema.into()); + } // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index cf5fd662c63..f4503706ea8 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1079,7 +1079,6 @@ pub(crate) mod tests { use super::*; use crate::error::IndexError; use crate::locking_tx_datastore::tx_state::PendingSchemaChange; - use crate::system_tables::ST_ENV_ID; use crate::system_tables::{ system_tables, StColumnRow, StConnectionCredentialsFields, StConstraintData, StConstraintFields, StConstraintRow, StEventTableFields, StIndexAlgorithm, StIndexFields, StIndexRow, StRowLevelSecurityFields, @@ -1093,6 +1092,10 @@ pub(crate) mod tests { ST_VIEW_ARG_NAME, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_NAME, ST_VIEW_ID, ST_VIEW_NAME, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_NAME, ST_VIEW_SUB_ID, ST_VIEW_SUB_NAME, }; + use crate::system_tables::{ + ST_CONNECTION_AUTH_ID, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, + ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, ST_PUBLISH_FENCE_ID, + }; use crate::traits::{IsolationLevel, MutTx}; use crate::Result; use core::{fmt, mem}; @@ -1560,6 +1563,12 @@ pub(crate) mod tests { TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_ENV_ID.into(), name: "st_env", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_DEPLOYMENT_ID.into(), name: "st_deployment", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_PUBLISH_FENCE_ID.into(), name: "st_publish_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_DEPLOYMENT_OPERATION_ID.into(), name: "st_deployment_operation", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONTAINER_FENCE_ID.into(), name: "st_container_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONNECTION_AUTH_ID.into(), name: "st_connection_auth", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONTAINER_ENVIRONMENT_ID.into(), name: "st_container_environment", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] @@ -1659,6 +1668,28 @@ pub(crate) mod tests { ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 2, name: "accessor_name", ty: AlgebraicType::String }, ColRow { table: ST_ENV_ID.into(), pos: 0, name: "key", ty: AlgebraicType::String }, ColRow { table: ST_ENV_ID.into(), pos: 1, name: "value", ty: AlgebraicType::String }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 0, name: "key", ty: AlgebraicType::U8 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 1, name: "revision", ty: AlgebraicType::U256 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 2, name: "last_operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 3, name: "payload", ty: AlgebraicType::bytes() }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 0, name: "key", ty: AlgebraicType::U8 }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 1, name: "publication_epoch", ty: AlgebraicType::U64 }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 2, name: "operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 0, name: "operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 1, name: "previous_revision", ty: AlgebraicType::option(AlgebraicType::U256) }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 2, name: "committed_revision", ty: AlgebraicType::U256 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 3, name: "commit_result", ty: AlgebraicType::bytes() }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 4, name: "expires_at", ty: AlgebraicType::I64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 0, name: "source_identity", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 1, name: "generation", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 2, name: "target_grant_revision", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 3, name: "target_set_hash", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 4, name: "allowed", ty: AlgebraicType::Bool }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 0, name: "connection_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 1, name: "sender_identity", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 2, name: "call_auth_flags", ty: AlgebraicType::U32 }, + ColRow { table: ST_CONTAINER_ENVIRONMENT_ID.into(), pos: 0, name: "generation", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_ENVIRONMENT_ID.into(), pos: 1, name: "payload", ty: AlgebraicType::bytes() }, ])); #[rustfmt::skip] assert_eq!(query.scan_st_indexes()?, map_array([ @@ -1692,6 +1723,12 @@ pub(crate) mod tests { IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, + IndexRow { id: 31, table: ST_DEPLOYMENT_ID.into(), col: col_list![0], name: "st_deployment_key_idx_btree", }, + IndexRow { id: 32, table: ST_PUBLISH_FENCE_ID.into(), col: col_list![0], name: "st_publish_fence_key_idx_btree", }, + IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col_list![0], name: "st_deployment_operation_operation_id_idx_btree", }, + IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col_list![0], name: "st_container_fence_source_identity_idx_btree", }, + IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col_list![0], name: "st_connection_auth_connection_id_idx_btree", }, + IndexRow { id: 36, table: ST_CONTAINER_ENVIRONMENT_ID.into(), col: col_list![0], name: "st_container_environment_generation_idx_btree", }, ])); let start = ST_RESERVED_SEQUENCE_RANGE as i128 + 1; #[rustfmt::skip] @@ -1738,6 +1775,12 @@ pub(crate) mod tests { ConstraintRow { constraint_id: 24, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 1], constraint_name: "st_column_accessor_table_name_col_name_key", }, ConstraintRow { constraint_id: 25, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 2], constraint_name: "st_column_accessor_table_name_accessor_name_key", }, ConstraintRow { constraint_id: 26, table_id: ST_ENV_ID.into(), unique_columns: col_list![0], constraint_name: "st_env_key_key", }, + ConstraintRow { constraint_id: 27, table_id: ST_DEPLOYMENT_ID.into(), unique_columns: col_list![0], constraint_name: "st_deployment_key_key", }, + ConstraintRow { constraint_id: 28, table_id: ST_PUBLISH_FENCE_ID.into(), unique_columns: col_list![0], constraint_name: "st_publish_fence_key_key", }, + ConstraintRow { constraint_id: 29, table_id: ST_DEPLOYMENT_OPERATION_ID.into(), unique_columns: col_list![0], constraint_name: "st_deployment_operation_operation_id_key", }, + ConstraintRow { constraint_id: 30, table_id: ST_CONTAINER_FENCE_ID.into(), unique_columns: col_list![0], constraint_name: "st_container_fence_source_identity_key", }, + ConstraintRow { constraint_id: 31, table_id: ST_CONNECTION_AUTH_ID.into(), unique_columns: col_list![0], constraint_name: "st_connection_auth_connection_id_key", }, + ConstraintRow { constraint_id: 32, table_id: ST_CONTAINER_ENVIRONMENT_ID.into(), unique_columns: col_list![0], constraint_name: "st_container_environment_generation_key", }, ])); // Verify we get back the tables correctly with the proper ids... @@ -2172,6 +2215,12 @@ pub(crate) mod tests { IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, IndexRow { id: 30, table: ST_ENV_ID.into(), col: col_list![0], name: "st_env_key_idx_btree", }, + IndexRow { id: 31, table: ST_DEPLOYMENT_ID.into(), col: col_list![0], name: "st_deployment_key_idx_btree", }, + IndexRow { id: 32, table: ST_PUBLISH_FENCE_ID.into(), col: col_list![0], name: "st_publish_fence_key_idx_btree", }, + IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col_list![0], name: "st_deployment_operation_operation_id_idx_btree", }, + IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col_list![0], name: "st_container_fence_source_identity_idx_btree", }, + IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col_list![0], name: "st_connection_auth_connection_id_idx_btree", }, + IndexRow { id: 36, table: ST_CONTAINER_ENVIRONMENT_ID.into(), col: col_list![0], name: "st_container_environment_generation_idx_btree", }, IndexRow { id: seq_start, table: FIRST_NON_SYSTEM_ID, col: col(0), name: "Foo_id_idx_btree", }, IndexRow { id: seq_start + 1, table: FIRST_NON_SYSTEM_ID, col: col(1), name: "Foo_name_idx_btree", }, IndexRow { id: seq_start + 2, table: FIRST_NON_SYSTEM_ID, col: col(2), name: "Foo_age_idx_btree", }, diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 042fa6cfc5f..56fc9a112ff 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -11,9 +11,10 @@ use super::{ use crate::{ error::ViewError, system_tables::{ - system_tables, ConnectionIdViaU128, StConnectionCredentialsFields, StConnectionCredentialsRow, - StViewColumnFields, StViewFields, StViewParamFields, StViewParamRow, StViewSubFields, - ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, ST_VIEW_ID, ST_VIEW_PARAM_ID, ST_VIEW_SUB_ID, + system_tables, ConnectionIdViaU128, StConnectionAuthFields, StConnectionCredentialsFields, + StConnectionCredentialsRow, StViewColumnFields, StViewFields, StViewParamFields, StViewParamRow, + StViewSubFields, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, ST_VIEW_ID, + ST_VIEW_PARAM_ID, ST_VIEW_SUB_ID, }, }; use crate::{ @@ -3271,7 +3272,13 @@ impl MutTxId { ); } } - self.delete_st_client_credentials(database_identity, connection_id) + self.delete_st_client_credentials(database_identity, connection_id)?; + self.delete_col_eq( + ST_CONNECTION_AUTH_ID, + StConnectionAuthFields::ConnectionId.col_id(), + &ConnectionIdViaU128::from(connection_id).into(), + )?; + Ok(()) } /// Look up a client row by identity and connection ID in the `st_clients` system table. diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index d39cd14923c..bc86001b9fa 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -175,7 +175,7 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result false, + ST_CONNECTION_CREDENTIALS_ID | ST_CONNECTION_AUTH_ID | ST_CONTAINER_ENVIRONMENT_ID => false, // We don't define any system views, so none of the view-related tables can be system meta-descriptors. ST_VIEW_ID | ST_VIEW_PARAM_ID | ST_VIEW_COLUMN_ID | ST_VIEW_SUB_ID | ST_VIEW_ARG_ID => false, ST_EVENT_TABLE_ID => { @@ -207,7 +207,9 @@ pub enum SystemTable { st_event_table = ST_EVENT_TABLE_ID.0 as _, } -pub fn system_tables() -> [TableSchema; 21] { +pub fn system_tables() -> [TableSchema; 27] { + let [deployment, publish_fence, deployment_operation, container_fence, connection_auth, container_environment] = + deployment_system_schemas(); [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -231,6 +233,12 @@ pub fn system_tables() -> [TableSchema; 21] { st_index_accessor_schema(), st_column_accessor_schema(), st_env_schema(), + deployment, + publish_fence, + deployment_operation, + container_fence, + connection_auth, + container_environment, ] } @@ -316,6 +324,8 @@ macro_rules! st_fields_enum { mod environment; pub use environment::*; +mod deployment; +pub use deployment::*; // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StTableFields { @@ -675,6 +685,7 @@ fn system_module_def() -> ModuleDef { .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); environment::register_table(&mut builder); + deployment::register_tables(&mut builder); let result = builder .finish() @@ -700,8 +711,9 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_EVENT_TABLE_NAME); validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); - environment::validate_table(&result); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); + environment::validate_table(&result); + deployment::validate_tables(&result); result } @@ -751,6 +763,7 @@ lazy_static::lazy_static! { m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); m.insert("st_env_key_key", ConstraintId(26)); + m.extend(deployment::CONSTRAINTS); m }; } @@ -790,6 +803,7 @@ lazy_static::lazy_static! { m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); m.insert("st_env_key_idx_btree", IndexId(30)); + m.extend(deployment::INDEXES); m }; } @@ -980,7 +994,7 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), ST_ENV_ID => Some(st_env_schema()), - _ => None, + table => deployment::system_schema(table), } } diff --git a/crates/datastore/src/system_tables/deployment.rs b/crates/datastore/src/system_tables/deployment.rs new file mode 100644 index 00000000000..4a793b6ac8a --- /dev/null +++ b/crates/datastore/src/system_tables/deployment.rs @@ -0,0 +1,253 @@ +//! Stable system schemas for container deployment state. +//! +//! The deployment payload and operation result have versioned binary encodings, +//! so adding a protocol version does not change the system table row layout. +//! Publication and target fences are operational metadata: application restore +//! must preserve/reconcile their current authority before admitting execution. + +use super::*; + +pub const ST_DEPLOYMENT_ID: TableId = TableId(22); +pub const ST_PUBLISH_FENCE_ID: TableId = TableId(23); +pub const ST_DEPLOYMENT_OPERATION_ID: TableId = TableId(24); +pub const ST_CONTAINER_FENCE_ID: TableId = TableId(25); +pub const ST_CONNECTION_AUTH_ID: TableId = TableId(26); +pub const ST_CONTAINER_ENVIRONMENT_ID: TableId = TableId(27); + +pub const ST_DEPLOYMENT_NAME: &str = "st_deployment"; +pub const ST_PUBLISH_FENCE_NAME: &str = "st_publish_fence"; +pub const ST_DEPLOYMENT_OPERATION_NAME: &str = "st_deployment_operation"; +pub const ST_CONTAINER_FENCE_NAME: &str = "st_container_fence"; +pub const ST_CONNECTION_AUTH_NAME: &str = "st_connection_auth"; +pub const ST_CONTAINER_ENVIRONMENT_NAME: &str = "st_container_environment"; + +st_fields_enum!(enum StDeploymentFields { + "key", Key = 0, + "revision", Revision = 1, + "last_operation_id", LastOperationId = 2, + "payload", Payload = 3, +}); +st_fields_enum!(enum StPublishFenceFields { + "key", Key = 0, + "publication_epoch", PublicationEpoch = 1, + "operation_id", OperationId = 2, +}); +st_fields_enum!(enum StDeploymentOperationFields { + "operation_id", OperationId = 0, + "previous_revision", PreviousRevision = 1, + "committed_revision", CommittedRevision = 2, + "commit_result", CommitResult = 3, + "expires_at", ExpiresAt = 4, +}); +st_fields_enum!(enum StContainerFenceFields { + "source_identity", SourceIdentity = 0, + "generation", Generation = 1, + "target_grant_revision", TargetGrantRevision = 2, + "target_set_hash", TargetSetHash = 3, + "allowed", Allowed = 4, +}); +st_fields_enum!(enum StConnectionAuthFields { + "connection_id", ConnectionId = 0, + "sender_identity", SenderIdentity = 1, + "call_auth_flags", CallAuthFlags = 2, +}); + +st_fields_enum!(enum StContainerEnvironmentFields { + "generation", Generation = 0, + "payload", Payload = 1, +}); + +/// Secret-bearing host state. Never expose its retained history through SQL or module syscalls. +#[derive(Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StContainerEnvironmentRow { + pub generation: u64, + pub payload: Box<[u8]>, +} + +impl std::fmt::Debug for StContainerEnvironmentRow { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StContainerEnvironmentRow") + .field("generation", &self.generation) + .field("payload", &"[redacted]") + .finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDeploymentRow { + pub key: u8, + pub revision: Hash, + pub last_operation_id: u128, + pub payload: Box<[u8]>, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StPublishFenceRow { + pub key: u8, + pub publication_epoch: u64, + pub operation_id: u128, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDeploymentOperationRow { + pub operation_id: u128, + pub previous_revision: Option, + pub committed_revision: Hash, + pub commit_result: Box<[u8]>, + pub expires_at: TimestampViaI64, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StContainerFenceRow { + pub source_identity: IdentityViaU256, + pub generation: u64, + pub target_grant_revision: u64, + pub target_set_hash: Hash, + pub allowed: bool, +} + +/// Captured host authentication for lifecycle cleanup, including crash recovery. +/// Only hosted connections need a row; absent rows retain ordinary flags zero. +/// JWT claims and sender equality never reconstruct these flags. +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StConnectionAuthRow { + pub connection_id: ConnectionIdViaU128, + pub sender_identity: IdentityViaU256, + pub call_auth_flags: u32, +} + +macro_rules! row_conversions { + ($($row:ty),+ $(,)?) => {$ ( + impl TryFrom> for $row { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } + } + impl From<$row> for ProductValue { + fn from(row: $row) -> Self { to_product_value(&row) } + } + )+}; +} + +row_conversions!( + StDeploymentRow, + StPublishFenceRow, + StDeploymentOperationRow, + StContainerFenceRow, + StConnectionAuthRow, + StContainerEnvironmentRow +); + +pub(super) fn register_tables(builder: &mut RawModuleDefV9Builder) { + fn register(builder: &mut RawModuleDefV9Builder, name: &'static str) { + let ty = builder.add_type::(); + builder + .build_table(name, *ty.as_ref().expect("system row must be a product")) + .with_type(TableType::System) + .with_access(v9::TableAccess::Private) + .with_primary_key(ColId(0)) + .with_unique_constraint(ColId(0)) + .with_index_no_accessor_name(btree(ColId(0))); + } + register::(builder, ST_DEPLOYMENT_NAME); + register::(builder, ST_PUBLISH_FENCE_NAME); + register::(builder, ST_DEPLOYMENT_OPERATION_NAME); + register::(builder, ST_CONTAINER_FENCE_NAME); + register::(builder, ST_CONNECTION_AUTH_NAME); + register::(builder, ST_CONTAINER_ENVIRONMENT_NAME); +} + +pub(super) fn validate_tables(def: &ModuleDef) { + validate_system_table::(def, ST_DEPLOYMENT_NAME); + validate_system_table::(def, ST_PUBLISH_FENCE_NAME); + validate_system_table::(def, ST_DEPLOYMENT_OPERATION_NAME); + validate_system_table::(def, ST_CONTAINER_FENCE_NAME); + validate_system_table::(def, ST_CONNECTION_AUTH_NAME); + validate_system_table::(def, ST_CONTAINER_ENVIRONMENT_NAME); +} + +pub(crate) fn deployment_system_schemas() -> [TableSchema; 6] { + [ + st_schema(ST_DEPLOYMENT_NAME, ST_DEPLOYMENT_ID), + st_schema(ST_PUBLISH_FENCE_NAME, ST_PUBLISH_FENCE_ID), + st_schema(ST_DEPLOYMENT_OPERATION_NAME, ST_DEPLOYMENT_OPERATION_ID), + st_schema(ST_CONTAINER_FENCE_NAME, ST_CONTAINER_FENCE_ID), + st_schema(ST_CONNECTION_AUTH_NAME, ST_CONNECTION_AUTH_ID), + st_schema(ST_CONTAINER_ENVIRONMENT_NAME, ST_CONTAINER_ENVIRONMENT_ID), + ] +} + +pub(super) fn system_schema(table: TableId) -> Option { + let name = match table { + ST_DEPLOYMENT_ID => ST_DEPLOYMENT_NAME, + ST_PUBLISH_FENCE_ID => ST_PUBLISH_FENCE_NAME, + ST_DEPLOYMENT_OPERATION_ID => ST_DEPLOYMENT_OPERATION_NAME, + ST_CONTAINER_FENCE_ID => ST_CONTAINER_FENCE_NAME, + ST_CONNECTION_AUTH_ID => ST_CONNECTION_AUTH_NAME, + ST_CONTAINER_ENVIRONMENT_ID => ST_CONTAINER_ENVIRONMENT_NAME, + _ => return None, + }; + Some(st_schema(name, table)) +} + +/// These tables are read through dedicated host operations by module code. +/// In particular, resolving a numeric table or index ID must not bypass this. +pub(super) fn is_module_restricted_deployment_table(table: TableId) -> bool { + matches!( + table, + ST_DEPLOYMENT_ID + | ST_PUBLISH_FENCE_ID + | ST_DEPLOYMENT_OPERATION_ID + | ST_CONTAINER_FENCE_ID + | ST_CONNECTION_AUTH_ID + | ST_CONTAINER_ENVIRONMENT_ID + ) +} + +/// Snapshot history is exclusively available through authenticated host operations. +pub fn is_host_only_read_table(table: TableId) -> bool { + table == ST_CONTAINER_ENVIRONMENT_ID +} + +pub(super) fn is_module_restricted_deployment_index(index: IndexId) -> bool { + INDEXES.iter().any(|(_, restricted)| *restricted == index) +} + +/// Environment management has its own validated SQL path. Deployment and +/// authorization metadata may only be changed by authenticated host operations. +pub fn is_host_managed_deployment_table(table: TableId) -> bool { + matches!( + table, + ST_DEPLOYMENT_ID + | ST_PUBLISH_FENCE_ID + | ST_DEPLOYMENT_OPERATION_ID + | ST_CONTAINER_FENCE_ID + | ST_CONNECTION_AUTH_ID + | ST_CONTAINER_ENVIRONMENT_ID + ) +} + +pub(super) const CONSTRAINTS: [(&str, ConstraintId); 6] = [ + ("st_deployment_key_key", ConstraintId(27)), + ("st_publish_fence_key_key", ConstraintId(28)), + ("st_deployment_operation_operation_id_key", ConstraintId(29)), + ("st_container_fence_source_identity_key", ConstraintId(30)), + ("st_connection_auth_connection_id_key", ConstraintId(31)), + ("st_container_environment_generation_key", ConstraintId(32)), +]; + +pub(super) const INDEXES: [(&str, IndexId); 6] = [ + ("st_deployment_key_idx_btree", IndexId(31)), + ("st_publish_fence_key_idx_btree", IndexId(32)), + ("st_deployment_operation_operation_id_idx_btree", IndexId(33)), + ("st_container_fence_source_identity_idx_btree", IndexId(34)), + ("st_connection_auth_connection_id_idx_btree", IndexId(35)), + ("st_container_environment_generation_idx_btree", IndexId(36)), +]; diff --git a/crates/datastore/src/system_tables/environment.rs b/crates/datastore/src/system_tables/environment.rs index 82395f4bd77..5369506f8a0 100644 --- a/crates/datastore/src/system_tables/environment.rs +++ b/crates/datastore/src/system_tables/environment.rs @@ -38,8 +38,8 @@ pub(crate) fn st_env_schema() -> TableSchema { } /// Module code must use env_get even when it guesses numeric identifiers. pub fn is_module_restricted_table(table: TableId) -> bool { - table == ST_ENV_ID + table == ST_ENV_ID || super::deployment::is_module_restricted_deployment_table(table) } pub fn is_module_restricted_index(index: IndexId) -> bool { - index == IndexId(30) + index == IndexId(30) || super::deployment::is_module_restricted_deployment_index(index) } diff --git a/crates/engine/src/durability.rs b/crates/engine/src/durability.rs index 9c938a6401b..844da464467 100644 --- a/crates/engine/src/durability.rs +++ b/crates/engine/src/durability.rs @@ -1,17 +1,14 @@ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; -use log::{error, info}; use spacetimedb_commitlog::payload::{ txdata::{Mutations, Ops}, Txdata, }; use spacetimedb_datastore::{execution_context::ReducerContext, traits::TxData}; use spacetimedb_durability::Transaction; -use spacetimedb_lib::Identity; use spacetimedb_sats::ProductValue; use crate::persistence::Durability; -use spacetimedb_runtime::Handle; pub(super) fn request_durability( durability: &Durability, @@ -32,21 +29,6 @@ pub(super) fn request_durability( })); } -pub(super) fn spawn_close(durability: Arc, runtime: &Handle, database_identity: Identity) { - let label = format!("[{database_identity}]"); - let runtime = runtime.clone(); - runtime.clone().spawn(async move { - match runtime.timeout(Duration::from_secs(10), durability.close()).await { - Err(_elapsed) => { - error!("{label} timeout waiting for durability shutdown"); - } - Ok(offset) => { - info!("{label} durability shut down at tx offset: {offset:?}"); - } - } - }); -} - fn prepare_tx_data_for_durability( tx_offset: u64, reducer_context: Option, diff --git a/crates/engine/src/error.rs b/crates/engine/src/error.rs index 7c1a3e11129..6fb5a9260bf 100644 --- a/crates/engine/src/error.rs +++ b/crates/engine/src/error.rs @@ -97,6 +97,8 @@ pub enum ViewError { #[derive(Error, Debug)] pub enum DBError { + #[error("database storage is closed")] + DatabaseClosed, #[error("LibError: {0}")] Lib(#[from] LibError), #[error("BufferError: {0}")] diff --git a/crates/engine/src/hosted_admission.rs b/crates/engine/src/hosted_admission.rs new file mode 100644 index 00000000000..59b6f30ef1c --- /dev/null +++ b/crates/engine/src/hosted_admission.rs @@ -0,0 +1,282 @@ +//! Transient receiving-host admission, separate from durable generation fences. +//! +//! A replayed fence can be older than current control authority. Every database +//! open starts closed, including cold maintenance opens and restored databases. +//! Trusted platform code must reconcile the complete incoming fence inventory +//! before completing a sweep. This state is never serialized or restored. + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +use anyhow::ensure; +use parking_lot::Mutex; + +#[derive(Default)] +struct State { + revision: u64, + fence_revision: u64, + sweeping: bool, +} + +#[derive(Default)] +struct Inner { + open: AtomicBool, + state: Mutex, +} + +/// Owned by one live RelationalDB. Opening another copy of the same database +/// Identity creates an independent, closed gate. +#[derive(Default)] +pub struct HostedAdmission(Arc); + +impl HostedAdmission { + pub fn is_open(&self) -> bool { + self.0.open.load(Ordering::Acquire) + } + + /// Start one bounded reconciliation. A competing sweep fails immediately. + /// Dropping its ticket keeps admission closed and permits a later retry. + pub fn begin(&self) -> anyhow::Result { + let mut state = self.0.state.lock(); + ensure!(!state.sweeping, "hosted admission reconciliation is already running"); + self.0.open.store(false, Ordering::Release); + let revision = state + .revision + .checked_add(1) + .filter(|value| *value != u64::MAX) + .ok_or_else(|| anyhow::anyhow!("hosted admission revision exhausted"))?; + state.revision = revision; + state.sweeping = true; + Ok(HostedAdmissionSweep { + inner: self.0.clone(), + revision, + fence_revision: state.fence_revision, + }) + } + + /// Read while holding the database transaction that observes fence rows. + pub fn fence_revision(&self) -> u64 { + self.0.state.lock().fence_revision + } + + /// Call before changing a durable fence, within its mutation transaction. + /// Even a later rollback conservatively invalidates an inventory scan. A + /// delayed installation after startup closes admission again, so it cannot + /// insert an unexamined allowed row behind a completed scan's cursor. + pub fn fences_changed(&self) -> anyhow::Result<(u64, u64)> { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + let previous = state.fence_revision; + let Some(next) = previous.checked_add(1).filter(|next| *next != u64::MAX) else { + state.fence_revision = u64::MAX; + state.revision = u64::MAX; + state.sweeping = false; + anyhow::bail!("hosted fence revision exhausted"); + }; + state.fence_revision = next; + Ok((previous, next)) + } + + /// Invalidate any outstanding sweep. This prevents new admission; it is + /// not a substitute for a transactional fence and positive actor drainage. + pub fn close(&self) { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + state.revision = state.revision.saturating_add(1); + state.sweeping = false; + } + + /// Permanently close a database whose storage writer is being shut down. + /// A retained handle cannot start a new sweep on that obsolete object. + pub fn seal(&self) { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + state.revision = u64::MAX; + state.sweeping = false; + } +} + +/// A ticket is tied to the exact database-open state that issued it. Completing +/// one cannot open a replacement database, or undo a later close operation. +#[must_use = "dropping the sweep leaves hosted admission closed"] +pub struct HostedAdmissionSweep { + inner: Arc, + revision: u64, + fence_revision: u64, +} + +impl HostedAdmissionSweep { + /// Retain this guard in the async owner when moving the ticket to physical + /// blocking work. Cancelling that owner invalidates this exact sweep, even + /// when dropping the blocking task's JoinHandle cannot stop its execution. + pub fn cancellation_guard(&self) -> HostedAdmissionCancellation { + HostedAdmissionCancellation { + inner: Some(self.inner.clone()), + revision: self.revision, + } + } + + /// Trusted platform code calls this only after durable fence installation, + /// actor completion and current control confirmation of the entire sweep. + pub fn complete(self) -> anyhow::Result<()> { + let revision = self.fence_revision; + self.complete_with_fence_revision(revision) + } + + /// Complete a scan which made its own confirmed fence mutations. The + /// caller holds the actual DB transaction and has verified every stored + /// fence against current authority at this exact physical revision. + pub fn complete_with_fence_revision(self, fence_revision: u64) -> anyhow::Result<()> { + let mut state = self.inner.state.lock(); + ensure!( + state.sweeping && state.revision == self.revision, + "hosted admission reconciliation was invalidated" + ); + ensure!( + fence_revision != u64::MAX && state.fence_revision == fence_revision, + "hosted fence inventory changed during reconciliation" + ); + state.sweeping = false; + self.inner.open.store(true, Ordering::Release); + Ok(()) + } +} + +/// Cancellation belongs to one sweep revision. A late guard cannot close a +/// newer retry, and completion and cancellation serialize on the same mutex. +#[must_use = "retain until physical completion and disarm only after success"] +pub struct HostedAdmissionCancellation { + inner: Option>, + revision: u64, +} + +impl HostedAdmissionCancellation { + pub fn disarm(mut self) { + self.inner = None; + } +} + +impl Drop for HostedAdmissionCancellation { + fn drop(&mut self) { + let Some(inner) = &self.inner else { return }; + let mut state = inner.state.lock(); + if state.revision == self.revision { + inner.open.store(false, Ordering::Release); + state.revision = state.revision.saturating_add(1); + state.sweeping = false; + } + } +} + +impl Drop for HostedAdmissionSweep { + fn drop(&mut self) { + let mut state = self.inner.state.lock(); + if state.revision == self.revision { + state.sweeping = false; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cancelled_sweep_retries_and_cannot_open_another_database() { + let original = HostedAdmission::default(); + let replacement = HostedAdmission::default(); + assert!(!original.is_open()); + let sweep = original.begin().unwrap(); + assert!(original.begin().is_err()); + drop(sweep); + assert!(!original.is_open()); + original.begin().unwrap().complete().unwrap(); + assert!(original.is_open()); + assert!(!replacement.is_open()); + } + + #[test] + fn stale_completion_and_drop_cannot_open_or_cancel_a_newer_sweep() { + let gate = HostedAdmission::default(); + let stale = gate.begin().unwrap(); + gate.close(); + let current = gate.begin().unwrap(); + assert!(stale.complete().is_err()); + assert!(!gate.is_open()); + assert!(gate.begin().is_err()); + current.complete().unwrap(); + assert!(gate.is_open()); + gate.close(); + assert!(!gate.is_open()); + } + + #[test] + fn revision_exhaustion_remains_closed() { + let gate = HostedAdmission::default(); + gate.0.state.lock().revision = u64::MAX - 2; + gate.begin().unwrap().complete().unwrap(); + assert!(gate.is_open()); + assert!(gate.begin().is_err()); + assert!(!gate.is_open()); + gate.close(); + assert!(gate.begin().is_err()); + assert!(!gate.is_open()); + } + + #[test] + fn owner_cancellation_invalidates_detached_completion_without_closing_a_newer_retry() { + let gate = HostedAdmission::default(); + let stale = gate.begin().unwrap(); + let cancellation = stale.cancellation_guard(); + drop(cancellation); + let current = gate.begin().unwrap(); + assert!(stale.complete().is_err()); + current.complete().unwrap(); + assert!(gate.is_open()); + + gate.close(); + let stale = gate.begin().unwrap(); + let cancellation = stale.cancellation_guard(); + gate.close(); + gate.begin().unwrap().complete().unwrap(); + drop(cancellation); + assert!(gate.is_open()); + assert!(stale.complete().is_err()); + } + + #[test] + fn only_an_acknowledged_completion_survives_owner_drop() { + let gate = HostedAdmission::default(); + let ticket = gate.begin().unwrap(); + let cancellation = ticket.cancellation_guard(); + ticket.complete().unwrap(); + drop(cancellation); + assert!(!gate.is_open()); + let ticket = gate.begin().unwrap(); + let cancellation = ticket.cancellation_guard(); + ticket.complete().unwrap(); + cancellation.disarm(); + assert!(gate.is_open()); + } + + #[test] + fn changed_fences_invalidate_scans_and_close_a_previously_open_gate() { + let gate = HostedAdmission::default(); + let ticket = gate.begin().unwrap(); + assert_eq!(gate.fences_changed().unwrap(), (0, 1)); + assert!(ticket.complete().is_err()); + let ticket = gate.begin().unwrap(); + assert_eq!(gate.fences_changed().unwrap(), (1, 2)); + ticket.complete_with_fence_revision(2).unwrap(); + assert!(gate.is_open()); + gate.fences_changed().unwrap(); + assert!(!gate.is_open()); + let ticket = gate.begin().unwrap(); + gate.0.state.lock().fence_revision = u64::MAX - 1; + assert!(gate.fences_changed().is_err()); + assert!(ticket.complete_with_fence_revision(u64::MAX).is_err()); + assert!(gate.begin().is_err()); + } +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index a38c80f3abb..07c4c4df7b5 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1,5 +1,6 @@ pub(crate) mod durability; pub mod error; +pub mod hosted_admission; pub mod metrics; pub mod persistence; pub mod relational_db; diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 19cda5b6fbe..5500903412a 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -1,4 +1,4 @@ -use crate::durability::{request_durability, spawn_close as spawn_durability_close}; +use crate::durability::request_durability; use crate::error::{DBError, RestoreSnapshotError}; use crate::metrics::ExecutionCounters; use crate::metrics::ENGINE_METRICS; @@ -7,6 +7,8 @@ use crate::util::asyncify; use crate::MetricsRecorderQueue; use anyhow::{anyhow, Context}; use enum_map::EnumMap; +use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; use spacetimedb_commitlog::repo::OnNewSegmentFn; use spacetimedb_commitlog::{self as commitlog, SizeOnDisk}; use spacetimedb_data_structures::map::HashSet; @@ -66,7 +68,9 @@ use spacetimedb_table::table_index::IndexKey; use std::borrow::Cow; use std::io; use std::ops::RangeBounds; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::sync::OnceLock; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; pub use super::snapshot::SnapshotWorker; @@ -102,8 +106,11 @@ pub type ConnectedClients = HashSet<(Identity, ConnectionId)>; pub struct RelationalDB { database_identity: Identity, owner_identity: Identity, + hosted_admission: super::hosted_admission::HostedAdmission, inner: Locking, + commits_closed: Arc, + shutdown: OnceLock>>>, durability: Option>, durability_runtime: Option, snapshot_worker: Option, @@ -142,9 +149,12 @@ impl std::fmt::Debug for RelationalDB { impl Drop for RelationalDB { fn drop(&mut self) { - // Attempt to flush the outstanding transactions. - if let (Some(durability), Some(runtime)) = (self.durability.take(), self.durability_runtime.take()) { - spawn_durability_close(durability, &runtime, self.database_identity); + self.hosted_admission.seal(); + if let Some(runtime) = &self.durability_runtime { + // Join the same owned close when a cancelled shutdown waiter was the + // last database owner. Never start a second provider close early. + let close = self.start_shutdown(runtime); + runtime.spawn(close); } } } @@ -164,12 +174,15 @@ impl RelationalDB { Self { inner, + commits_closed: Default::default(), + shutdown: Default::default(), durability, durability_runtime, snapshot_worker, database_identity, owner_identity, + hosted_admission: Default::default(), row_count_fn: default_row_count_fn(database_identity), disk_size_fn, @@ -194,6 +207,12 @@ impl RelationalDB { }); } + /// Transient container admission for this exact database open. The trusted + /// receiving host opens it only after reconciling current incoming fences. + pub fn hosted_admission(&self) -> &super::hosted_admission::HostedAdmission { + &self.hosted_admission + } + /// Open a database, which may or may not already exist. /// /// # Initialization @@ -364,23 +383,51 @@ impl RelationalDB { /// Shut down the database, without dropping it. /// - /// If the database is in-memory only, this does nothing. - /// Otherwise, it instructs the durability layer to shut down - /// and waits until all outstanding transactions are reported as durable. - /// - /// After calling this method, calling [Self::commit_tx_downgrade] or - /// [Self::commit_tx] will panic. - /// - /// Returns `None` if the database is in-memory only, - /// or nothing has been durably persisted yet. - /// - /// Returns the durable [TxOffset] in a `Some` otherwise. + /// Permanently closes hosted and commit admission, then joins the configured + /// durability writer. Transactions already holding the exclusive transaction + /// lock can finish before closure; later commits roll back with `DatabaseClosed`. + /// Caller cancellation does not cancel the physical close. pub async fn shutdown(&self) -> Option { - if let Some(durability) = &self.durability { - return durability.close().await; + self.hosted_admission.seal(); + match &self.durability_runtime { + Some(runtime) => self.start_shutdown(runtime).await, + None => self.start_shutdown(&Handle::tokio_current()).await, } + } - None + /// Close using the owning host runtime, including in-memory simulation databases. + pub async fn shutdown_with_runtime(&self, runtime: &Handle) -> Option { + self.hosted_admission.seal(); + self.start_shutdown(runtime).await + } + + fn start_shutdown(&self, runtime: &Handle) -> Shared>> { + self.shutdown + .get_or_init(|| { + let inner = self.inner.clone(); + let closed = self.commits_closed.clone(); + let durability = self.durability.clone(); + // A shutdown owns this task through both lock acquisition and actual + // writer completion. The blocking lock never occupies a runtime worker. + let close_runtime = runtime.clone(); + let task = runtime.spawn(async move { + close_runtime + .spawn_blocking(move || { + let tx = inner.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + closed.store(true, Ordering::Relaxed); + let _ = inner.rollback_mut_tx(tx); + }) + .await; + match durability { + Some(durability) => durability.close().await, + None => None, + } + }); + async move { task.await.expect("database writer close task panicked") } + .boxed() + .shared() + }) + .clone() } /// Create any system tables that are missing from the datastore. @@ -877,6 +924,13 @@ impl RelationalDB { tx: MutTx, ) -> Result, TxMetrics, Option)>, DBError> { log::trace!("COMMIT MUT TX"); + // `tx` holds the same exclusive lock used by start_shutdown. The flag + // cannot change between this check and the durability append. + if self.commits_closed.load(Ordering::Relaxed) { + let (_, metrics, reducer) = self.rollback_mut_tx(tx); + self.report_tx_metrics(reducer, None, Some(metrics), None); + return Err(DBError::DatabaseClosed); + } let reducer_context = tx.ctx.reducer_context().cloned(); // TODO: Never returns `None` -- should it? @@ -895,8 +949,15 @@ impl RelationalDB { } #[tracing::instrument(level = "trace", skip_all)] - pub fn commit_tx_downgrade(&self, tx: MutTx, workload: Workload) -> (Arc, TxMetrics, Tx) { + pub fn commit_tx_downgrade(&self, tx: MutTx, workload: Workload) -> Result<(Arc, TxMetrics, Tx), DBError> { log::trace!("COMMIT MUT TX"); + // `tx` holds the same exclusive lock used by start_shutdown. The flag + // cannot change between this check and the durability append. + if self.commits_closed.load(Ordering::Relaxed) { + let (_, metrics, reducer) = self.rollback_mut_tx(tx); + self.report_tx_metrics(reducer, None, Some(metrics), None); + return Err(DBError::DatabaseClosed); + } let reducer_context = tx.ctx.reducer_context().cloned(); let (tx_data, tx_metrics, tx, datastore_memory_bytes) = @@ -907,7 +968,7 @@ impl RelationalDB { self.maybe_do_snapshot(&tx_data); self.observe_datastore_memory(datastore_memory_bytes); - (tx_data, tx_metrics, tx) + Ok((tx_data, tx_metrics, tx)) } /// Get the [`DurableOffset`] of this database, or `None` if this is an @@ -1629,6 +1690,7 @@ impl RelationalDB { self.with_auto_commit(Workload::Internal, |mut_tx| { self.clear_all_views(mut_tx)?; self.clear_table(mut_tx, ST_CONNECTION_CREDENTIALS_ID)?; + self.clear_table(mut_tx, spacetimedb_datastore::system_tables::ST_CONNECTION_AUTH_ID)?; self.clear_table(mut_tx, ST_CLIENT_ID)?; self.clear_table(mut_tx, ST_VIEW_SUB_ID)?; Ok(()) @@ -2433,6 +2495,9 @@ pub mod tests_utils { } } +#[cfg(test)] +mod shutdown_tests; + #[cfg(test)] mod tests { #![allow(clippy::disallowed_macros)] diff --git a/crates/engine/src/relational_db/shutdown_tests.rs b/crates/engine/src/relational_db/shutdown_tests.rs new file mode 100644 index 00000000000..06c31b2fd83 --- /dev/null +++ b/crates/engine/src/relational_db/shutdown_tests.rs @@ -0,0 +1,65 @@ +use super::*; +use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; +use tests_utils::TestDB; + +#[test] +fn operation_drain_shutdown_serializes_transactions_and_survives_cancelled_waiter() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let fixture = TestDB::durable().unwrap(); + let db = fixture.db.clone(); + runtime.block_on(async { + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql); + insert(&mut tx, "BEFORE", "committed"); + let closing = tokio::spawn({ + let db = db.clone(); + async move { db.shutdown().await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while db.shutdown.get().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + closing.abort(); + assert!(closing.await.unwrap_err().is_cancelled()); + assert!(!db.commits_closed.load(Ordering::Relaxed)); + // This SQL/view transaction acquired the exclusive lock before shutdown. + let (_, _, read) = db.commit_tx_downgrade(tx, Workload::Sql).unwrap(); + let _ = db.release_tx(read); + assert!(db.shutdown().await.is_some()); + assert!(db.commits_closed.load(Ordering::Relaxed)); + for downgrade in [false, true] { + let mut late = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Subscribe); + insert(&mut late, "LATE", "must-roll-back"); + let error = if downgrade { + db.commit_tx_downgrade(late, Workload::Subscribe).err().unwrap() + } else { + db.commit_tx(late).err().unwrap() + }; + assert!(matches!(error, DBError::DatabaseClosed)); + } + let read = db.begin_tx(Workload::Internal); + assert_eq!(get(&read, "BEFORE").as_deref(), Some("committed")); + assert_eq!(get(&read, "LATE"), None); + let _ = db.release_tx(read); + }); +} + +fn insert(tx: &mut MutTx, key: &str, value: &str) { + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.into(), + value: value.into(), + }, + ) + .unwrap(); +} + +fn get(tx: &Tx, key: &str) -> Option { + tx.iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into())) + .unwrap() + .next() + .map(|row| StEnvRow::try_from(row).unwrap().value) +} diff --git a/crates/engine/src/snapshot.rs b/crates/engine/src/snapshot.rs index 2ea20b641a6..825bb0b5178 100644 --- a/crates/engine/src/snapshot.rs +++ b/crates/engine/src/snapshot.rs @@ -7,7 +7,11 @@ use std::{ }; use anyhow::Context as _; -use futures::{channel::mpsc, StreamExt as _}; +use futures::{ + channel::mpsc, + future::{BoxFuture, Shared}, + FutureExt as _, StreamExt as _, +}; use log::{info, warn}; use parking_lot::RwLock; use prometheus::{Histogram, IntGauge}; @@ -62,6 +66,7 @@ pub struct SnapshotWorker { snapshot_created: watch::Sender>, request_snapshot: mpsc::UnboundedSender, snapshot_repository: Arc, + completion: Shared>>>, } impl SnapshotWorker { @@ -89,12 +94,16 @@ impl SnapshotWorker { rt: rt.clone(), }), }; - rt.spawn(actor.run()); + let task = rt.spawn(actor.run()); + let completion = async move { task.await.map_err(|error| Arc::::from(error.to_string())) } + .boxed() + .shared(); Self { snapshot_created, request_snapshot: request_tx, snapshot_repository, + completion, } } @@ -103,6 +112,14 @@ impl SnapshotWorker { Self::new(snapshot_repository, compression, Handle::tokio_current()) } + /// Permanently close snapshot admission and join all previously accepted + /// snapshot and compression I/O. All clones observe the same completion. + /// Cancelling a waiter does not cancel the worker or its blocking I/O. + pub async fn shutdown(&self) -> Result<(), Arc> { + self.request_snapshot.close_channel(); + self.completion.clone().await + } + /// Finish the initialization of [Self] by passing a [SnapshotDatabaseState], /// or replace the current [SnapshotDatabaseState] with a new one. /// @@ -386,3 +403,63 @@ impl Compressor { } } } + +#[cfg(test)] +mod shutdown_tests { + use super::*; + use spacetimedb_datastore::{ + execution_context::Workload, + system_tables::{StEnvRow, ST_ENV_ID}, + traits::{IsolationLevel, MutTx as _}, + }; + use spacetimedb_paths::{server::SnapshotsPath, FromPathUnchecked}; + use spacetimedb_snapshot::SnapshotRepository; + use spacetimedb_table::page_pool::PagePool; + + #[tokio::test(flavor = "multi_thread")] + async fn terminal_shutdown_drains_accepted_snapshot_after_waiter_cancellation() { + let dir = tempfile::tempdir().unwrap(); + let repository = Arc::new( + SnapshotRepository::open(SnapshotsPath::from_path_unchecked(dir.path()), Identity::ONE, 1).unwrap(), + ); + let datastore = Locking::bootstrap(Identity::ONE, PagePool::new_for_test()).unwrap(); + let mut tx = datastore.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: "PERSISTED".into(), + value: "snapshot drain".into(), + }, + ) + .unwrap(); + datastore.commit_mut_tx(tx).unwrap(); + let worker = SnapshotWorker::new_tokio_current(repository.clone(), Compression::Enabled); + worker.set_state(datastore.committed_state.clone()); + let state = datastore.committed_state.write_arc(); + worker.request_snapshot(); + let mut first = tokio::spawn({ + let worker = worker.clone(); + async move { worker.shutdown().await } + }); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut first) + .await + .is_err()); + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + let mut second = tokio::spawn({ + let worker = worker.clone(); + async move { worker.shutdown().await } + }); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut second) + .await + .is_err()); + assert!(worker.request_snapshot.unbounded_send(Request::TakeSnapshot).is_err()); + assert_eq!(repository.latest_snapshot().unwrap(), None); + drop(state); + second.await.unwrap().unwrap(); + worker.shutdown().await.unwrap(); + assert_eq!(repository.latest_snapshot().unwrap(), Some(0)); + let snapshot = repository.read_snapshot(0, &PagePool::new_for_test()).unwrap(); + assert_eq!(snapshot.tx_offset, 0); + } +} diff --git a/crates/engine/src/sql/ast.rs b/crates/engine/src/sql/ast.rs index 892430ba1ea..4759c89e91d 100644 --- a/crates/engine/src/sql/ast.rs +++ b/crates/engine/src/sql/ast.rs @@ -34,6 +34,9 @@ impl SchemaView for SchemaViewer<'_, T> { } fn schema_for_table(&self, table_id: TableId) -> Option> { + if spacetimedb_datastore::system_tables::is_host_only_read_table(table_id) { + return None; + } self.tx .get_schema(table_id) .filter(|schema| self.auth.has_read_access(schema.table_access)) diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index b07774b3ee6..de3ab21159a 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -38,6 +38,7 @@ spacetimedb-memory-usage = { workspace = true, optional = true } spacetimedb-metrics = { workspace = true, optional = true } anyhow.workspace = true +thiserror.workspace = true derive_more.workspace = true hex.workspace = true serde = { workspace = true, optional = true } diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs new file mode 100644 index 00000000000..02d9c581824 --- /dev/null +++ b/crates/lib/src/container.rs @@ -0,0 +1,478 @@ +//! Shared, versioned container deployment data. +//! +//! A decoded declaration is untrusted. Publication must normalize and validate +//! the complete effective deployment before hashing or admitting it. Runtime +//! capabilities and database authorization are additional admission checks. + +use crate::{bsatn, hash_bytes, Hash, SpacetimeType}; +use std::{collections::BTreeSet, fmt, str::FromStr}; + +#[cfg(feature = "serde")] +pub mod endpoints; +#[cfg(feature = "serde")] +pub mod operations; + +/// Version of the normalized deployment encoding, independent of module ABI. +pub const CONTAINER_SPEC_VERSION: u32 = 1; +pub const MAX_ARGV_ENTRIES: usize = 256; +pub const MAX_ENV_KEYS: usize = 256; +pub const MAX_PORTS: usize = 16; +pub const MAX_EXEC_STRING_BYTES: usize = 32 * 1024; +/// Includes NUL terminators, 64-bit pointer arrays, and reserved startup space. +pub const MAX_EXEC_BYTES: usize = 128 * 1024; +pub const EXEC_RESERVED_BYTES: usize = 4096; +pub const DEFAULT_STOP_GRACE_MS: u32 = 30_000; +pub const MAX_STOP_GRACE_MS: u32 = 120_000; + +/// The digest of an OCI object. This is never a SpacetimeDB Keccak-256 program key. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, SpacetimeType)] +#[sats(crate = crate)] +pub enum OciDigest { + Sha256(Hash), +} + +impl OciDigest { + pub const fn sha256(bytes: [u8; 32]) -> Self { + Self::Sha256(Hash::from_byte_array(bytes)) + } + + pub fn as_bytes(&self) -> &[u8; 32] { + match self { + Self::Sha256(hash) => &hash.data, + } + } +} + +impl fmt::Display for OciDigest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Sha256(bytes) => write!(f, "sha256:{}", bytes.to_hex()), + } + } +} + +impl FromStr for OciDigest { + type Err = ContainerValidationError; + + fn from_str(value: &str) -> Result { + let hex = value + .strip_prefix("sha256:") + .ok_or_else(|| invalid("image_manifest", "only sha256 OCI digests are supported"))?; + if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(invalid( + "image_manifest", + "expected 64 lowercase hexadecimal digest characters", + )); + } + let mut bytes = [0; 32]; + hex::decode_to_slice(hex, &mut bytes).map_err(|_| invalid("image_manifest", "invalid SHA-256 digest"))?; + Ok(Self::sha256(bytes)) + } +} + +#[cfg(feature = "serde")] +impl serde::Serialize for OciDigest { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +#[cfg(feature = "serde")] +impl<'de> serde::Deserialize<'de> for OciDigest { + fn deserialize>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ImagePlatform { + pub os: String, + pub architecture: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum ContainerMode { + Service, + Job, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum RestartPolicy { + Never, + OnFailure, + Always, +} + +impl RestartPolicy { + /// Only process termination drives this policy. Readiness is independent. + pub fn restarts_after(self, exit_code: Option) -> bool { + match self { + Self::Never => false, + Self::OnFailure => exit_code != Some(0), + Self::Always => true, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerResources { + pub cpu_millicores: u64, + pub memory_bytes: u64, + pub scratch_bytes: u64, + /// Linux tasks, including threads and commands started through exec. + pub pids_max: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum PortProtocol { + Http, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum PortExposure { + Public, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ReadinessProbe { + Tcp(TcpProbe), + Http(HttpProbe), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct TcpProbe { + pub timeout_ms: u32, + pub interval_ms: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct HttpProbe { + pub path: String, + pub timeout_ms: u32, + pub interval_ms: u32, +} + +impl Default for ReadinessProbe { + fn default() -> Self { + Self::Tcp(TcpProbe { + timeout_ms: 1000, + interval_ms: 5000, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerPort { + pub name: String, + pub port: u16, + pub protocol: PortProtocol, + /// Required in the public input, with no implicit exposure default. + pub exposure: PortExposure, + #[cfg_attr(feature = "serde", serde(default))] + pub readiness_probe: ReadinessProbe, +} + +/// A placeholder declaration is never admitted until the mount protocol ships. +/// Keeping explicit declarations lets Stage 1 return an unsupported-feature error. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerMount { + pub database: String, + pub source: String, + pub target: String, + pub read_only: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerSpec { + pub image_manifest: OciDigest, + pub image_platform: ImagePlatform, + /// Effective complete argv after applying OCI Entrypoint/Cmd or override. + pub argv: Vec, + pub user: String, + pub working_directory: String, + pub mode: ContainerMode, + pub restart: RestartPolicy, + pub env_keys: Vec, + pub resources: ContainerResources, + pub ports: Vec, + pub mounts: Vec, + pub stop_grace_ms: u32, +} + +/// Explicit component removal is distinct from omission or an empty replacement. +#[derive(Clone, Debug, Default, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +#[expect( + clippy::large_enum_variant, + reason = "the normalized publish request owns its single container spec" +)] +pub enum ContainerAction { + #[default] + Keep, + Set(ContainerSpec), + Remove, +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid container {field}: {reason}")] +pub struct ContainerValidationError { + pub field: &'static str, + pub reason: &'static str, +} + +fn invalid(field: &'static str, reason: &'static str) -> ContainerValidationError { + ContainerValidationError { field, reason } +} + +/// Limits are operational configuration; normalized specs are checked again at +/// admission against the selected node's enforceable capacities and capabilities. +#[derive(Clone, Copy, Debug)] +pub struct ContainerSpecLimits { + pub resources: ContainerResources, +} + +impl Default for ContainerSpecLimits { + fn default() -> Self { + Self { + resources: ContainerResources { + cpu_millicores: 64_000, + memory_bytes: 128 * 1024 * 1024 * 1024, + scratch_bytes: 1024 * 1024 * 1024 * 1024, + pids_max: 4096, + }, + } + } +} + +impl ContainerSpec { + /// Sort semantically unordered declarations so equivalent specs hash alike. + /// Duplicates are rejected, never silently deduplicated. + pub fn normalize(mut self, limits: &ContainerSpecLimits) -> Result { + self.validate(limits)?; + self.env_keys.sort_unstable(); + self.ports.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + Ok(self) + } + + pub fn validate(&self, limits: &ContainerSpecLimits) -> Result<(), ContainerValidationError> { + if self.image_platform.os != "linux" || !matches!(self.image_platform.architecture.as_str(), "amd64" | "arm64") + { + return Err(invalid("image_platform", "expected linux/amd64 or linux/arm64")); + } + if !self.mounts.is_empty() { + return Err(invalid("mounts", "SpacetimeFS mounts are not supported by Stage 1")); + } + if self.mode == ContainerMode::Job && self.restart == RestartPolicy::Always { + return Err(invalid("restart", "jobs cannot use the always restart policy")); + } + if self.argv.is_empty() || self.argv.len() > MAX_ARGV_ENTRIES || self.argv[0].is_empty() { + return Err(invalid( + "argv", + "a nonempty command with at most 256 arguments is required", + )); + } + validate_exec_size(&self.argv, &[])?; + if self.user.len() > 255 || self.user.bytes().any(|b| b == 0 || b.is_ascii_control()) { + return Err(invalid( + "user", + "user must fit 255 bytes and contain no control characters", + )); + } + if !self.working_directory.starts_with('/') + || self.working_directory.len() > 4096 + || self.working_directory.contains('\0') + { + return Err(invalid( + "working_directory", + "expected an absolute Linux path of at most 4096 bytes", + )); + } + if self.stop_grace_ms > MAX_STOP_GRACE_MS { + return Err(invalid("stop_grace_ms", "stop grace exceeds the supported deadline")); + } + let r = self.resources; + let max = limits.resources; + if r.cpu_millicores == 0 + || r.cpu_millicores > max.cpu_millicores + || r.memory_bytes == 0 + || r.memory_bytes > max.memory_bytes + || r.scratch_bytes == 0 + || r.scratch_bytes > max.scratch_bytes + || r.pids_max == 0 + || r.pids_max > max.pids_max + { + return Err(invalid( + "resources", + "resource reservations must be positive and within server limits", + )); + } + if self.env_keys.len() > MAX_ENV_KEYS { + return Err(invalid("env_keys", "too many environment keys")); + } + let mut keys = BTreeSet::new(); + for key in &self.env_keys { + validate_env_key(key)?; + if !keys.insert(key) { + return Err(invalid("env_keys", "duplicate environment key")); + } + } + if self.ports.len() > MAX_PORTS { + return Err(invalid("ports", "too many declared ports")); + } + let (mut names, mut numbers) = (BTreeSet::new(), BTreeSet::new()); + for port in &self.ports { + let bytes = port.name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 32 + || !bytes[0].is_ascii_lowercase() + || !bytes + .iter() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || *b == b'-') + { + return Err(invalid("ports.name", "expected [a-z][a-z0-9-]{0,31}")); + } + if port.port == 0 || !names.insert(&port.name) || !numbers.insert(port.port) { + return Err(invalid("ports", "port names and nonzero port numbers must be unique")); + } + let (timeout, interval) = match &port.readiness_probe { + ReadinessProbe::Tcp(TcpProbe { + timeout_ms, + interval_ms, + }) => (*timeout_ms, *interval_ms), + ReadinessProbe::Http(HttpProbe { + path, + timeout_ms, + interval_ms, + }) => { + if !path.starts_with('/') + || path.starts_with("//") + || path.len() > 2048 + || path + .bytes() + .any(|b| b.is_ascii_control() || b == b' ' || b == b'\\' || b == b'#') + { + return Err(invalid( + "readiness_probe.path", + "expected a bounded origin-relative HTTP path", + )); + } + (*timeout_ms, *interval_ms) + } + }; + if timeout == 0 || timeout > 30_000 || interval == 0 || interval > 300_000 || timeout > interval { + return Err(invalid("readiness_probe", "invalid probe timeout or interval")); + } + } + Ok(()) + } + + /// Domain-separated, versioned BSATN encoding, after normalization. + /// This hashes the container spec only; the full deployment also includes + /// the module selection and uses its own revision domain. + pub fn canonical_hash(&self, limits: &ContainerSpecLimits) -> Result { + let normalized = self.clone().normalize(limits)?; + let encoded = bsatn::to_vec(&(CONTAINER_SPEC_VERSION, normalized)) + .expect("encoding an in-memory container specification cannot fail"); + let mut bytes = b"spacetimedb/container-spec\0".to_vec(); + bytes.extend(encoded); + Ok(hash_bytes(&bytes)) + } +} + +pub fn validate_env_key(key: &str) -> Result<(), ContainerValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > 256 + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(invalid("env_keys", "invalid POSIX environment variable name")); + } + if key.starts_with("SPACETIMEDB_") { + return Err(invalid( + "env_keys", + "SPACETIMEDB_ variables are reserved for the platform", + )); + } + Ok(()) +} + +/// Call after merging image environment, the database snapshot, and platform +/// variables. Error messages deliberately contain neither argv nor env values. +pub fn validate_exec_size(argv: &[String], env: &[String]) -> Result<(), ContainerValidationError> { + let count = argv + .len() + .checked_add(env.len()) + .and_then(|n| n.checked_add(2)) + .ok_or_else(|| invalid("exec", "too many arguments or environment entries"))?; + let mut total = count + .checked_mul(8) + .and_then(|n| n.checked_add(EXEC_RESERVED_BYTES)) + .ok_or_else(|| invalid("exec", "argument and environment size overflow"))?; + for value in argv.iter().chain(env) { + if value.contains('\0') || value.len() >= MAX_EXEC_STRING_BYTES { + return Err(invalid( + "exec", + "argument or environment entry contains NUL or exceeds the per-entry limit", + )); + } + total = total + .checked_add(value.len() + 1) + .ok_or_else(|| invalid("exec", "argument and environment size overflow"))?; + } + if total > MAX_EXEC_BYTES { + return Err(invalid( + "exec", + "combined argument and environment size exceeds the startup limit", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/container/endpoints.rs b/crates/lib/src/container/endpoints.rs new file mode 100644 index 00000000000..d5ab7964b11 --- /dev/null +++ b/crates/lib/src/container/endpoints.rs @@ -0,0 +1,57 @@ +//! Public address discovery. These addresses carry no administrative or runtime +//! authority and do not promise that an application is currently ready. + +use super::PortProtocol; +use crate::Identity; +use serde::{Deserialize, Serialize}; + +pub const ENDPOINTS_PENDING: &str = "endpoints_pending"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerEndpoints { + pub database_identity: Identity, + pub endpoints: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerEndpoint { + pub name: String, + pub protocol: PortProtocol, + pub url: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovery_wire_contains_only_explicit_public_addresses() { + let response = ContainerEndpoints { + database_identity: Identity::ZERO, + endpoints: vec![ContainerEndpoint { + name: "http".into(), + protocol: PortProtocol::Http, + url: "https://aaaqeayeaudaocajbifqydiob4.container.example.net/".into(), + }], + }; + let encoded = serde_json::to_value(&response).unwrap(); + assert_eq!(encoded.as_object().unwrap().len(), 2); + assert_eq!( + encoded["endpoints"][0], + serde_json::json!({ + "name": "http", + "protocol": "http", + "url": "https://aaaqeayeaudaocajbifqydiob4.container.example.net/" + }) + ); + assert_eq!( + serde_json::from_value::(encoded.clone()).unwrap(), + response + ); + let mut changed = encoded; + changed["endpoints"][0]["upstream"] = serde_json::json!("10.0.0.1:8080"); + assert!(serde_json::from_value::(changed).is_err()); + } +} diff --git a/crates/lib/src/container/operations.rs b/crates/lib/src/container/operations.rs new file mode 100644 index 00000000000..f6b499f1db8 --- /dev/null +++ b/crates/lib/src/container/operations.rs @@ -0,0 +1,199 @@ +//! Control-plane container inspection and idempotent lifecycle messages. +use crate::{container::endpoints::ContainerEndpoint, deployment::uuid_json, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContainerAction { + Start, + Stop, + Restart, +} +impl ContainerAction { + pub const fn path(self) -> &'static str { + match self { + Self::Start => "start", + Self::Stop => "stop", + Self::Restart => "restart", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerOperationRequest { + #[serde(with = "uuid_json")] + pub request_id: Uuid, +} + +/// Confirms admission only. It does not claim physical stop or runtime readiness. +/// An exact retry returns the generation originally recorded for this request. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerOperationReceipt { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub request_id: Uuid, + pub action: ContainerAction, + #[serde(with = "decimal_u64")] + pub generation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DesiredState { + Stopped, + Running, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ObservedState { + Pending, + Starting, + Running, + Ready, + Draining, + Stopped, + Completed, + Failed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Condition { + None, + Fencing, + NoLeader, + NodeUnavailable, + Capacity, + BalanceUnavailable, + BalanceExhausted, + TargetUnavailable, + PullFailed, + LaunchFailed, + ReadinessFailed, + ExitFailure, + OutOfMemory, + NodePressure, + LeaseExpired, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CurrentInstance { + #[serde(with = "decimal_u64")] + pub generation: u64, + pub state: ObservedState, + pub observed_revision: Option, + /// Application snapshot identity only. No values or hashes of values. + pub applied_env_generation: Option, + pub exit_code: Option, + pub oom_killed: bool, + pub condition: Condition, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperationalState { + pub desired_revision: Hash, + pub desired_state: DesiredState, + #[serde(with = "decimal_u64")] + pub generation: u64, + pub condition: Condition, + pub restart_pending: bool, + pub restart_attempt: u32, + /// Unix milliseconds, represented exactly for browser clients. + #[serde(with = "decimal_u64")] + pub restart_not_before_ms: u64, + /// Reports only this operational generation. A historical attempt is never + /// presented as the currently authorized replacement. + pub current_instance: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum EndpointStatus { + Available { endpoints: Vec }, + Pending, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerStatus { + pub database_identity: Identity, + pub published: bool, + pub operational: Option, + /// Address discovery is independent of readiness and remains available + /// while stopped. Pending never means an empty declaration. + pub endpoints: EndpointStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContainerErrorCode { + InvalidRequest, + AccessDenied, + NotFound, + Conflict, + Unavailable, + OutcomeUnknown, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerApiError { + pub error: ContainerErrorCode, +} + +mod decimal_u64 { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(number: &u64, serializer: S) -> Result { + serializer.collect_str(number) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + let value: u64 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal u64")); + } + Ok(value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn receipt_preserves_full_generation_and_rejects_lossy_or_extra_fields() { + let receipt = ContainerOperationReceipt { + database_identity: Identity::ONE, + request_id: Uuid::from_u128(0x01950000000070008000000000000001), + action: ContainerAction::Restart, + generation: u64::MAX, + }; + let mut value = serde_json::to_value(&receipt).unwrap(); + assert_eq!(value["generation"], u64::MAX.to_string()); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + receipt + ); + for generation in [ + serde_json::json!(1), + serde_json::json!("01"), + serde_json::json!("18446744073709551616"), + ] { + value["generation"] = generation; + assert!(serde_json::from_value::(value.clone()).is_err()); + } + value = serde_json::to_value(receipt).unwrap(); + value["credential"] = serde_json::json!("unexpected"); + assert!(serde_json::from_value::(value).is_err()); + } + #[test] + fn pending_discovery_is_distinct_from_empty() { + assert_ne!( + serde_json::to_value(EndpointStatus::Pending).unwrap(), + serde_json::to_value(EndpointStatus::Available { endpoints: vec![] }).unwrap() + ); + } +} diff --git a/crates/lib/src/container/tests.rs b/crates/lib/src/container/tests.rs new file mode 100644 index 00000000000..dd7c208f43c --- /dev/null +++ b/crates/lib/src/container/tests.rs @@ -0,0 +1,246 @@ +use super::*; + +fn spec() -> ContainerSpec { + ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec!["API_KEY".into(), "DATABASE_URL".into()], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![ContainerPort { + name: "http".into(), + port: 8080, + protocol: PortProtocol::Http, + exposure: PortExposure::Public, + readiness_probe: ReadinessProbe::default(), + }], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } +} + +#[test] +fn oci_digest_is_strict_and_cannot_be_used_for_path_traversal() { + let digest = format!("sha256:{}", "ab".repeat(32)); + assert_eq!(digest.parse::().unwrap().to_string(), digest); + for value in [ + "sha256:../../host", + "sha256:abc", + "sha512:abc", + "blake3:abc", + "SHA256:abc", + "sha256:", + ] { + assert!(value.parse::().is_err(), "{value}"); + } + assert!(format!("sha256:{}", "AB".repeat(32)).parse::().is_err()); + assert!(format!("sha256:{}\n", "ab".repeat(32)).parse::().is_err()); +} + +#[test] +fn canonical_hash_ignores_declaration_order_but_preserves_launch_semantics() { + let limits = ContainerSpecLimits::default(); + let mut original = spec(); + original.ports.push(ContainerPort { + name: "admin".into(), + port: 8081, + ..original.ports[0].clone() + }); + let hash = original.canonical_hash(&limits).unwrap(); + let mut reordered = original.clone(); + reordered.env_keys.reverse(); + reordered.ports.reverse(); + assert_eq!(reordered.canonical_hash(&limits).unwrap(), hash); + let mut changed = reordered; + changed.argv.push("--read-only".into()); + assert_ne!(changed.canonical_hash(&limits).unwrap(), hash); + assert_ne!( + ContainerSpec { + resources: ContainerResources { + pids_max: 63, + ..original.resources + }, + ..original.clone() + } + .canonical_hash(&limits) + .unwrap(), + hash + ); + let bytes = bsatn::to_vec(&original).unwrap(); + assert_eq!(bsatn::from_slice::(&bytes).unwrap(), original); +} + +#[test] +fn duplicate_declarations_are_not_silently_deduplicated() { + let limits = ContainerSpecLimits::default(); + let mut value = spec(); + value.env_keys.push(value.env_keys[0].clone()); + assert_eq!(value.normalize(&limits).unwrap_err().field, "env_keys"); + let mut value = spec(); + value.ports.push(ContainerPort { + name: "another".into(), + ..value.ports[0].clone() + }); + assert_eq!(value.normalize(&limits).unwrap_err().field, "ports"); + let mut value = spec(); + value.ports.push(ContainerPort { + port: 9090, + ..value.ports[0].clone() + }); + assert_eq!(value.normalize(&limits).unwrap_err().field, "ports"); +} + +#[test] +fn readiness_cannot_change_authority_or_inject_a_request() { + for path in [ + "https://internal/", + "//internal/", + "/\\internal/", + "/health\r\nHost: internal", + "/health#fragment", + ] { + let mut value = spec(); + value.ports[0].readiness_probe = ReadinessProbe::Http(HttpProbe { + path: path.into(), + timeout_ms: 1000, + interval_ms: 5000, + }); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "readiness_probe.path" + ); + } + let mut value = spec(); + value.ports[0].readiness_probe = ReadinessProbe::Http(HttpProbe { + path: "/health?full=1".into(), + timeout_ms: 1000, + interval_ms: 5000, + }); + value.validate(&ContainerSpecLimits::default()).unwrap(); +} + +#[test] +fn stage_one_rejects_mounts_and_unsupported_platforms() { + let mut value = spec(); + value.mounts.push(ContainerMount { + database: "self".into(), + source: "/".into(), + target: "/spacetime".into(), + read_only: false, + }); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "mounts" + ); + let mut value = spec(); + value.image_platform.os = "windows".into(); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "image_platform" + ); +} + +#[test] +fn startup_size_counts_environment_and_pointers_without_leaking_values() { + let secret = "THIS_VALUE_MUST_NOT_APPEAR_IN_ERRORS"; + let env = vec![format!("KEY={secret}\0")]; + let error = validate_exec_size(&spec().argv, &env).unwrap_err().to_string(); + assert!(!error.contains(secret)); + assert!(!error.contains("KEY=")); + let bounded_strings = vec!["x".repeat(MAX_EXEC_STRING_BYTES - 1); 5]; + assert!(validate_exec_size(&spec().argv, &bounded_strings).is_err()); + let many_empty_entries = vec![String::new(); MAX_EXEC_BYTES / 8]; + assert!(validate_exec_size(&[], &many_empty_entries).is_err()); + assert!(validate_exec_size(&spec().argv, &["KEY=value".into()]).is_ok()); +} + +#[test] +fn tenant_environment_cannot_override_platform_discovery_or_credentials() { + for key in [ + "SPACETIMEDB_DATABASE_IDENTITY", + "SPACETIMEDB_SERVER_URI", + "SPACETIMEDB_CREDENTIAL_BROKER", + "SPACETIMEDB_TOKEN", + "A=B", + "0BAD", + "BAD\0KEY", + ] { + assert!(validate_env_key(key).is_err(), "{key:?}"); + } + for key in ["API_KEY", "_CUSTOM", "path", "A1"] { + validate_env_key(key).unwrap(); + } +} + +#[test] +fn zero_or_overflowing_resource_requests_are_not_admitted() { + let limits = ContainerSpecLimits::default(); + for cpu in [0, u64::MAX, limits.resources.cpu_millicores + 1] { + let mut value = spec(); + value.resources.cpu_millicores = cpu; + assert_eq!(value.validate(&limits).unwrap_err().field, "resources"); + } + let mut value = spec(); + value.resources.scratch_bytes = 0; + assert!(value.validate(&limits).is_err()); +} + +#[test] +fn successful_jobs_and_on_failure_services_remain_terminal() { + assert!(!RestartPolicy::Never.restarts_after(None)); + assert!(!RestartPolicy::Never.restarts_after(Some(1))); + assert!(!RestartPolicy::OnFailure.restarts_after(Some(0))); + assert!(RestartPolicy::OnFailure.restarts_after(None)); + assert!(RestartPolicy::OnFailure.restarts_after(Some(1))); + let mut job = spec(); + job.mode = ContainerMode::Job; + job.restart = RestartPolicy::Always; + assert_eq!( + job.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "restart" + ); +} + +#[cfg(feature = "serde")] +#[test] +fn json_requires_explicit_port_exposure_and_rejects_privileged_fields() { + let original = spec(); + let json = serde_json::to_value(&original).unwrap(); + assert_eq!(serde_json::from_value::(json.clone()).unwrap(), original); + let mut missing = json.clone(); + missing["ports"][0].as_object_mut().unwrap().remove("exposure"); + assert!(serde_json::from_value::(missing).is_err()); + let mut injected = json; + injected + .as_object_mut() + .unwrap() + .insert("privileged".into(), true.into()); + assert!(serde_json::from_value::(injected).is_err()); + let keep: ContainerAction = serde_json::from_str(r#"{"action":"keep"}"#).unwrap(); + assert_eq!(keep, ContainerAction::Keep); + let remove: ContainerAction = serde_json::from_str(r#"{"action":"remove"}"#).unwrap(); + assert_eq!(remove, ContainerAction::Remove); + assert!(serde_json::from_str::(r#"{"action":"remove","value":{}}"#).is_err()); +} + +#[test] +fn concrete_container_actions_preserve_wire_tags() { + let container = spec(); + assert_eq!(bsatn::to_vec(&ContainerAction::Keep).unwrap(), [0]); + assert_eq!(bsatn::to_vec(&ContainerAction::Remove).unwrap(), [2]); + let mut expected = vec![1]; + expected.extend(bsatn::to_vec(&container).unwrap()); + assert_eq!(bsatn::to_vec(&ContainerAction::Set(container)).unwrap(), expected); +} diff --git a/crates/lib/src/container_environment.rs b/crates/lib/src/container_environment.rs new file mode 100644 index 00000000000..f2ff648a45e --- /dev/null +++ b/crates/lib/src/container_environment.rs @@ -0,0 +1,40 @@ +//! Immutable application environment identity, independent of expiring launch credentials. +//! +//! These values describe a request. They do not authenticate a client or grant +//! access to environment values. Only the trusted host/control protocol may +//! capture, read, or close a snapshot. + +use crate::{Hash, Identity, SpacetimeType, Uuid}; + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct EnvironmentSnapshotScope { + pub cluster: String, + pub database_id: u64, + pub database_identity: Identity, + pub node_id: u64, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub node_incarnation: Uuid, + pub generation: u64, + pub deployment_revision: Hash, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub start_request: Uuid, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub env_generation: Uuid, + /// The complete sorted, unique key list of the committed container declaration. + pub env_keys: Vec, +} + +/// Stable identity of a committed capture. No secret values or value hashes. +/// A durability barrier belongs to each proof, not to this immutable receipt. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct EnvironmentSnapshotReceipt { + pub scope: EnvironmentSnapshotScope, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub capture_receipt: Uuid, +} diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs new file mode 100644 index 00000000000..538d5b7dc23 --- /dev/null +++ b/crates/lib/src/deployment.rs @@ -0,0 +1,256 @@ +//! Normalized deployment protocol shared by publication coordinators and hosts. +//! +//! Module artifacts and OCI objects are uploaded separately. A deployment is +//! immutable configuration, never a place for runtime credentials or env values. + +use crate::container::{ContainerAction, ContainerSpec, ContainerSpecLimits, ContainerValidationError}; +use crate::{bsatn, hash_bytes, Hash, SpacetimeType, Uuid}; + +#[cfg(feature = "serde")] +pub mod api; +pub mod manifest; + +pub const PUBLISH_PROTOCOL_VERSION: u32 = 1; +pub const SYSTEM_EMPTY_MODULE_VERSION: u32 = 1; +/// Immutable built-in program, also used by clients for authorized migration +/// preflight. Replacing these bytes requires a new system-module version. +pub const SYSTEM_EMPTY_MODULE_V1_BYTES: &[u8] = include_bytes!("deployment/system_empty_v1.wasm"); +/// Immutable Keccak-256 program identity of the version-1 bundled empty Wasm +/// module. Control can verify initial program bytes without linking the host. +pub const SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH: Hash = Hash::from_byte_array([ + 0x83, 0xcc, 0x1c, 0xc8, 0x79, 0x4f, 0x7a, 0x9a, 0x54, 0x0a, 0x07, 0x43, 0xd0, 0xf8, 0x74, 0xbf, 0x61, 0x3b, 0x76, + 0x67, 0x6c, 0xa9, 0xe8, 0xec, 0xbd, 0xc4, 0xcf, 0xca, 0x19, 0xc7, 0x09, 0xa5, +]); +/// SHA-256 descriptor of the same immutable bundled version-1 bytes. Clients +/// can name SystemEmpty in a prepared manifest without linking the host. +pub const SYSTEM_EMPTY_MODULE_V1_ARTIFACT: manifest::ModuleArtifact = manifest::ModuleArtifact { + digest: crate::container::OciDigest::sha256([ + 0x02, 0xbd, 0x77, 0xcf, 0x25, 0xb4, 0x79, 0x08, 0xf4, 0x06, 0x4c, 0x3d, 0xdf, 0x0f, 0xa7, 0xec, 0xcc, 0xa8, + 0xc6, 0x16, 0x5d, 0x0a, 0x41, 0xc4, 0x45, 0x54, 0x69, 0x54, 0x6b, 0x14, 0xce, 0x02, + ]), + size_bytes: 250, +}; +pub const MAX_DEPLOYMENT_BYTES: usize = 256 * 1024; +pub const PUBLISH_RETRY_WINDOW_MS: u64 = 7 * 24 * 60 * 60 * 1000; +pub const MAX_OPERATION_CLOCK_SKEW_MS: u64 = 5 * 60 * 1000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum UserModuleKind { + Wasm, + Js, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct UserModule { + pub kind: UserModuleKind, + /// The existing module program hash, not an OCI object digest. + pub program_hash: Hash, +} + +/// Explicit module replacement/removal, with its own exported schema name. +#[derive(Clone, Debug, Default, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ModuleAction { + #[default] + Keep, + Set(UserModule), + Remove, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "kind", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ModuleComponent { + SystemEmpty(u32), + User(UserModule), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct DeploymentSpecV1 { + pub module: ModuleComponent, + pub container: Option, +} + +/// Persist the discriminant along with the payload. Unknown encodings fail to +/// decode; a restore must never fall back to an empty or older deployment. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde( + tag = "version", + content = "deployment", + rename_all = "snake_case", + deny_unknown_fields + ) +)] +pub enum DeploymentSpec { + V1(DeploymentSpecV1), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct PublishEnvelope { + pub version: u32, + #[cfg_attr(feature = "serde", serde(with = "uuid_json"))] + pub operation_id: Uuid, + /// Exact compare-and-set precondition. None means no deployment record has + /// been installed yet, rather than permission to overwrite any revision. + pub expected_revision: Option, + #[cfg_attr(feature = "serde", serde(default))] + pub module_action: ModuleAction, + #[cfg_attr(feature = "serde", serde(default))] + pub container_action: ContainerAction, +} + +#[derive(Debug, thiserror::Error)] +pub enum DeploymentValidationError { + #[error("unsupported publish protocol version")] + UnsupportedVersion, + #[error("unsupported platform empty-module version")] + UnsupportedEmptyModule, + #[error("operation_id must be a version 7 UUID")] + InvalidOperationId, + #[error("publication operation has expired")] + ExpiredOperation, + #[error("publication operation timestamp is too far in the future")] + FutureOperation, + #[error("deployment exceeds the protocol size limit")] + TooLarge, + #[error("deployment encoding is invalid or unsupported")] + InvalidEncoding, + #[error(transparent)] + Container(#[from] ContainerValidationError), +} + +impl DeploymentSpec { + pub fn current(&self) -> &DeploymentSpecV1 { + match self { + Self::V1(spec) => spec, + } + } + + pub fn normalize(self, limits: &ContainerSpecLimits) -> Result { + let Self::V1(mut spec) = self; + if let ModuleComponent::SystemEmpty(version) = spec.module + && version != SYSTEM_EMPTY_MODULE_VERSION + { + return Err(DeploymentValidationError::UnsupportedEmptyModule); + } + spec.container = spec.container.map(|spec| spec.normalize(limits)).transpose()?; + let spec = Self::V1(spec); + spec.encode()?; + Ok(spec) + } + + pub fn encode(&self) -> Result, DeploymentValidationError> { + let bytes = bsatn::to_vec(self).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + Ok(bytes.into()) + } + + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + bsatn::from_slice(bytes).map_err(|_| DeploymentValidationError::InvalidEncoding) + } + + /// Call on the result of normalize. Unlike a request fingerprint, this is + /// independent of operation ID, publisher, and mutable execution status. + pub fn revision(&self) -> Result { + let mut bytes = b"spacetimedb/deployment\0".to_vec(); + bytes.extend_from_slice(&self.encode()?); + Ok(hash_bytes(bytes)) + } +} + +impl PublishEnvelope { + pub fn resolve( + &self, + previous: Option<&DeploymentSpec>, + limits: &ContainerSpecLimits, + ) -> Result { + if self.version != PUBLISH_PROTOCOL_VERSION { + return Err(DeploymentValidationError::UnsupportedVersion); + } + use spacetimedb_sats::uuid::Version; + if !matches!(self.operation_id.get_version(), Some(Version::V7)) { + return Err(DeploymentValidationError::InvalidOperationId); + } + let prior = previous.map(DeploymentSpec::current); + let module = match &self.module_action { + ModuleAction::Keep => prior + .map(|p| p.module.clone()) + .unwrap_or(ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION)), + ModuleAction::Set(module) => ModuleComponent::User(module.clone()), + ModuleAction::Remove => ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + }; + let container = match &self.container_action { + ContainerAction::Keep => prior.and_then(|p| p.container.clone()), + ContainerAction::Set(container) => Some(container.clone()), + ContainerAction::Remove => None, + }; + DeploymentSpec::V1(DeploymentSpecV1 { module, container }).normalize(limits) + } + + pub fn requires_container_permission(&self) -> bool { + matches!(self.container_action, ContainerAction::Set(_)) + } +} + +/// UUIDv7 embeds its creation millisecond. This makes an expired retry +/// distinguishable from a new request even after its ledger row is collected. +/// A fresh UUID with a changed timestamp is a different operation. +pub fn operation_expiry_ms(id: Uuid, now_ms: u64) -> Result { + if id.get_version() != Some(spacetimedb_sats::uuid::Version::V7) { + return Err(DeploymentValidationError::InvalidOperationId); + } + let created_ms = (id.as_u128() >> 80) as u64; + if created_ms > now_ms.saturating_add(MAX_OPERATION_CLOCK_SKEW_MS) { + return Err(DeploymentValidationError::FutureOperation); + } + let expires_ms = created_ms + PUBLISH_RETRY_WINDOW_MS; + if now_ms >= expires_ms { + return Err(DeploymentValidationError::ExpiredOperation); + } + Ok(expires_ms) +} + +#[cfg(feature = "serde")] +pub mod uuid_json { + use super::Uuid; + pub fn serialize(id: &Uuid, serializer: S) -> Result { + serializer.collect_str(id) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + Uuid::parse_str(&value).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/deployment/api.rs b/crates/lib/src/deployment/api.rs new file mode 100644 index 00000000000..c1adcebef6e --- /dev/null +++ b/crates/lib/src/deployment/api.rs @@ -0,0 +1,117 @@ +//! HTTP publication messages shared by the CLI, dashboard and Cloud. Artifacts +//! are uploaded separately; no message carries credentials or environment values. + +use super::{manifest::PreparedDeploymentManifest, uuid_json, DeploymentSpec, PUBLISH_PROTOCOL_VERSION}; +use crate::{container::OciDigest, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactReference { + pub digest: OciDigest, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishRequest { + pub manifest: PreparedDeploymentManifest, + /// Must match the server-generated reservation when creating a database. + pub creation: Option, + /// Original uploaded OCI index or executable manifest. Required for Set. + /// Keep uses the prior retained executable manifest; Remove has no image. + pub image_source: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CreationOptions { + pub parent: Option, + pub organization: Option, + pub num_replicas: Option, + #[serde(default = "default_anti_affinity")] + pub enforce_anti_affinity: bool, +} +fn default_anti_affinity() -> bool { + true +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReserveDatabaseRequest { + pub version: u32, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub options: CreationOptions, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DatabaseReservation { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub expires_at: String, + pub staging_open: bool, + pub artifact_endpoint: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PublicationPhase { + Prepared, + Quiescing, + Committed, + Activating, + Complete, + AbortedBeforeCommit, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicationStatus { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub phase: PublicationPhase, + pub expected_revision: Option, + pub proposed_revision: Hash, + pub error: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeploymentStatus { + pub database_identity: Identity, + /// None means this database has not yet used managed publication. + pub revision: Option, + pub deployment: DeploymentSpec, + /// Lets Keep select exactly the currently installed program bytes. + pub module_artifact: ArtifactReference, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishPermission { + pub identity: Identity, + pub can_publish: bool, + /// Decimal string preserves all u64 revision values in browser clients. + pub source_revision: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicationCapabilities { + pub version: u32, + pub enabled: bool, + pub artifact_endpoint: Option, +} +impl PublicationCapabilities { + pub fn disabled() -> Self { + Self { + version: PUBLISH_PROTOCOL_VERSION, + enabled: false, + artifact_endpoint: None, + } + } +} diff --git a/crates/lib/src/deployment/manifest.rs b/crates/lib/src/deployment/manifest.rs new file mode 100644 index 00000000000..0727643b0c1 --- /dev/null +++ b/crates/lib/src/deployment/manifest.rs @@ -0,0 +1,172 @@ +//! Immutable publication recovery inputs. The manifest's SHA-256 artifact +//! digest binds migration intent as well as the effective deployment. Its +//! digest differs from the deployment revision, which excludes migration policy. + +use super::{DeploymentSpec, DeploymentValidationError, PublishEnvelope, MAX_DEPLOYMENT_BYTES}; +use crate::container::{ContainerSpecLimits, OciDigest}; +use crate::{bsatn, Hash, SpacetimeType}; + +pub const MAX_MODULE_ARTIFACT_BYTES: u64 = 32 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ModuleArtifact { + /// SHA-256 of the complete stored bytes, distinct from the module's Keccak hash. + pub digest: OciDigest, + pub size_bytes: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "policy", content = "token", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum PreparedMigrationPolicy { + Compatible, + /// The existing migration token binds database Identity and old/new module + /// hashes. Recovery must retain the originally acknowledged policy. + BreakClients(Hash), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct PreparedDeploymentManifestV1 { + pub envelope: PublishEnvelope, + pub deployment: DeploymentSpec, + /// Always retained, including for the bundled empty module. Genesis and + /// later recovery must select exactly the admitted program bytes. + pub module_artifact: ModuleArtifact, + pub migration_policy: PreparedMigrationPolicy, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde( + tag = "version", + content = "manifest", + rename_all = "snake_case", + deny_unknown_fields + ) +)] +pub enum PreparedDeploymentManifest { + V1(PreparedDeploymentManifestV1), +} + +impl PreparedDeploymentManifest { + pub fn current(&self) -> &PreparedDeploymentManifestV1 { + match self { + Self::V1(manifest) => manifest, + } + } + + /// Validate before retaining the artifact. This checks the encoding and + /// metadata; the artifact service verifies SHA-256/length and the host + /// validates the selected program, capabilities and actual migration. + pub fn validate(&self, limits: &ContainerSpecLimits) -> Result<(), DeploymentValidationError> { + let manifest = self.current(); + if manifest.module_artifact.size_bytes == 0 || manifest.module_artifact.size_bytes > MAX_MODULE_ARTIFACT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + if manifest.deployment.clone().normalize(limits)? != manifest.deployment { + return Err(DeploymentValidationError::InvalidEncoding); + } + // Using the prepared components as the Keep baseline verifies every + // explicit Set/Remove action without inventing a prior deployment. + // The host separately checks the real prior state under its fence. + if manifest.envelope.resolve(Some(&manifest.deployment), limits)? != manifest.deployment { + return Err(DeploymentValidationError::InvalidEncoding); + } + self.encode()?; + Ok(()) + } + + pub fn encode(&self) -> Result, DeploymentValidationError> { + let bytes = bsatn::to_vec(self).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + Ok(bytes.into()) + } + + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + let manifest: Self = bsatn::from_slice(bytes).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + // Reject trailing or noncanonical bytes even if the decoder accepts + // them, so every retained descriptor names one unambiguous manifest. + if manifest.encode()?.as_ref() != bytes { + return Err(DeploymentValidationError::InvalidEncoding); + } + Ok(manifest) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::deployment::{DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; + + fn manifest() -> PreparedDeploymentManifest { + PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope: PublishEnvelope { + version: super::super::PUBLISH_PROTOCOL_VERSION, + operation_id: crate::Uuid::from_u128(0x01991ec4000070008000000000000001), + expected_revision: None, + module_action: super::super::ModuleAction::Keep, + container_action: crate::container::ContainerAction::Keep, + }, + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: None, + }), + module_artifact: ModuleArtifact { + digest: OciDigest::sha256([17; 32]), + size_bytes: 250, + }, + migration_policy: PreparedMigrationPolicy::Compatible, + }) + } + + #[test] + fn retained_manifest_preserves_migration_intent_without_changing_revision() { + let first = manifest(); + let mut acknowledged = first.clone(); + let PreparedDeploymentManifest::V1(value) = &mut acknowledged; + value.migration_policy = PreparedMigrationPolicy::BreakClients(Hash::from_byte_array([32; 32])); + assert_eq!( + first.current().deployment.revision().unwrap(), + acknowledged.current().deployment.revision().unwrap() + ); + let encoded = acknowledged.encode().unwrap(); + assert_ne!(first.encode().unwrap(), encoded); + assert_eq!(PreparedDeploymentManifest::decode(&encoded).unwrap(), acknowledged); + acknowledged.validate(&Default::default()).unwrap(); + } + + #[test] + fn retained_manifest_rejects_unknown_encoding_and_invalid_module_bounds() { + let mut value = manifest(); + let mut trailing = value.encode().unwrap().to_vec(); + trailing.push(0); + assert!(PreparedDeploymentManifest::decode(&trailing).is_err()); + let mut unknown = value.encode().unwrap().to_vec(); + unknown[0] = 1; + assert!(PreparedDeploymentManifest::decode(&unknown).is_err()); + let PreparedDeploymentManifest::V1(inner) = &mut value; + inner.module_artifact.size_bytes = 0; + assert!(value.validate(&Default::default()).is_err()); + let PreparedDeploymentManifest::V1(inner) = &mut value; + inner.module_artifact.size_bytes = MAX_MODULE_ARTIFACT_BYTES + 1; + assert!(value.validate(&Default::default()).is_err()); + } +} diff --git a/crates/lib/src/deployment/system_empty_v1.wasm b/crates/lib/src/deployment/system_empty_v1.wasm new file mode 100644 index 0000000000000000000000000000000000000000..e2b57b1e4ffa90a21dbb30d3cec1feee3d843818 GIT binary patch literal 250 zcmY+9L2JV>427TNY=Z_eLw6qSA1En Uuid { + Uuid::parse_str("01991ec4-0000-7000-8000-000000000001").unwrap() +} + +fn request(module_action: ModuleAction) -> PublishEnvelope { + PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: operation_id(), + expected_revision: None, + module_action, + container_action: ContainerAction::Keep, + } +} + +#[test] +fn component_actions_preserve_or_explicitly_remove_the_module() { + let limits = ContainerSpecLimits::default(); + let user = UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(b"valid module artifact"), + }; + let set = request(ModuleAction::Set(user.clone())).resolve(None, &limits).unwrap(); + assert_eq!(set.current().module, ModuleComponent::User(user)); + let keep = request(ModuleAction::Keep).resolve(Some(&set), &limits).unwrap(); + assert_eq!(set, keep); + let remove = request(ModuleAction::Remove).resolve(Some(&set), &limits).unwrap(); + assert_eq!( + remove.current().module, + ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION) + ); + assert_ne!(remove.revision().unwrap(), keep.revision().unwrap()); +} + +#[test] +fn concrete_module_actions_preserve_wire_tags_and_export_distinct_names() { + let module = UserModule { + kind: UserModuleKind::Js, + program_hash: hash_bytes(b"module"), + }; + assert_eq!(bsatn::to_vec(&ModuleAction::Keep).unwrap(), [0]); + assert_eq!(bsatn::to_vec(&ModuleAction::Remove).unwrap(), [2]); + let mut expected = vec![1]; + expected.extend(bsatn::to_vec(&module).unwrap()); + assert_eq!(bsatn::to_vec(&ModuleAction::Set(module)).unwrap(), expected); + + use crate::db::raw_def::v10::{RawModuleDefV10Builder, RawModuleDefV10Section}; + let mut builder = RawModuleDefV10Builder::new(); + builder.add_type::(); + let raw = builder.finish(); + let names: Vec<_> = raw + .sections + .iter() + .filter_map(|section| match section { + RawModuleDefV10Section::Types(types) => Some(types), + _ => None, + }) + .flatten() + .map(|ty| &*ty.source_name.source_name) + .collect(); + let distinct: std::collections::BTreeSet<_> = names.iter().copied().collect(); + assert_eq!( + names.len(), + distinct.len(), + "publish envelope exports duplicate type names" + ); + assert!(distinct.contains("ModuleAction")); + assert!(distinct.contains("ContainerAction")); +} + +#[test] +fn unknown_deployments_do_not_fall_back_to_an_empty_module() { + assert!(DeploymentSpec::decode(&[255]).is_err()); + let mut envelope = request(ModuleAction::Keep); + envelope.version += 1; + assert!(matches!( + envelope.resolve(None, &ContainerSpecLimits::default()), + Err(DeploymentValidationError::UnsupportedVersion) + )); + let spec = DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(9000), + container: None, + }); + assert!(matches!( + spec.normalize(&ContainerSpecLimits::default()), + Err(DeploymentValidationError::UnsupportedEmptyModule) + )); +} + +#[test] +fn operation_age_is_enforced_even_when_no_ledger_row_remains() { + let id = operation_id(); + let created_ms = (id.as_u128() >> 80) as u64; + assert_eq!( + operation_expiry_ms(id, created_ms).unwrap(), + created_ms + PUBLISH_RETRY_WINDOW_MS + ); + assert!(matches!( + operation_expiry_ms(id, created_ms + PUBLISH_RETRY_WINDOW_MS), + Err(DeploymentValidationError::ExpiredOperation) + )); + assert!(matches!( + operation_expiry_ms(id, created_ms - MAX_OPERATION_CLOCK_SKEW_MS - 1), + Err(DeploymentValidationError::FutureOperation) + )); + assert!(matches!( + operation_expiry_ms(Uuid::NIL, created_ms), + Err(DeploymentValidationError::InvalidOperationId) + )); +} + +#[cfg(feature = "serde")] +#[test] +fn uuid_json_is_lossless_and_omission_means_keep() { + let envelope = request(ModuleAction::Keep); + let mut json = serde_json::to_value(&envelope).unwrap(); + assert_eq!(json["operation_id"], operation_id().to_string()); + json.as_object_mut().unwrap().remove("module_action"); + json.as_object_mut().unwrap().remove("container_action"); + let parsed: PublishEnvelope = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(parsed, envelope); + assert!(!parsed.requires_container_permission()); + json["publisher_identity"] = "untrusted owner override".into(); + assert!(serde_json::from_value::(json).is_err()); + let binary = bsatn::to_vec(&envelope).unwrap(); + assert_eq!(bsatn::from_slice::(&binary).unwrap(), envelope); +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 55d5621a16d..8918e7b90f7 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -11,7 +11,10 @@ use std::any::TypeId; use std::collections::{btree_map, BTreeMap}; pub mod connection_id; +pub mod container; +pub mod container_environment; pub mod db; +pub mod deployment; mod direct_index_key; pub mod environment; pub mod error; diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml new file mode 100644 index 00000000000..f32bc83ab59 --- /dev/null +++ b/crates/oci/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "spacetimedb-oci" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license-file = "../../LICENSE.txt" +description = "Bounded OCI image validation for SpacetimeDB container publishing" + +[dependencies] +spacetimedb-lib = { workspace = true, features = ["serde"] } +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2 = "0.10" +flate2.workspace = true +tar.workspace = true +zstd = "0.13" + +[lints] +workspace = true diff --git a/crates/oci/src/layers.rs b/crates/oci/src/layers.rs new file mode 100644 index 00000000000..c7d99cb39de --- /dev/null +++ b/crates/oci/src/layers.rs @@ -0,0 +1,373 @@ +//! Stream-verify layer expansion without extracting anything onto the host. +//! Resource admission uses measured bytes and bounded metadata, never compressed +//! sizes alone. The runtime must also enforce a dedicated finite cache filesystem. + +use crate::{Descriptor, OciDigest}; +use anyhow::{bail, ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + io::{self, Read}, +}; + +#[derive(Clone, Copy, Debug)] +pub struct LayerLimits { + pub max_uncompressed_bytes: u64, + pub max_regular_file_bytes: u64, + pub max_entries: u64, + pub max_metadata_bytes: usize, + pub max_path_bytes: usize, + pub zstd_window_log_max: u32, +} +impl Default for LayerLimits { + fn default() -> Self { + Self { + max_uncompressed_bytes: 128 * 1024 * 1024 * 1024, + max_regular_file_bytes: 64 * 1024 * 1024 * 1024, + max_entries: 1_000_000, + max_metadata_bytes: 64 * 1024, + max_path_bytes: 4096, + zstd_window_log_max: 27, + } + } +} + +#[derive(Clone, Copy, Debug, Serialize)] +pub struct VerifiedLayerSize { + pub uncompressed_tar_bytes: u64, + pub entries: u64, + pub regular_file_bytes: u64, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct VerifiedImageSize { + pub compressed_bytes: u64, + pub uncompressed_tar_bytes: u64, + pub entries: u64, + pub cache_reservation_bytes: u64, +} +impl VerifiedImageSize { + /// Conservative cache accounting, supplemented by the backend's hard + /// filesystem bound. Includes temporary compressed/unpacked copies and + /// per-entry/per-layer filesystem metadata. The trusted artifact service + /// supplies these verified measurements, never a publisher JSON field. + pub fn from_layers(compressed_bytes: u64, layers: &[VerifiedLayerSize]) -> Result { + let mut tar = 0u64; + let mut entries = 0u64; + for layer in layers { + tar = tar + .checked_add(layer.uncompressed_tar_bytes) + .context("expanded image size overflow")?; + entries = entries + .checked_add(layer.entries) + .context("image entry count overflow")?; + } + let reservation = compressed_bytes + .checked_mul(2) + .and_then(|n| tar.checked_mul(3).and_then(|v| n.checked_add(v))) + .and_then(|n| entries.checked_mul(64 * 1024).and_then(|v| n.checked_add(v))) + .and_then(|n| { + (layers.len() as u64) + .checked_mul(16 * 1024 * 1024) + .and_then(|v| n.checked_add(v)) + }) + .context("image cache reservation overflow")?; + Ok(Self { + compressed_bytes, + uncompressed_tar_bytes: tar, + entries, + cache_reservation_bytes: reservation, + }) + } +} + +struct HashBounded { + inner: R, + hash: Sha256, + count: u64, + limit: u64, +} +impl Read for HashBounded { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + if self.count == self.limit { + let mut extra = [0u8; 1]; + if self.inner.read(&mut extra)? == 0 { + return Ok(0); + } + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "layer exceeds configured byte bound", + )); + } + let maximum = buf.len().min((self.limit - self.count).min(usize::MAX as u64) as usize); + let count = self.inner.read(&mut buf[..maximum])?; + self.count += count as u64; + self.hash.update(&buf[..count]); + Ok(count) + } +} + +pub fn verify_layer( + reader: impl Read, + descriptor: &Descriptor, + diff_id: OciDigest, + limits: LayerLimits, +) -> Result { + verify_layer_with_check(reader, descriptor, diff_id, limits, || Ok(())) +} + +/// Check cancellation/deadline between both compressed and expanded reads. +/// The caller retains its worker admission until this synchronous operation ends. +pub fn verify_layer_with_check( + reader: impl Read, + descriptor: &Descriptor, + diff_id: OciDigest, + limits: LayerLimits, + check: impl Fn() -> io::Result<()> + Copy, +) -> Result { + struct Checked { + reader: R, + check: F, + } + impl io::Result<()>> Read for Checked { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + (self.check)()?; + self.reader.read(buf) + } + } + check()?; + ensure!( + descriptor.size > 0 && descriptor.size <= crate::MAX_IMAGE_BYTES, + "invalid compressed layer size" + ); + ensure!( + descriptor.urls.is_empty() && descriptor.data.is_none(), + "external layer sources are unsupported" + ); + let mut compressed = HashBounded { + inner: Checked { reader, check }, + hash: Sha256::new(), + count: 0, + limit: descriptor.size, + }; + let decoder: Box = match descriptor.media_type.as_str() { + "application/vnd.oci.image.layer.v1.tar" | "application/vnd.docker.image.rootfs.diff.tar" => { + Box::new(&mut compressed) + } + "application/vnd.oci.image.layer.v1.tar+gzip" | "application/vnd.docker.image.rootfs.diff.tar.gzip" => { + Box::new(flate2::read::MultiGzDecoder::new(&mut compressed)) + } + "application/vnd.oci.image.layer.v1.tar+zstd" => { + let mut decoder = zstd::stream::read::Decoder::new(&mut compressed)?; + decoder.window_log_max(limits.zstd_window_log_max)?; + Box::new(decoder) + } + _ => bail!("unsupported or foreign layer media type"), + }; + let mut expanded = HashBounded { + inner: Checked { reader: decoder, check }, + hash: Sha256::new(), + count: 0, + limit: limits.max_uncompressed_bytes, + }; + let (entries, regular_file_bytes) = scan_tar(&mut expanded, limits)?; + let uncompressed_tar_bytes = expanded.count; + ensure!( + OciDigest::sha256(expanded.hash.clone().finalize().into()) == diff_id, + "layer uncompressed SHA-256 does not match rootfs diff ID" + ); + drop(expanded); + ensure!( + compressed.count == descriptor.size + && OciDigest::sha256(compressed.hash.finalize().into()) == descriptor.digest, + "compressed layer SHA-256 or length mismatch" + ); + Ok(VerifiedLayerSize { + uncompressed_tar_bytes, + entries, + regular_file_bytes, + }) +} + +fn scan_tar(reader: &mut impl Read, limits: LayerLimits) -> Result<(u64, u64)> { + let mut entries = 0u64; + let mut regular_bytes = 0u64; + let mut pax = BTreeMap::new(); + let mut global = BTreeMap::new(); + let mut long_path = None; + let mut long_link = None; + loop { + let mut block = [0u8; 512]; + let first = reader.read(&mut block[..1])?; + if first == 0 { + break; + } + reader.read_exact(&mut block[1..]).context("truncated TAR header")?; + if block.iter().all(|&b| b == 0) { + continue; + } + entries = entries.checked_add(1).context("TAR entry count overflow")?; + ensure!(entries <= limits.max_entries, "too many layer entries"); + let header = tar::Header::from_byte_slice(&block); + let checksum = block[..148] + .iter() + .chain(&block[156..]) + .map(|&b| u32::from(b)) + .sum::() + + 8 * 32; + ensure!(header.cksum()? == checksum, "invalid TAR header checksum"); + let kind = header.entry_type().as_byte(); + let header_size = header.entry_size()?; + if matches!(kind, b'x' | b'g' | b'L' | b'K') { + ensure!( + header_size <= limits.max_metadata_bytes as u64, + "TAR metadata entry is too large" + ); + let mut bytes = vec![0; header_size as usize]; + reader.read_exact(&mut bytes)?; + skip_padding(reader, header_size)?; + match kind { + b'x' => { + ensure!(pax.is_empty(), "duplicate local PAX metadata"); + pax = parse_pax(&bytes)?; + } + b'g' => { + let update = parse_pax(&bytes)?; + global.extend(update); + ensure!( + global.len() <= 128 + && global.iter().map(|(k, v)| k.len() + v.len()).sum::() + <= limits.max_metadata_bytes, + "global PAX metadata exceeds bound" + ); + } + b'L' => { + ensure!(long_path.is_none(), "duplicate GNU long path"); + long_path = Some(trim_nul(bytes)); + } + b'K' => { + ensure!(long_link.is_none(), "duplicate GNU long link"); + long_link = Some(trim_nul(bytes)); + } + _ => unreachable!(), + } + continue; + } + let property = |name: &str| pax.get(name).or_else(|| global.get(name)); + let size = property("size") + .map(|v| v.parse::()) + .transpose()? + .unwrap_or(header_size); + ensure!(size <= limits.max_regular_file_bytes, "layer file exceeds size bound"); + let raw_path = header.path_bytes(); + let path = property("path") + .map(String::as_bytes) + .or(long_path.as_deref()) + .unwrap_or(&raw_path); + validate_path(path, limits.max_path_bytes, kind == b'5')?; + match kind { + 0 | b'0' | b'7' => { + regular_bytes = regular_bytes.checked_add(size).context("layer file size overflow")?; + } + b'5' => ensure!(size == 0, "directory entry has data"), + b'1' | b'2' => { + ensure!(size == 0, "link entry has data"); + let raw_link = header.link_name_bytes(); + let link = property("linkpath") + .map(String::as_bytes) + .or(long_link.as_deref()) + .or(raw_link.as_deref()) + .context("link target is missing")?; + ensure!( + !link.is_empty() && link.len() <= limits.max_path_bytes && !link.contains(&0), + "invalid link target" + ); + if kind == b'1' { + validate_path(link, limits.max_path_bytes, false)?; + } + // Absolute symbolic links are normal inside a Linux image. + // Extraction remains the confined runtime unpacker's job. + } + _ => bail!("unsupported sparse, special-device, or unknown TAR entry type"), + } + skip_exact(reader, size)?; + skip_padding(reader, size)?; + pax.clear(); + long_path = None; + long_link = None; + } + ensure!( + pax.is_empty() && long_path.is_none() && long_link.is_none(), + "orphaned TAR extension metadata" + ); + Ok((entries, regular_bytes)) +} + +fn trim_nul(mut bytes: Vec) -> Vec { + while bytes.last() == Some(&0) { + bytes.pop(); + } + bytes +} +fn skip_exact(reader: &mut impl Read, mut bytes: u64) -> Result<()> { + let mut buffer = [0u8; 64 * 1024]; + while bytes > 0 { + let count = bytes.min(buffer.len() as u64) as usize; + reader.read_exact(&mut buffer[..count])?; + bytes -= count as u64; + } + Ok(()) +} +fn skip_padding(reader: &mut impl Read, size: u64) -> Result<()> { + skip_exact(reader, (512 - size % 512) % 512) +} +fn validate_path(path: &[u8], max: usize, root_directory: bool) -> Result<()> { + ensure!( + !path.is_empty() && path.len() <= max && !path.contains(&0) && !path.starts_with(b"/"), + "invalid layer entry path" + ); + ensure!( + !path.split(|&b| b == b'/').any(|p| p == b".."), + "layer entry path escapes image root" + ); + ensure!( + root_directory || path.split(|&b| b == b'/').any(|p| !p.is_empty() && p != b"."), + "invalid root file entry" + ); + Ok(()) +} +fn parse_pax(mut bytes: &[u8]) -> Result> { + let mut result = BTreeMap::new(); + while !bytes.is_empty() { + let space = bytes.iter().position(|&b| b == b' ').context("invalid PAX record")?; + ensure!(space > 0 && space <= 10, "invalid PAX length"); + let length = std::str::from_utf8(&bytes[..space])?.parse::()?; + ensure!( + length > space + 2 && length <= bytes.len() && bytes[length - 1] == b'\n', + "invalid PAX record length" + ); + let record = std::str::from_utf8(&bytes[space + 1..length - 1])?; + let (key, value) = record.split_once('=').context("invalid PAX property")?; + ensure!( + key.len() <= 256 + && (matches!( + key, + "path" | "linkpath" | "size" | "mtime" | "atime" | "ctime" | "uid" | "gid" | "uname" | "gname" + ) || key.starts_with("SCHILY.xattr.")), + "unsupported sparse or unknown PAX property" + ); + ensure!( + result.insert(key.to_owned(), value.to_owned()).is_none() && result.len() <= 128, + "duplicate or excessive PAX properties" + ); + bytes = &bytes[length..]; + } + Ok(result) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oci/src/layers/tests.rs b/crates/oci/src/layers/tests.rs new file mode 100644 index 00000000000..044ec7bb43e --- /dev/null +++ b/crates/oci/src/layers/tests.rs @@ -0,0 +1,130 @@ +use super::*; +use std::io::Write; + +fn archive() -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_ustar(); + header.set_path("usr/bin/main").unwrap(); + header.set_size(4); + header.set_mode(0o755); + header.set_cksum(); + builder.append(&header, &b"data"[..]).unwrap(); + builder.into_inner().unwrap() +} +fn descriptor(bytes: &[u8], media: &str) -> Descriptor { + Descriptor { + digest: crate::sha256(bytes), + size: bytes.len() as u64, + media_type: media.into(), + platform: None, + urls: vec![], + data: None, + artifact_type: None, + } +} +fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} +#[test] +fn verifies_plain_gzip_and_zstd_against_actual_expanded_bytes() { + let tar = archive(); + let diff = crate::sha256(&tar); + for (bytes, media) in [ + (tar.clone(), "application/vnd.oci.image.layer.v1.tar"), + (gzip(&tar), "application/vnd.oci.image.layer.v1.tar+gzip"), + ( + zstd::stream::encode_all(&tar[..], 1).unwrap(), + "application/vnd.oci.image.layer.v1.tar+zstd", + ), + ] { + let result = verify_layer(&bytes[..], &descriptor(&bytes, media), diff, LayerLimits::default()).unwrap(); + assert_eq!(result.uncompressed_tar_bytes, tar.len() as u64); + assert_eq!(result.entries, 1); + assert_eq!(result.regular_file_bytes, 4); + assert!( + VerifiedImageSize::from_layers(bytes.len() as u64, &[result]) + .unwrap() + .cache_reservation_bytes + > result.uncompressed_tar_bytes + ); + } +} +#[test] +fn decompression_bomb_is_stopped_before_declared_diff_id_can_be_trusted() { + let expanded = vec![0u8; 1024 * 1024]; + let compressed = gzip(&expanded); + let result = verify_layer( + &compressed[..], + &descriptor(&compressed, "application/vnd.oci.image.layer.v1.tar+gzip"), + crate::sha256(&expanded), + LayerLimits { + max_uncompressed_bytes: 4096, + ..LayerLimits::default() + }, + ); + assert!(result.unwrap_err().to_string().contains("byte bound")); +} +#[test] +fn catches_digest_mismatch_truncation_and_entry_limit() { + let bytes = archive(); + let descriptor = descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar"); + assert!(verify_layer(&bytes[..], &descriptor, crate::sha256(b"wrong"), LayerLimits::default()).is_err()); + assert!(verify_layer( + &bytes[..100], + &descriptor, + crate::sha256(&bytes), + LayerLimits::default() + ) + .is_err()); + assert!(verify_layer( + &bytes[..], + &descriptor, + crate::sha256(&bytes), + LayerLimits { + max_entries: 0, + ..LayerLimits::default() + } + ) + .is_err()); +} +#[test] +fn extension_size_is_bounded_before_allocating_and_sparse_metadata_is_rejected() { + let mut header = tar::Header::new_gnu(); + header.set_path("pax").unwrap(); + header.set_entry_type(tar::EntryType::XHeader); + header.set_size(1 << 30); + header.set_cksum(); + let bytes = header.as_bytes().to_vec(); + assert!(verify_layer( + &bytes[..], + &descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar"), + crate::sha256(&bytes), + LayerLimits::default() + ) + .unwrap_err() + .to_string() + .contains("metadata entry")); + assert!(parse_pax(b"25 GNU.sparse.size=123456\n").is_err()); + assert!(validate_path(b"safe/../../host", 4096, false).is_err()); + assert!(validate_path(b"/absolute", 4096, false).is_err()); +} +#[test] +fn counts_concatenated_gzip_members_and_checks_pax_sizes() { + let tar = archive(); + let mut bytes = gzip(&tar); + bytes.extend(gzip(&tar)); + let mut both = tar.clone(); + both.extend(&tar); + let result = verify_layer( + &bytes[..], + &descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar+gzip"), + crate::sha256(&both), + LayerLimits::default(), + ) + .unwrap(); + assert_eq!(result.entries, 2); + assert_eq!(parse_pax(b"10 size=4\n").unwrap()["size"], "4"); + assert!(parse_pax(b"99 size=4\n").is_err()); +} diff --git a/crates/oci/src/lib.rs b/crates/oci/src/lib.rs new file mode 100644 index 00000000000..aa1bc63f19b --- /dev/null +++ b/crates/oci/src/lib.rs @@ -0,0 +1,377 @@ +//! Validate immutable OCI objects before accepting a container deployment. +//! +//! Registry references and index annotations are discovery inputs. Only verified +//! object bytes and an exact selected platform establish the published image. + +pub mod layers; + +use anyhow::{bail, ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::container::{ImagePlatform, OciDigest, MAX_ARGV_ENTRIES, MAX_ENV_KEYS, MAX_EXEC_STRING_BYTES}; +use std::collections::BTreeMap; + +pub const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; +pub const OCI_INDEX: &str = "application/vnd.oci.image.index.v1+json"; +pub const OCI_CONFIG: &str = "application/vnd.oci.image.config.v1+json"; +pub const DOCKER_MANIFEST: &str = "application/vnd.docker.distribution.manifest.v2+json"; +pub const DOCKER_INDEX: &str = "application/vnd.docker.distribution.manifest.list.v2+json"; +pub const DOCKER_CONFIG: &str = "application/vnd.docker.container.image.v1+json"; +pub const MAX_MANIFEST_BYTES: usize = 4 * 1024 * 1024; +pub const MAX_CONFIG_BYTES: usize = 1024 * 1024; +pub const MAX_LAYERS: usize = 256; +pub const MAX_INDEX_ENTRIES: usize = 256; +pub const MAX_IMAGE_BYTES: u64 = 64 * 1024 * 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Descriptor { + pub media_type: String, + pub digest: OciDigest, + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub urls: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact_type: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Platform { + pub os: String, + pub architecture: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub variant: Option, + #[serde(default, rename = "os.version", skip_serializing_if = "Option::is_none")] + pub os_version: Option, + #[serde(default, rename = "os.features", skip_serializing_if = "Vec::is_empty")] + pub os_features: Vec, +} + +impl Platform { + fn matches(&self, requested: &ImagePlatform) -> bool { + self.os == requested.os + && self.architecture == requested.architecture + && self.os_version.as_deref().is_none_or(str::is_empty) + && self.os_features.is_empty() + && matches!( + (self.architecture.as_str(), self.variant.as_deref()), + (_, None | Some("")) | ("arm64", Some("v8")) + ) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Manifest { + pub schema_version: u32, + pub media_type: String, + pub config: Descriptor, + pub layers: Vec, + #[serde(default)] + artifact_type: Option, + #[serde(default)] + subject: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImageIndex { + pub schema_version: u32, + pub media_type: String, + pub manifests: Vec, +} + +pub fn sha256(bytes: &[u8]) -> OciDigest { + OciDigest::sha256(Sha256::digest(bytes).into()) +} + +pub fn verify_object(descriptor: &Descriptor, bytes: &[u8]) -> Result<()> { + ensure!( + descriptor.size == bytes.len() as u64, + "OCI object length differs from its descriptor" + ); + ensure!( + sha256(bytes) == descriptor.digest, + "OCI object SHA-256 differs from its descriptor" + ); + Ok(()) +} + +fn validate_descriptor(descriptor: &Descriptor, max_bytes: u64) -> Result<()> { + ensure!( + descriptor.size > 0 && descriptor.size <= max_bytes, + "OCI descriptor size exceeds admission bounds" + ); + ensure!( + descriptor.urls.is_empty(), + "external OCI descriptor URLs are not supported" + ); + ensure!(descriptor.data.is_none(), "inline OCI descriptor data is not supported"); + ensure!( + descriptor.artifact_type.is_none(), + "OCI artifacts are not executable images" + ); + Ok(()) +} + +pub fn parse_manifest(bytes: &[u8]) -> Result { + ensure!(bytes.len() <= MAX_MANIFEST_BYTES, "OCI manifest is too large"); + let manifest: Manifest = serde_json::from_slice(bytes).context("invalid OCI image manifest")?; + ensure!(manifest.schema_version == 2, "unsupported OCI manifest schema version"); + ensure!( + matches!(manifest.media_type.as_str(), OCI_MANIFEST | DOCKER_MANIFEST), + "unsupported image manifest media type" + ); + ensure!( + manifest.artifact_type.is_none() && manifest.subject.is_none(), + "OCI artifact manifests are not executable images" + ); + ensure!(manifest.layers.len() <= MAX_LAYERS, "too many OCI image layers"); + validate_descriptor(&manifest.config, MAX_CONFIG_BYTES as u64)?; + ensure!( + matches!(manifest.config.media_type.as_str(), OCI_CONFIG | DOCKER_CONFIG), + "unsupported image config media type" + ); + let mut total = manifest.config.size; + for layer in &manifest.layers { + validate_descriptor(layer, MAX_IMAGE_BYTES)?; + ensure!( + matches!( + layer.media_type.as_str(), + "application/vnd.oci.image.layer.v1.tar" + | "application/vnd.oci.image.layer.v1.tar+gzip" + | "application/vnd.oci.image.layer.v1.tar+zstd" + | "application/vnd.docker.image.rootfs.diff.tar" + | "application/vnd.docker.image.rootfs.diff.tar.gzip" + ), + "unsupported or foreign image layer media type" + ); + total = total.checked_add(layer.size).context("OCI image size overflow")?; + } + ensure!(total <= MAX_IMAGE_BYTES, "OCI image exceeds compressed object quota"); + Ok(manifest) +} + +/// Select exactly one executable image for the requested platform. BuildKit may +/// include attestation descriptors for unknown/unknown; they are never executed. +pub fn select_platform(bytes: &[u8], requested: &ImagePlatform) -> Result { + validate_platform(requested)?; + ensure!(bytes.len() <= MAX_MANIFEST_BYTES, "OCI image index is too large"); + let index: ImageIndex = serde_json::from_slice(bytes).context("invalid OCI image index")?; + ensure!( + index.schema_version == 2 && matches!(index.media_type.as_str(), OCI_INDEX | DOCKER_INDEX), + "unsupported image index format" + ); + ensure!( + index.manifests.len() <= MAX_INDEX_ENTRIES, + "too many image index entries" + ); + let mut selected = None; + for descriptor in index.manifests { + if !descriptor.platform.as_ref().is_some_and(|p| p.matches(requested)) { + continue; + } + validate_descriptor(&descriptor, MAX_MANIFEST_BYTES as u64)?; + ensure!( + matches!(descriptor.media_type.as_str(), OCI_MANIFEST | DOCKER_MANIFEST), + "selected platform is not an image manifest" + ); + ensure!( + selected.replace(descriptor).is_none(), + "OCI index has ambiguous images for the selected platform" + ); + } + selected.context("OCI image does not contain the selected Linux platform") +} + +fn validate_platform(platform: &ImagePlatform) -> Result<()> { + ensure!( + platform.os == "linux" && matches!(platform.architecture.as_str(), "amd64" | "arm64"), + "unsupported container platform" + ); + Ok(()) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ImageConfig { + pub architecture: String, + pub os: String, + #[serde(default)] + pub variant: Option, + #[serde(default)] + pub config: ContainerConfig, + pub rootfs: RootFs, +} + +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct ContainerConfig { + #[serde(default)] + pub entrypoint: Option>, + #[serde(default)] + pub cmd: Option>, + #[serde(default)] + pub user: String, + #[serde(default, rename = "WorkingDir")] + pub working_directory: String, + #[serde(default)] + pub env: Option>, + #[serde(default)] + pub volumes: Option>, +} + +impl std::fmt::Debug for ContainerConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ContainerConfig") + .field("entrypoint", &self.entrypoint) + .field("cmd", &self.cmd) + .field("user", &self.user) + .field("working_directory", &self.working_directory) + .field( + "env_keys", + &self.env.as_ref().map(|env| { + env.iter() + .map(|v| v.split('=').next().unwrap_or("")) + .collect::>() + }), + ) + .field("volumes", &self.volumes.as_ref().map(|v| v.keys().collect::>())) + .finish() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct RootFs { + #[serde(rename = "type")] + pub kind: String, + pub diff_ids: Vec, +} + +pub fn parse_config(bytes: &[u8], manifest: &Manifest, platform: &ImagePlatform) -> Result { + validate_platform(platform)?; + ensure!(bytes.len() <= MAX_CONFIG_BYTES, "OCI image config is too large"); + verify_object(&manifest.config, bytes)?; + let image: ImageConfig = serde_json::from_slice(bytes).context("invalid OCI image config")?; + ensure!( + Platform { + os: image.os.clone(), + architecture: image.architecture.clone(), + variant: image.variant.clone(), + os_version: None, + os_features: vec![] + } + .matches(platform), + "image config platform does not match selected platform" + ); + ensure!( + image.rootfs.kind == "layers" && image.rootfs.diff_ids.len() == manifest.layers.len(), + "image rootfs does not match layer descriptors" + ); + ensure!( + image.config.volumes.as_ref().is_none_or(BTreeMap::is_empty), + "image-declared volumes are unsupported in Stage 1" + ); + image.config.environment()?; + for argv in [&image.config.entrypoint, &image.config.cmd].into_iter().flatten() { + ensure!(argv.len() <= MAX_ARGV_ENTRIES, "too many image command arguments"); + for arg in argv { + validate_string(arg)?; + } + } + validate_string(&image.config.user)?; + validate_string(&image.config.working_directory)?; + ensure!( + image.config.working_directory.is_empty() || image.config.working_directory.starts_with('/'), + "image working directory must be absolute" + ); + Ok(image) +} + +fn validate_string(value: &str) -> Result<()> { + ensure!( + value.len() < MAX_EXEC_STRING_BYTES && !value.contains('\0'), + "invalid container startup string" + ); + Ok(()) +} + +impl ContainerConfig { + /// Keep image defaults separately from the normalized spec. Environment + /// values remain in the immutable image, never in hot deployment metadata. + pub fn environment(&self) -> Result> { + let env = self.env.as_deref().unwrap_or_default(); + ensure!(env.len() <= MAX_ENV_KEYS, "too many image environment variables"); + let mut result = BTreeMap::new(); + for value in env { + validate_string(value)?; + let (key, value) = value + .split_once('=') + .context("image environment entry must contain '='")?; + ensure!( + valid_env_key(key) && !key.starts_with("SPACETIMEDB_"), + "invalid or reserved image environment key" + ); + ensure!( + result.insert(key.to_owned(), value.to_owned()).is_none(), + "duplicate image environment key" + ); + } + Ok(result) + } + + pub fn argv(&self, override_command: Option<&[String]>) -> Result> { + let argv = match override_command { + Some(argv) => argv.to_vec(), + None => self + .entrypoint + .iter() + .flatten() + .chain(self.cmd.iter().flatten()) + .cloned() + .collect(), + }; + ensure!( + !argv.is_empty() && !argv[0].is_empty() && argv.len() <= MAX_ARGV_ENTRIES, + "image needs a nonempty main command" + ); + for arg in &argv { + validate_string(arg)?; + } + Ok(argv) + } +} + +pub fn valid_env_key(key: &str) -> bool { + let mut bytes = key.bytes(); + key.len() <= 256 + && bytes.next().is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') + && bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') +} + +/// Descriptor closure used to pin all required objects before desired-state +/// commit. Reject a digest repeated with conflicting lengths or media types. +pub fn object_closure(manifest_descriptor: Descriptor, manifest: &Manifest) -> Result> { + validate_descriptor(&manifest_descriptor, MAX_MANIFEST_BYTES as u64)?; + let mut seen = BTreeMap::new(); + let mut objects = Vec::with_capacity(manifest.layers.len() + 2); + for descriptor in std::iter::once(manifest_descriptor) + .chain(std::iter::once(manifest.config.clone())) + .chain(manifest.layers.iter().cloned()) + { + match seen.insert(descriptor.digest, (descriptor.size, descriptor.media_type.clone())) { + Some(previous) if previous != (descriptor.size, descriptor.media_type.clone()) => { + bail!("OCI digest has conflicting descriptors") + } + Some(_) => {} + None => objects.push(descriptor), + } + } + Ok(objects) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oci/src/tests.rs b/crates/oci/src/tests.rs new file mode 100644 index 00000000000..6fe4192bfa2 --- /dev/null +++ b/crates/oci/src/tests.rs @@ -0,0 +1,166 @@ +use super::*; +use serde_json::{json, Value}; + +fn platform() -> ImagePlatform { + ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + } +} +fn config() -> Value { + json!({"architecture":"arm64","os":"linux","config":{ + "Entrypoint":["node"],"Cmd":["server.js"],"User":"1000:1000","WorkingDir":"/app", + "Env":["PATH=/usr/bin","APP_KEY=image-secret"] + },"rootfs":{"type":"layers","diff_ids":[sha256(b"uncompressed layer")]}}) +} +fn fixture(config: &Value) -> (Vec, Vec, Descriptor) { + let config_bytes = serde_json::to_vec(config).unwrap(); + let manifest = serde_json::to_vec(&json!({ + "schemaVersion":2,"mediaType":OCI_MANIFEST, + "config":{"mediaType":OCI_CONFIG,"digest":sha256(&config_bytes),"size":config_bytes.len()}, + "layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":sha256(b"layer"),"size":5}] + })) + .unwrap(); + let descriptor = Descriptor { + media_type: OCI_MANIFEST.into(), + digest: sha256(&manifest), + size: manifest.len() as u64, + platform: Some(Platform { + os: "linux".into(), + architecture: "arm64".into(), + variant: Some("v8".into()), + os_version: None, + os_features: vec![], + }), + urls: vec![], + data: None, + artifact_type: None, + }; + (config_bytes, manifest, descriptor) +} + +#[test] +fn immutable_image_preserves_defaults_and_explicit_command_replaces_all_argv() { + let (bytes, raw, descriptor) = fixture(&config()); + verify_object(&descriptor, &raw).unwrap(); + let manifest = parse_manifest(&raw).unwrap(); + let image = parse_config(&bytes, &manifest, &platform()).unwrap(); + assert_eq!(image.config.argv(None).unwrap(), ["node", "server.js"]); + assert_eq!(image.config.argv(Some(&["/bin/sh".into()])).unwrap(), ["/bin/sh"]); + assert_eq!(image.config.user, "1000:1000"); + assert_eq!(image.config.working_directory, "/app"); + assert_eq!(image.config.environment().unwrap()["APP_KEY"], "image-secret"); + assert!(!format!("{image:?}").contains("image-secret")); + assert_eq!(object_closure(descriptor, &manifest).unwrap().len(), 3); +} + +#[test] +fn content_verification_checks_actual_sha256_and_length() { + assert_eq!( + sha256(b"abc").to_string(), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let (_, bytes, mut descriptor) = fixture(&config()); + assert!(verify_object(&descriptor, b"different").is_err()); + descriptor.size -= 1; + assert!(verify_object(&descriptor, &bytes).is_err()); +} + +#[test] +fn platform_selection_ignores_attestations_and_rejects_ambiguity_or_absence() { + let (_, _, descriptor) = fixture(&config()); + let mut attestation = descriptor.clone(); + attestation.platform = Some(Platform { + os: "unknown".into(), + architecture: "unknown".into(), + variant: None, + os_version: None, + os_features: vec![], + }); + let index = |descriptors: Vec| { + serde_json::to_vec(&json!({"schemaVersion":2,"mediaType":OCI_INDEX,"manifests":descriptors})).unwrap() + }; + assert_eq!( + select_platform(&index(vec![attestation.clone(), descriptor.clone()]), &platform()).unwrap(), + descriptor + ); + assert!(select_platform(&index(vec![descriptor.clone(), descriptor]), &platform()).is_err()); + assert!(select_platform(&index(vec![attestation]), &platform()).is_err()); +} + +#[test] +fn foreign_layers_urls_inline_data_and_artifacts_fail_closed() { + let (_, bytes, _) = fixture(&config()); + let baseline: Value = serde_json::from_slice(&bytes).unwrap(); + for (key, value) in [ + ("urls", json!(["http://169.254.169.254/latest/meta-data/"])), + ("data", json!("inline")), + ("artifactType", json!("application/test")), + ] { + let mut manifest = baseline.clone(); + manifest["layers"][0][key] = value; + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + } + let mut manifest = baseline.clone(); + manifest["layers"][0]["mediaType"] = json!("application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + let mut manifest = baseline; + manifest["artifactType"] = json!("application/test"); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); +} + +#[test] +fn config_rejects_volumes_wrong_platform_rootfs_and_reserved_environment() { + for (field, value) in [ + ("Volumes", json!({"/escape":{}})), + ("Env", json!(["SPACETIMEDB_IDENTITY=forged"])), + ("Env", json!(["KEY=a", "KEY=b"])), + ("Env", json!(["NO_EQUALS"])), + ("Env", json!(["KEY=contains\u{0}nul"])), + ("WorkingDir", json!("relative/path")), + ] { + let mut config = config(); + config["config"][field] = value; + let (bytes, raw, _) = fixture(&config); + assert!( + parse_config(&bytes, &parse_manifest(&raw).unwrap(), &platform()).is_err(), + "{field}" + ); + } + for (field, value) in [ + ("architecture", json!("amd64")), + ("rootfs", json!({"type":"layers","diff_ids":[]})), + ] { + let mut config = config(); + config[field] = value; + let (bytes, raw, _) = fixture(&config); + assert!(parse_config(&bytes, &parse_manifest(&raw).unwrap(), &platform()).is_err()); + } +} + +#[test] +fn size_limits_and_conflicting_descriptors_are_enforced() { + let (_, bytes, descriptor) = fixture(&config()); + let mut manifest: Value = serde_json::from_slice(&bytes).unwrap(); + manifest["layers"][0]["size"] = json!(MAX_IMAGE_BYTES); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + assert!(parse_manifest(&vec![b' '; MAX_MANIFEST_BYTES + 1]).is_err()); + let mut manifest = parse_manifest(&bytes).unwrap(); + let mut conflicting = manifest.layers[0].clone(); + conflicting.size += 1; + manifest.layers.push(conflicting); + assert!(object_closure(descriptor, &manifest).is_err()); +} + +#[test] +fn empty_scratch_image_requires_explicit_main_command() { + let config = ContainerConfig::default(); + assert!(config.argv(None).is_err()); + assert!(config.argv(Some(&[])).is_err()); + assert!(config.argv(Some(&["".into()])).is_err()); + assert_eq!(config.argv(Some(&["/main".into()])).unwrap(), ["/main"]); + assert!(valid_env_key("_A0")); + assert!(!valid_env_key("0A")); + assert!(valid_env_key(&"A".repeat(256))); + assert!(!valid_env_key(&"A".repeat(257))); +} diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index fdf85987338..532d69176d3 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -32,6 +32,10 @@ directives = [ # Maximum number of JS procedure isolates per database. Omit to use the number # of cores reported by the OS. # procedure-instance-pool-size = 8 +# Wall-clock bound for each JS startup, schema description and function call. +# Defaults to 120 seconds; must be positive and no greater than 120 seconds. +# Timed-out reducers fail and roll back, including lifecycle calls during publish. +# execution-timeout = "120s" [v8-heap-policy] # Check the V8 heap after this many requests. Set to 0 to disable. diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 55c5795f533..f04329d9505 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -35,6 +35,8 @@ futures.workspace = true [dev-dependencies] env_logger.workspace = true +spacetimedb-auth.workspace = true +spacetimedb-datastore.workspace = true serial_test.workspace = true [lints] diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 465e22c3d6e..fa0b9275f4d 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -67,6 +67,12 @@ pub struct ModuleHandle { } impl ModuleHandle { + /// Access the real standalone control/host environment for integration + /// tests that publish, migrate, or recover the running module. + pub fn environment(&self) -> &StandaloneEnv { + &self.env + } + async fn call_reducer_result(&self, reducer: &str, args: FunctionArgs) -> anyhow::Result { let result = self .client diff --git a/crates/testing/tests/deployment_publish.rs b/crates/testing/tests/deployment_publish.rs new file mode 100644 index 00000000000..dd3b7323967 --- /dev/null +++ b/crates/testing/tests/deployment_publish.rs @@ -0,0 +1,204 @@ +//! Exercise publication against the real host and Wasm module, including the +//! unchanged-program path and retries after a later module has been installed. +use serial_test::serial; +use spacetimedb::db::deployment::{current_deployment, install_publication_fence, DeploymentCommit}; +use spacetimedb::host::{FunctionArgs, UpdateDatabaseResult}; +use spacetimedb::messages::control_db::HostType; +use spacetimedb_client_api::{ControlStateReadAccess, NodeDelegate}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::ST_DEPLOYMENT_OPERATION_ID; +use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; +use spacetimedb_lib::{hash_bytes, sats::product, ConnectionId, Hash, Identity, Uuid}; +use spacetimedb_schema::auto_migrate::{MigrationPolicy, MigrationToken}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +fn prepared(bytes: &[u8], epoch: u64, previous: Option, command: &str) -> DeploymentCommit { + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + DeploymentCommit { + operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(epoch)), + publication_epoch: epoch, + publisher: Identity::ZERO, + expected_revision: previous, + prepared_manifest_hash: hash_bytes(epoch.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(bytes), + }), + container: Some(ContainerSpec { + image_manifest: OciDigest::sha256([3; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec![command.into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 1000, + }), + }), + } +} + +fn fence(module: &spacetimedb::host::ModuleHost, request: &DeploymentCommit) { + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); +} + +async fn confirmed(result: UpdateDatabaseResult) { + let (offset, durable) = match result { + UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + } + | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { + tx_offset, + durable_offset, + } + | UpdateDatabaseResult::DeploymentAlreadyCommitted { + tx_offset, + durable_offset, + .. + } => (tx_offset, durable_offset), + other => panic!("expected a durable publication result, got {other:?}"), + }; + tokio::time::timeout(Duration::from_secs(10), async { + let offset = offset.await.unwrap(); + if let Some(mut durable) = durable { + durable.wait_for(offset).await.unwrap(); + } + }) + .await + .unwrap(); +} + +#[test] +#[serial] +fn container_only_changes_and_old_retries_preserve_the_current_module() { + let compiled = CompiledModule::compile("hosted-auth-test", CompilationMode::Debug); + let bytes = compiled.program_bytes(); + compiled.with_module_async(DEFAULT_CONFIG, |handle| async move { + let env = handle.environment(); + let database = env.get_database_by_identity(&handle.db_identity).await.unwrap().unwrap(); + let host = env.leader(database.id).await.unwrap(); + let module = host.module().await.unwrap(); + let first = prepared(&bytes, 1, None, "/app/first"); + fence(&module, &first); + // Even the first in-progress container publication fences the legacy + // raw API, including a byte-identical module update. + assert!(host.update(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible).await.is_err()); + // The rejected updater must restore the old host in its controller. + assert_eq!(host.module().await.unwrap().info.module_hash, hash_bytes(&bytes)); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); + confirmed(result).await; + let first_revision = first.deployment.revision().unwrap(); + let second = prepared(&bytes, 2, Some(first_revision), "/app/second"); + fence(&host.module().await.unwrap(), &second); + confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, second.clone()).await.unwrap()).await; + assert_ne!(first_revision, second.deployment.revision().unwrap()); + + // A valid custom section changes program bytes/hash without changing + // its definition. This exercises the actual Wasm migration and swap. + let mut newer_bytes = bytes.to_vec(); + newer_bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let third = prepared(&newer_bytes, 3, Some(second.deployment.revision().unwrap()), "/app/third"); + fence(&host.module().await.unwrap(), &third); + confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, third.clone()).await.unwrap()).await; + let newest = third.deployment.revision().unwrap(); + assert_eq!(host.module().await.unwrap().info.module_hash, hash_bytes(&newer_bytes)); + let before_retry = host.module().await.unwrap(); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); + assert!(matches!(&result, UpdateDatabaseResult::DeploymentAlreadyCommitted { result, .. } if result.revision == first_revision)); + confirmed(result).await; + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, before_retry.info.module_hash); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + }); + current.call_reducer(Identity::ZERO, None, None, None, None, "private_only", FunctionArgs::Nullary).await.unwrap().outcome.into_result().unwrap(); + + // A stale CAS and a forged association between bytes and declaration + // cannot change either component, and rejection leaves service usable. + let stale = prepared(&bytes, 4, Some(first_revision), "/app/stale"); + fence(¤t, &stale); + assert!(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, stale).await.is_err()); + let mismatched = prepared(&bytes, 5, Some(newest), "/app/mismatch"); + fence(¤t, &mismatched); + assert!(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, mismatched).await.is_err()); + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, hash_bytes(&newer_bytes)); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + }); + + let empty = spacetimedb::host::empty_module::program(1).unwrap(); + let mut remove_module = prepared(&empty.bytes, 6, Some(newest), "/app/image-only"); + let DeploymentSpec::V1(spec) = &mut remove_module.deployment; + spec.module = ModuleComponent::SystemEmpty(1); + // An init request cannot execute this instance's schema while storing + // the bytes of a different valid, prepared program. + assert!(current.init_database_with_deployment(empty.clone(), Some(remove_module.clone())).await.is_err()); + + // Removing a nonempty table fails during migration execution, after + // the tentative program and deployment/receipt writes. All roll back. + let observation_name = current.info.module_def.tables() + .find(|table| table.name.to_ascii_lowercase().contains("observation")) + .unwrap().name.to_string(); + let observation_table = current.relational_db().with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<_> { + let table = tx.table_id_from_name(&observation_name)?.unwrap(); + tx.insert_via_serialize_bsatn(table, &product![ConnectionId::from_u128(700), Identity::ZERO, false, Identity::ZERO, false])?; + Ok(table) + }).unwrap(); + let policy = MigrationPolicy::BreakClients(MigrationToken { + database_identity: handle.db_identity, + old_module_hash: current.info.module_hash, + new_module_hash: empty.hash, + }.hash()); + fence(¤t, &remove_module); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, empty.bytes.clone(), policy.clone(), remove_module.clone()).await.unwrap(); + assert!(matches!(result, UpdateDatabaseResult::ErrorExecutingMigration(ref error) if error.to_string().contains("table contains data")), "{result:?}"); + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, hash_bytes(&newer_bytes)); + assert_eq!(current.relational_db().program().unwrap().unwrap().hash, hash_bytes(&newer_bytes)); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + assert_eq!(tx.table_row_count(observation_table), Some(1)); + }); + current.relational_db().with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + tx.clear_table(observation_table)?; + Ok(()) + }).unwrap(); + // The same operation remains eligible after an execution rollback. + confirmed(host.update_with_deployment(database, HostType::Wasm, empty.bytes.clone(), policy, remove_module.clone()).await.unwrap()).await; + let current = host.module().await.unwrap(); + assert!(current.info.module_def.tables().next().is_none()); + assert!(spacetimedb::host::empty_module::matches_program(1, ¤t.relational_db().program().unwrap().unwrap())); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, remove_module.deployment.revision().unwrap()); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(4)); + }); + }); +} diff --git a/crates/testing/tests/hosted_invocation.rs b/crates/testing/tests/hosted_invocation.rs new file mode 100644 index 00000000000..7c9b3c5528c --- /dev/null +++ b/crates/testing/tests/hosted_invocation.rs @@ -0,0 +1,261 @@ +//! Exercise Rust Wasm bindings and host admission using actual signed proofs. +use serial_test::serial; +use spacetimedb::auth::hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}; +use spacetimedb::auth::invocation::InvocationCaller; +use spacetimedb::auth::JwtKeys; +use spacetimedb::db::deployment::install_container_fence; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_auth::identity::ConnectionAuthCtx; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + StContainerFenceRow, ST_CLIENT_ID, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, +}; +use spacetimedb_lib::sats::{product, AlgebraicValue, ProductValue}; +use spacetimedb_lib::{bsatn, ConnectionId, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::{Duration, SystemTime}; + +fn authenticate(source: Identity, target: Identity, lifetime: Duration) -> ConnectionAuthCtx { + let keys = JwtKeys::generate().unwrap(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + source_database: source, + target_database: target, + generation: 1, + grant_revision: 1, + lease_expires_at: now + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "test.platform", + &binding, + now, + now + lifetime, + "wasm-integration", + ) + .unwrap(); + HostedTokenValidator::new([("test.platform".into(), keys.public)]) + .unwrap() + .validate_token(&token, target, now, |issuer, requested_source, requested_target| { + (issuer == "test.platform" && requested_source == source && requested_target == target).then_some(binding) + }) + .unwrap() + .into_connection_auth() + .unwrap() +} + +fn install_fence(module: &ModuleHost, source: Identity, generation: u64, allowed: bool) { + let db = module.relational_db(); + db.with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + install_container_fence( + db, + tx, + &StContainerFenceRow { + source_identity: source.into(), + generation, + target_grant_revision: generation, + target_set_hash: spacetimedb_lib::hash_bytes(b"configured targets"), + allowed, + }, + )?; + Ok(()) + }) + .unwrap(); +} + +fn assert_connection_count(module: &ModuleHost, expected: u64) { + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(expected)); + } + Ok(()) + }) + .unwrap(); +} + +fn arguments(values: ProductValue) -> FunctionArgs { + FunctionArgs::Bsatn(bsatn::to_vec(&values).unwrap().into()) +} + +async fn call( + module: &ModuleHost, + caller: impl Into + Send, + connection: Option, + reducer: &str, + args: ProductValue, +) -> anyhow::Result<()> { + module + .call_reducer(caller, connection, None, None, None, reducer, arguments(args)) + .await? + .outcome + .into_result() +} + +#[test] +#[serial] +fn hosted_wasm_calls_preserve_authority_and_disconnect_after_revocation() { + CompiledModule::compile("hosted-auth-test", CompilationMode::Debug).with_module_async( + DEFAULT_CONFIG, + |handle| async move { + let module = handle.client.module(); + let target = handle.db_identity; + let foreign = Identity::ONE; + let owner = Identity::ZERO; + let self_connection = ConnectionId::from_u128(101); + let foreign_connection = ConnectionId::from_u128(102); + let failed_disconnect = ConnectionId::from_u128(999); + let expired_connection = ConnectionId::from_u128(103); + install_fence(&module, target, 1, true); + install_fence(&module, foreign, 1, true); + let self_auth = authenticate(target, target, Duration::from_secs(30)); + let foreign_auth = authenticate(foreign, target, Duration::from_secs(30)); + let expiring_auth = authenticate(target, target, Duration::from_secs(3)); + for (auth, connection) in [ + (self_auth.clone(), self_connection), + (foreign_auth.clone(), foreign_connection), + (self_auth.clone(), failed_disconnect), + (expiring_auth.clone(), expired_connection), + ] { + module.call_identity_connected(auth, connection).await.unwrap(); + } + assert_connection_count(&module, 4); + for (auth, sender, connection, internal) in [ + (&self_auth, target, self_connection, true), + (&foreign_auth, foreign, foreign_connection, false), + ] { + call( + &module, + auth, + Some(connection), + "inspect_context", + product![sender, Some(connection), internal, true], + ) + .await + .unwrap(); + let result = module + .call_procedure( + auth, + Some(connection), + None, + "inspect_procedure", + arguments(product![sender, Some(connection), internal]), + ) + .await; + assert_eq!(result.result.unwrap().return_val, AlgebraicValue::Bool(true)); + for lifecycle in ["connected", "disconnected"] { + assert!(call(&module, auth, Some(connection), lifecycle, product![]) + .await + .is_err()); + } + } + // Ordinary owner calls and equal database identities remain external. + for ordinary in [target, owner] { + call( + &module, + ordinary, + None, + "inspect_context", + product![ordinary, Option::::None, false, false], + ) + .await + .unwrap(); + assert!(call(&module, ordinary, None, "internal_only", product![]) + .await + .is_err()); + } + call(&module, owner, None, "private_only", product![]).await.unwrap(); + call(&module, &self_auth, Some(self_connection), "internal_only", product![]) + .await + .unwrap(); + call(&module, &self_auth, Some(self_connection), "private_only", product![]) + .await + .unwrap(); + for reducer in ["internal_only", "private_only"] { + assert!( + call(&module, &foreign_auth, Some(foreign_connection), reducer, product![]) + .await + .is_err() + ); + } + call(&module, owner, None, "schedule_check", product![]).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let result = module + .call_procedure(owner, None, None, "scheduled_finished", FunctionArgs::Nullary) + .await; + if result.result.unwrap().return_val == AlgebraicValue::Bool(true) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("scheduled call did not observe internal authority"); + // Tokens remain cryptographically valid, but their persisted grants are revoked. + install_fence(&module, target, 2, false); + install_fence(&module, foreign, 2, false); + assert!( + call(&module, &self_auth, Some(self_connection), "internal_only", product![]) + .await + .is_err() + ); + let result = module + .call_procedure( + &self_auth, + Some(self_connection), + None, + "inspect_procedure", + arguments(product![target, Some(self_connection), true]), + ) + .await; + assert!(result.result.is_err()); + if let Ok(remaining) = expiring_auth + .hosted + .as_ref() + .unwrap() + .expires_at() + .duration_since(SystemTime::now()) + { + tokio::time::sleep(remaining + Duration::from_millis(20)).await; + } + assert!(call( + &module, + &expiring_auth, + Some(expired_connection), + "internal_only", + product![] + ) + .await + .is_err()); + // Host cleanup retains captured flags and JWT sender after revocation and expiry. + for (sender, connection) in [ + (target, self_connection), + (foreign, foreign_connection), + (target, failed_disconnect), + (target, expired_connection), + ] { + module.call_identity_disconnected(sender, connection).await.unwrap(); + } + assert_connection_count(&module, 0); + for (connection, sender, internal, disconnected) in [ + (self_connection, target, true, true), + (foreign_connection, foreign, false, true), + (failed_disconnect, target, true, false), + (expired_connection, target, true, true), + ] { + call( + &module, + owner, + None, + "inspect_observation", + product![connection, sender, internal, disconnected], + ) + .await + .unwrap(); + } + }, + ); +} diff --git a/modules/hosted-auth-test/Cargo.toml b/modules/hosted-auth-test/Cargo.toml new file mode 100644 index 00000000000..b37b9b44332 --- /dev/null +++ b/modules/hosted-auth-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "hosted-auth-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/hosted-auth-test/src/lib.rs b/modules/hosted-auth-test/src/lib.rs new file mode 100644 index 00000000000..f6380f686eb --- /dev/null +++ b/modules/hosted-auth-test/src/lib.rs @@ -0,0 +1,141 @@ +//! Actual host integration fixture for verified container authentication. +use spacetimedb::{ConnectionId, Identity, ProcedureContext, ReducerContext, Table}; + +#[spacetimedb::table(accessor = observations)] +pub struct Observation { + #[primary_key] + connection: ConnectionId, + sender: Identity, + internal: bool, + jwt_identity: Identity, + disconnected: bool, +} + +#[spacetimedb::reducer(client_connected)] +pub fn connected(ctx: &ReducerContext) { + ctx.db.observations().insert(Observation { + connection: ctx.connection_id().unwrap(), + sender: ctx.sender(), + internal: ctx.sender_auth().is_internal(), + jwt_identity: ctx.sender_auth().jwt().unwrap().identity(), + disconnected: false, + }); +} + +#[spacetimedb::reducer(client_disconnected)] +pub fn disconnected(ctx: &ReducerContext) -> Result<(), String> { + let connection = ctx.connection_id().unwrap(); + let mut observation = ctx.db.observations().connection().find(connection).unwrap(); + assert_eq!(ctx.sender(), observation.sender); + assert_eq!(ctx.sender_auth().is_internal(), observation.internal); + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), observation.jwt_identity); + // Exercise host fallback cleanup after a user callback rejects disconnect. + if connection == ConnectionId::from_u128(999) { + return Err("intentional disconnect failure".into()); + } + observation.disconnected = true; + ctx.db.observations().connection().update(observation); + Ok(()) +} + +#[spacetimedb::reducer] +pub fn inspect_context( + ctx: &ReducerContext, + sender: Identity, + connection: Option, + internal: bool, + jwt: bool, +) { + assert_eq!(ctx.sender(), sender); + assert_eq!(ctx.connection_id(), connection); + assert_eq!(ctx.sender_auth().is_internal(), internal); + assert_eq!(ctx.sender_auth().has_jwt(), jwt); + if jwt { + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), sender); + } +} + +#[spacetimedb::reducer(internal)] +pub fn internal_only(ctx: &ReducerContext) { + assert!(ctx.sender_auth().is_internal()); +} + +#[spacetimedb::reducer(private)] +pub fn private_only(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer] +pub fn inspect_observation( + ctx: &ReducerContext, + connection: ConnectionId, + sender: Identity, + internal: bool, + disconnected: bool, +) { + let observation = ctx.db.observations().connection().find(connection).unwrap(); + assert_eq!(observation.sender, sender); + assert_eq!(observation.jwt_identity, sender); + assert_eq!(observation.internal, internal); + assert_eq!(observation.disconnected, disconnected); +} + +#[spacetimedb::procedure] +pub fn inspect_procedure( + ctx: &mut ProcedureContext, + sender: Identity, + connection: Option, + internal: bool, +) -> bool { + assert_eq!(ctx.sender(), sender); + assert_eq!(ctx.connection_id(), connection); + assert_eq!(ctx.sender_auth().is_internal(), internal); + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), sender); + ctx.with_tx(|tx| { + assert_eq!(tx.sender(), sender); + assert_eq!(tx.connection_id(), connection); + assert_eq!(tx.sender_auth().is_internal(), internal); + assert_eq!(tx.sender_auth().jwt().unwrap().identity(), sender); + }); + true +} + +#[spacetimedb::table(accessor = scheduled_checks, scheduled(scheduled_check))] +pub struct ScheduledCheck { + #[primary_key] + #[auto_inc] + id: u64, + scheduled_at: spacetimedb::ScheduleAt, +} + +#[spacetimedb::reducer] +pub fn schedule_check(ctx: &ReducerContext) { + ctx.db.scheduled_checks().insert(ScheduledCheck { + id: 0, + scheduled_at: ctx.timestamp.into(), + }); +} + +#[spacetimedb::reducer] +pub fn scheduled_check(ctx: &ReducerContext, _job: ScheduledCheck) { + assert_eq!(ctx.sender(), ctx.database_identity()); + assert_eq!(ctx.connection_id(), None); + assert!(ctx.sender_auth().is_internal()); + assert!(!ctx.sender_auth().has_jwt()); + ctx.db.observations().insert(Observation { + connection: ConnectionId::from_u128(777), + sender: ctx.sender(), + internal: true, + jwt_identity: ctx.sender(), + disconnected: false, + }); +} + +#[spacetimedb::procedure] +pub fn scheduled_finished(ctx: &mut ProcedureContext) -> bool { + ctx.with_tx(|tx| { + tx.db + .observations() + .connection() + .find(ConnectionId::from_u128(777)) + .is_some() + }) +} diff --git a/sdks/rust/src/client_cache.rs b/sdks/rust/src/client_cache.rs index ab0a90980bd..afe8237ffa6 100644 --- a/sdks/rust/src/client_cache.rs +++ b/sdks/rust/src/client_cache.rs @@ -2,6 +2,9 @@ //! //! This module is internal, and may incompatibly change without warning. +#[cfg(test)] +mod uuid_tests; + use crate::callbacks::CallbackId; use crate::db_connection::{debug_log, PendingMutation, SharedCell}; use crate::spacetime_module::{InModule, SpacetimeModule, TableUpdate, WithBsatn}; @@ -449,7 +452,9 @@ impl TableHandle { /// See [`DbContextImpl::queue_mutation`]. fn queue_mutation(&self, mutation: PendingMutation) { - self.pending_mutations.unbounded_send(mutation).unwrap(); + // A retained table handle may outlive terminal connection cleanup. + // The closed queue rejects and releases its callback captures. + let _ = self.pending_mutations.unbounded_send(mutation); } /// Called by the autogenerated implementation of the [`crate::Table`] method of the same name. diff --git a/sdks/rust/src/client_cache/uuid_tests.rs b/sdks/rust/src/client_cache/uuid_tests.rs new file mode 100644 index 00000000000..a9516acc81b --- /dev/null +++ b/sdks/rust/src/client_cache/uuid_tests.rs @@ -0,0 +1,75 @@ +use super::*; +use spacetimedb_lib::Uuid; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Debug)] +struct CountedRow { + id: Uuid, + value: u64, + clones: Arc, +} +impl Clone for CountedRow { + fn clone(&self) -> Self { + self.clones.fetch_add(1, Ordering::SeqCst); + Self { + id: self.id, + value: self.value, + clones: self.clones.clone(), + } + } +} + +fn row(id: Uuid, value: u64, clones: &Arc) -> WithBsatn { + WithBsatn { + bsatn: spacetimedb_lib::bsatn::to_vec(&(id, value)).unwrap().into(), + row: CountedRow { + id, + value, + clones: clones.clone(), + }, + } +} + +#[test] +fn uuid_unique_lookup_clones_only_the_match_and_tracks_updates_and_deletes() { + let clones = Arc::new(AtomicUsize::new(0)); + let mut cache = TableCache::::new(None); + // This is the exact registration emitted by Rust codegen. + cache.add_unique_constraint::("id", |row| &row.id); + let ids = (0u8..128) + .map(|n| Uuid::from_random_bytes_v4([n; 16])) + .collect::>(); + cache.apply_diff(&TableUpdate { + inserts: ids + .iter() + .enumerate() + .map(|(n, id)| row(*id, n as u64, &clones)) + .collect(), + deletes: vec![], + }); + clones.store(0, Ordering::SeqCst); + // UniqueConstraintHandle::find delegates to this lookup and clones only + // its selected row. A whole-cache snapshot would increment this 128 times. + let found = cache.find_by_unique_index("id", &ids[63]).cloned().unwrap(); + assert_eq!((found.id, found.value), (ids[63], 63)); + assert_eq!(clones.load(Ordering::SeqCst), 1); + let absent = Uuid::from_random_bytes_v4([255; 16]); + assert!(cache.find_by_unique_index("id", &absent).cloned().is_none()); + assert_eq!(clones.load(Ordering::SeqCst), 1); + + cache.apply_diff(&TableUpdate { + inserts: vec![row(ids[63], 999, &clones)], + deletes: vec![row(ids[63], 63, &clones)], + }); + clones.store(0, Ordering::SeqCst); + assert_eq!(cache.find_by_unique_index("id", &ids[63]).cloned().unwrap().value, 999); + assert_eq!(clones.load(Ordering::SeqCst), 1); + cache.apply_diff(&TableUpdate { + inserts: vec![], + deletes: vec![row(ids[63], 999, &clones)], + }); + clones.store(0, Ordering::SeqCst); + assert!(cache.find_by_unique_index("id", &ids[63]).cloned().is_none()); + assert_eq!(clones.load(Ordering::SeqCst), 0); + assert_eq!(cache.find_by_unique_index("id", &ids[64]).unwrap().value, 64); +} diff --git a/sdks/rust/src/db_connection.rs b/sdks/rust/src/db_connection.rs index 332aac1b322..8d114f4cbce 100644 --- a/sdks/rust/src/db_connection.rs +++ b/sdks/rust/src/db_connection.rs @@ -55,6 +55,9 @@ use tokio::{ pub(crate) type SharedCell = Arc>; +#[cfg(all(test, not(feature = "browser")))] +mod builder_tests; + #[cfg(not(feature = "browser"))] type SharedAsyncCell = Arc>; #[cfg(feature = "browser")] @@ -136,6 +139,12 @@ impl DbContextImpl { /// applying its mutations to the client cache and invoking callbacks. fn process_message(&self, msg: ParsedMessage) -> crate::Result<()> { self.debug_log(|out| writeln!(out, "`process_message`: {msg:?}")); + if !self.is_active() && !matches!(&msg, ParsedMessage::Error(_)) { + // Local disconnect invalidated callbacks. Drain already received + // results without interpreting them as outstanding healthy calls; + // the socket's terminal event must still reach end_connection. + return Ok(()); + } match msg { // Error: route as a connection error if we never finished connecting, // otherwise treat it as an erroneous disconnect. @@ -317,17 +326,27 @@ impl DbContextImpl { /// /// Returns the terminal error that should be returned from `advance_*` methods. fn end_connection(&self, callback_error: Option) -> crate::Error { - let mut inner = self.inner.lock().unwrap(); let return_error = callback_error.clone().unwrap_or(crate::Error::Disconnected); + let (lifecycle, db_callbacks, mut subscriptions, connect_callback, disconnect_callback, connect_error_callback) = { + let mut inner = self.inner.lock().unwrap(); + let lifecycle = inner.connection_lifecycle; + inner.connection_lifecycle = ConnectionLifecycle::Ended; + ( + lifecycle, + std::mem::take(&mut inner.db_callbacks), + std::mem::take(&mut inner.subscriptions), + inner.on_connect.take(), + inner.on_disconnect.take(), + inner.on_connect_error.take(), + ) + }; - let lifecycle = inner.connection_lifecycle; - if lifecycle == ConnectionLifecycle::Ended { - return return_error; - } - inner.connection_lifecycle = ConnectionLifecycle::Ended; - - // Set `send_chan` to `None`, since `Self::is_active` checks that. - *self.send_chan.lock().unwrap() = None; + // Serialize request enqueueing with terminal closure. A retained + // DbConnection must not retain in-flight callback captures indefinitely. + let outgoing = self.send_chan.lock().unwrap().take(); + self.discard_pending_requests(); + drop((outgoing, db_callbacks, connect_callback)); + subscriptions.on_disconnect(&self.make_event_ctx(callback_error.clone())); match lifecycle { ConnectionLifecycle::Connecting => { @@ -335,26 +354,59 @@ impl DbContextImpl { source: InternalError::new("Connection closed before receiving the initial connection message"), }); let ctx: M::ErrorContext = self.make_event_ctx(Some(callback_error.clone())); - if let Some(connect_error_callback) = inner.on_connect_error.take() { + if let Some(connect_error_callback) = connect_error_callback { connect_error_callback(&ctx, callback_error.clone()); } callback_error } ConnectionLifecycle::Connected => { let ctx: M::ErrorContext = self.make_event_ctx(callback_error.clone()); - if let Some(disconnect_callback) = inner.on_disconnect.take() { + if let Some(disconnect_callback) = disconnect_callback { disconnect_callback(&ctx, callback_error.clone()); } - - // Call the `on_disconnect` method for all subscriptions. - inner.subscriptions.on_disconnect(&ctx); - return_error } ConnectionLifecycle::Ended => return_error, } } + fn discard_pending_requests(&self) { + let (reducer_callbacks, procedure_callbacks) = { + let mut inner = self.inner.lock().unwrap(); + ( + std::mem::take(&mut inner.reducer_callbacks), + std::mem::take(&mut inner.procedure_callbacks), + ) + }; + let mut queued = Vec::new(); + { + // There is exactly one supported advance_* caller, and get_message + // releases this guard before applying a terminal message. Avoid a + // blocking_lock here because advance_one_message_async runs in Tokio. + #[cfg(not(feature = "browser"))] + let mut pending = self + .pending_mutations_recv + .try_lock() + .expect("concurrent SDK message advancement"); + #[cfg(feature = "browser")] + let mut pending = self.pending_mutations_recv.lock().unwrap(); + pending.close(); + while let Ok(Some(mutation)) = pending.try_next() { + queued.push(mutation); + } + } + // Destructors may reenter SDK methods. Drop user captures only after + // releasing the connection and pending-queue locks. These calls have an + // unknown outcome; do not manufacture a successful reducer completion. + for mutation in queued { + if let PendingMutation::Subscribe { handle, .. } = &mutation { + handle.cancel_pending_callbacks(); + } + drop(mutation); + } + drop((reducer_callbacks, procedure_callbacks)); + } + fn make_event_ctx>(&self, event: E) -> Ctx { let imp = self.clone(); Ctx::new(imp, event) @@ -362,7 +414,9 @@ impl DbContextImpl { /// Apply all queued [`PendingMutation`]s. fn apply_pending_mutations(&self) -> crate::Result<()> { - while let Ok(Some(pending_mutation)) = get_lock_sync(&self.pending_mutations_recv).try_next() { + loop { + let pending = get_lock_sync(&self.pending_mutations_recv).try_next(); + let Ok(Some(pending_mutation)) = pending else { break }; self.apply_mutation(pending_mutation)?; } @@ -372,6 +426,16 @@ impl DbContextImpl { /// Apply an individual [`PendingMutation`]. fn apply_mutation(&self, mutation: PendingMutation) -> crate::Result<()> { self.debug_log(|out| writeln!(out, "`apply_mutation`: {mutation:?}")); + if !self.is_active() + && matches!( + &mutation, + PendingMutation::InvokeReducerWithCallback { .. } | PendingMutation::InvokeProcedureWithCallback { .. } + ) + { + // A call may have been queued behind the disconnect mutation. Drop + // its captures and continue driving the actual terminal event. + return Ok(()); + } match mutation { // Subscribe: register the subscription in the [`SubscriptionManager`] // and send the `Subscribe` WS message. @@ -491,6 +555,7 @@ impl DbContextImpl { // eventually resulting in disconnect callbacks being called // if the initial connection had completed. *self.send_chan.lock().unwrap() = None; + self.discard_pending_requests(); } // Callback stuff: these all do what you expect. @@ -605,13 +670,13 @@ impl DbContextImpl { // This may be unnecessary, but `tokio::select` does not document any ordering guarantees, // and if both `pending_mutations.next()` and `recv.next()` have values ready, // we want to process the pending mutation first. - if let Ok(pending_mutation) = pending_mutations.try_next() { - return Message::Local(pending_mutation.unwrap()); + if let Ok(Some(pending_mutation)) = pending_mutations.try_next() { + return Message::Local(pending_mutation); } #[cfg(not(feature = "browser"))] tokio::select! { - pending_mutation = pending_mutations.next() => Message::Local(pending_mutation.unwrap()), + Some(pending_mutation) = pending_mutations.next() => Message::Local(pending_mutation), incoming_message = recv.next() => Message::Ws(incoming_message), } @@ -621,7 +686,10 @@ impl DbContextImpl { pin_mut!(pending_fut, recv_fut); futures::select! { - pending_mutation = pending_fut => Message::Local(pending_mutation.unwrap()), + pending_mutation = pending_fut => match pending_mutation { + Some(pending_mutation) => Message::Local(pending_mutation), + None => Message::Ws(recv_fut.await), + }, incoming_message = recv_fut => Message::Ws(incoming_message), } } @@ -716,8 +784,7 @@ impl DbContextImpl { } self.pending_mutations_send .unbounded_send(PendingMutation::Disconnect) - .unwrap(); - Ok(()) + .map_err(|_| crate::Error::Disconnected) } /// Add a [`PendingMutation`] to the `pending_mutations` queue, @@ -725,8 +792,14 @@ impl DbContextImpl { /// /// This is used to defer operations which would otherwise need to hold a lock on `self.inner`, /// as otherwise running those operations within a callback would deadlock. - fn queue_mutation(&self, mutation: PendingMutation) { + fn queue_mutation(&self, mutation: PendingMutation) -> crate::Result<()> { + let outgoing = self.send_chan.lock().unwrap(); + if outgoing.is_none() { + drop(outgoing); + return Err(crate::Error::Disconnected); + } self.pending_mutations_send.unbounded_send(mutation).unwrap(); + Ok(()) } /// Called by autogenerated table access methods. @@ -757,8 +830,7 @@ impl DbContextImpl { self.queue_mutation(PendingMutation::InvokeReducerWithCallback { reducer: reducer.into(), callback: Box::new(callback), - }); - Ok(()) + }) } /// Called by the autogenerated `DbConnection` method of the same name. @@ -788,7 +860,7 @@ impl DbContextImpl { + Send + 'static, ) { - self.queue_mutation(PendingMutation::InvokeProcedureWithCallback { + let _ = self.queue_mutation(PendingMutation::InvokeProcedureWithCallback { procedure: procedure_name, args: bsatn::to_vec(&args).expect("Failed to BSATN serialize procedure args"), callback: Box::new(move |ctx, ret| { @@ -953,6 +1025,25 @@ but you must call one of them, or else the connection will never progress. Ok(::new(imp)) } + /// Open a connection asynchronously on the current Tokio runtime. + /// + /// Unlike [`Self::build`], the WebSocket handshake does not block this task. + /// Dropping this future while the handshake is pending closes its socket + /// and releases the builder's callbacks. Callers can impose their own + /// connection deadline with [`tokio::time::timeout`]. Background connection + /// tasks are started only after the handshake succeeds. + /// + /// Requires an active Tokio runtime. As with [`Self::build`], the returned + /// connection must be advanced explicitly to receive events. + #[cfg(not(feature = "browser"))] + pub async fn build_async(self) -> crate::Result { + let handle = runtime::Handle::try_current().map_err(|error| { + InternalError::new("DbConnectionBuilder::build_async requires a Tokio runtime").with_cause(error) + })?; + let imp = self.build_native_impl(handle).await?; + Ok(::new(imp)) + } + #[cfg(feature = "browser")] pub async fn build(self) -> crate::Result { let imp = self.build_impl().await?; @@ -963,6 +1054,19 @@ but you must call one of them, or else the connection will never progress. /// to construct a [`DbContextImpl`]. #[cfg(not(feature = "browser"))] fn build_impl(self) -> crate::Result> { + let (runtime, handle) = enter_or_create_runtime()?; + // Keep an SDK-owned runtime outside its own block_on future. If the + // handshake fails, dropping that runtime inside the future would panic. + let imp = tokio::task::block_in_place(|| handle.block_on(self.build_native_impl(handle.clone())))?; + imp.inner.lock().unwrap().runtime = runtime; + Ok(imp) + } + + /// Share native construction between the synchronous and asynchronous API. + /// The handshake is the only suspension point. Until it succeeds, this + /// future owns the socket and callbacks without spawning background tasks. + #[cfg(not(feature = "browser"))] + async fn build_native_impl(self, handle: runtime::Handle) -> crate::Result> { let extra_logging = self .additional_logging_path .map(|path| { @@ -973,18 +1077,15 @@ but you must call one of them, or else the connection will never progress. .transpose()? .map(|file| Arc::new(StdMutex::new(file))); - let (runtime, handle) = enter_or_create_runtime()?; - let connection_id_override = get_connection_id_override(); - let ws_connection = tokio::task::block_in_place(|| { - handle.block_on(WsConnection::connect( - self.uri.unwrap(), - self.database_name.as_ref().unwrap(), - self.token.as_deref(), - connection_id_override, - self.params, - )) - }) + let ws_connection = WsConnection::connect( + self.uri.unwrap(), + self.database_name.as_ref().unwrap(), + self.token.as_deref(), + connection_id_override, + self.params, + ) + .await .map_err(|source| crate::Error::FailedToConnect { source: InternalError::new("Failed to initiate WebSocket connection").with_cause(source), })?; @@ -998,7 +1099,7 @@ but you must call one of them, or else the connection will never progress. let (pending_mutations_send, pending_mutations_recv) = mpsc::unbounded(); let pending_mutations_recv = Arc::new(TokioMutex::new(pending_mutations_recv)); - let inner_ctx = build_db_ctx_inner(runtime, self.on_connect, self.on_connect_error, self.on_disconnect); + let inner_ctx = build_db_ctx_inner(None, self.on_connect, self.on_connect_error, self.on_disconnect); Ok(build_db_ctx( handle, inner_ctx, @@ -1622,3 +1723,6 @@ pub(crate) fn next_query_set_id() -> QuerySetId { id: NEXT_QUERY_SET_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed), } } + +#[cfg(all(test, not(feature = "browser")))] +mod terminal_tests; diff --git a/sdks/rust/src/db_connection/builder_tests.rs b/sdks/rust/src/db_connection/builder_tests.rs new file mode 100644 index 00000000000..7158c02b9e5 --- /dev/null +++ b/sdks/rust/src/db_connection/builder_tests.rs @@ -0,0 +1,89 @@ +use super::terminal_tests::bindings::RemoteModule; +use super::*; +use std::{net::SocketAddr, time::Duration}; +use tokio::{io::AsyncReadExt, net::TcpListener, sync::oneshot, task::JoinHandle}; + +async fn stalled_handshake_peer() -> (SocketAddr, oneshot::Receiver<()>, JoinHandle<()>) { + // This owned numeric-loopback listener never completes the HTTP upgrade. + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (ready_tx, ready_rx) = oneshot::channel(); + let peer = tokio::spawn(async move { + let (mut socket, remote) = listener.accept().await.unwrap(); + assert!(remote.ip().is_loopback()); + let mut request = Vec::new(); + let mut buffer = [0u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = socket.read(&mut buffer).await.unwrap(); + assert_ne!(count, 0, "client closed before sending the upgrade request"); + request.extend_from_slice(&buffer[..count]); + assert!(request.len() <= 16 * 1024); + } + ready_tx.send(()).unwrap(); + assert_eq!( + socket.read(&mut buffer).await.unwrap(), + 0, + "pending handshake socket was retained" + ); + }); + (address, ready_rx, peer) +} + +async fn check_pending_handshake_cleanup(use_timeout: bool) { + let (address, ready, peer) = stalled_handshake_peer().await; + let capture = Arc::new(()); + let weak_capture = Arc::downgrade(&capture); + let mut connect = Box::pin( + DbConnectionBuilder::::new() + .with_uri(format!("http://{address}")) + .with_database_name("disposable-builder-handshake-test") + .on_connect(move |_, _, _| drop(capture)) + .build_async(), + ); + tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + result = &mut connect => panic!("stalled handshake unexpectedly completed: {:?}", result.err()), + result = ready => result.unwrap(), + } + }) + .await + .expect("async builder blocked the executor or never initiated its handshake"); + assert!(weak_capture.upgrade().is_some()); + + if use_timeout { + assert!(tokio::time::timeout(Duration::from_millis(30), connect).await.is_err()); + } else { + drop(connect); + } + assert!( + weak_capture.upgrade().is_none(), + "cancelled builder retained callback captures" + ); + tokio::time::timeout(Duration::from_secs(2), peer) + .await + .expect("cancelled handshake did not close its socket") + .unwrap(); +} + +// A current-thread runtime also proves the asynchronous builder does not use +// block_in_place or block_on while waiting for the peer's handshake response. +#[tokio::test] +async fn async_builder_timeout_closes_stalled_handshake_and_releases_callbacks() { + check_pending_handshake_cleanup(true).await; +} + +#[tokio::test] +async fn async_builder_cancellation_closes_stalled_handshake_and_releases_callbacks() { + check_pending_handshake_cleanup(false).await; +} + +#[test] +fn sync_builder_returns_connect_errors_without_dropping_its_runtime_inside_block_on() { + // The SDK rejects a query in the host URI before attempting any network I/O. + // Running outside Tokio exercises the synchronous API's owned runtime. + let result = DbConnectionBuilder::::new() + .with_uri("http://127.0.0.1:1/?unexpected=query") + .with_database_name("disposable-builder-error-test") + .build(); + assert!(matches!(result, Err(crate::Error::FailedToConnect { .. }))); +} diff --git a/sdks/rust/src/db_connection/terminal_tests.rs b/sdks/rust/src/db_connection/terminal_tests.rs new file mode 100644 index 00000000000..397c36a34ae --- /dev/null +++ b/sdks/rust/src/db_connection/terminal_tests.rs @@ -0,0 +1,250 @@ +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[path = "../../tests/connect_disconnect_client/src/module_bindings/mod.rs"] +pub(super) mod bindings; +use bindings::RemoteModule; + +#[derive(spacetimedb_lib::ser::Serialize)] +#[sats(crate = spacetimedb_lib)] +struct Args {} +impl InModule for Args { + type Module = RemoteModule; +} +impl From for bindings::Reducer { + fn from(_: Args) -> Self { + Self::IdentityConnected + } +} + +struct DropProbe { + context: DbContextImpl, + drops: Arc, +} +impl Drop for DropProbe { + fn drop(&mut self) { + assert!( + self.context.inner.try_lock().is_ok(), + "capture dropped under inner lock" + ); + assert!( + self.context.send_chan.try_lock().is_ok(), + "capture dropped under send lock" + ); + assert!( + self.context.pending_mutations_recv.try_lock().is_ok(), + "capture dropped under queue lock" + ); + self.drops.fetch_add(1, Ordering::SeqCst); + } +} + +fn fixture( + runtime: &Runtime, + disconnects: Arc, +) -> ( + DbContextImpl, + mpsc::UnboundedSender>, +) { + let inner = build_db_ctx_inner::( + None, + None, + None, + Some(Box::new(move |_, error| { + assert!(error.is_none()); + disconnects.fetch_add(1, Ordering::SeqCst); + })), + ); + inner.lock().unwrap().connection_lifecycle = ConnectionLifecycle::Connected; + let (outgoing, outgoing_recv) = mpsc::unbounded(); + // Keep the transport receiver alive without any network or server fixture. + let (incoming, incoming_recv) = mpsc::unbounded(); + let (pending, pending_recv) = mpsc::unbounded(); + let context = build_db_ctx( + runtime.handle().clone(), + inner, + outgoing, + Arc::new(TokioMutex::new(incoming_recv)), + pending, + Arc::new(TokioMutex::new(pending_recv)), + Some(ConnectionId::from_u128(1)), + None, + ); + runtime.spawn(async move { + let mut outgoing_recv = outgoing_recv; + while outgoing_recv.next().await.is_some() {} + }); + (context, incoming) +} + +fn queue_calls(context: &DbContextImpl, drops: &Arc) { + let reducer = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context + .invoke_reducer_with_callback(Args {}, move |_, _| { + let _capture = reducer; + panic!("unknown-outcome reducer must not report completion"); + }) + .unwrap(); + let procedure = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context.invoke_procedure_with_callback::<_, ()>("procedure", Args {}, move |_, _| { + let _capture = procedure; + panic!("unknown-outcome procedure must not report completion"); + }); +} + +#[test] +fn terminal_disconnect_releases_inflight_and_queued_requests_with_retained_context() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, _incoming) = fixture(&runtime, disconnects.clone()); + queue_calls(&context, &drops); + context.frame_tick().unwrap(); + queue_calls(&context, &drops); + assert_eq!(drops.load(Ordering::SeqCst), 0); + assert!(matches!(context.end_connection(None), crate::Error::Disconnected)); + assert_eq!(drops.load(Ordering::SeqCst), 4); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); + assert!(!context.is_active()); + + let late = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + assert!(matches!( + context.invoke_reducer_with_callback(Args {}, move |_, _| drop(late)), + Err(crate::Error::Disconnected) + )); + let late = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context.invoke_procedure_with_callback::<_, ()>("procedure", Args {}, move |_, _| drop(late)); + assert_eq!(drops.load(Ordering::SeqCst), 6); + context.end_connection(None); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); +} + +#[test] +fn queued_disconnect_releases_calls_and_drains_late_results_until_terminal_event() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, incoming) = fixture(&runtime, disconnects.clone()); + queue_calls(&context, &drops); + context.frame_tick().unwrap(); + context.disconnect().unwrap(); + queue_calls(&context, &drops); + incoming + .unbounded_send(ParsedMessage::ReducerResult { + request_id: u32::MAX, + timestamp: Timestamp::UNIX_EPOCH, + result: Ok(Ok(bindings::DbUpdate::default())), + }) + .unwrap(); + incoming + .unbounded_send(ParsedMessage::ProcedureResult { + request_id: u32::MAX, + result: Ok(Bytes::new()), + }) + .unwrap(); + context.frame_tick().unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 4); + assert_eq!( + disconnects.load(Ordering::SeqCst), + 0, + "local request is not the terminal callback" + ); + assert!(!context.is_active()); + drop(incoming); + assert!(matches!( + runtime.block_on(context.advance_one_message_async()), + Err(crate::Error::Disconnected) + )); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); +} + +#[test] +fn retained_table_and_subscription_handles_cannot_retain_callbacks_after_terminal() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, _incoming) = fixture(&runtime, disconnects); + let table = context.get_table::("connected"); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + table.on_insert(move |_, _| { + let _ = &capture; + }); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let registered = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.frame_tick().unwrap(); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let queued = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.end_connection(None); + assert_eq!(drops.load(Ordering::SeqCst), 3); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(®istered)); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&queued)); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + table.on_insert(move |_, _| { + let _ = &capture; + }); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let subscription = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + assert_eq!(drops.load(Ordering::SeqCst), 5); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&subscription)); + assert_eq!(table.iter().count(), 0); +} + +#[test] +fn cancelling_before_initial_message_releases_registered_subscription_without_callbacks() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, incoming) = fixture(&runtime, disconnects.clone()); + context.inner.lock().unwrap().connection_lifecycle = ConnectionLifecycle::Connecting; + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let subscription = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.frame_tick().unwrap(); + context.disconnect().unwrap(); + context.frame_tick().unwrap(); + drop(incoming); + assert!(matches!( + runtime.block_on(context.advance_one_message_async()), + Err(crate::Error::Disconnected) + )); + assert_eq!(disconnects.load(Ordering::SeqCst), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&subscription)); +} diff --git a/sdks/rust/src/lib.rs b/sdks/rust/src/lib.rs index e7efd286514..e08938b04f3 100644 --- a/sdks/rust/src/lib.rs +++ b/sdks/rust/src/lib.rs @@ -10,6 +10,9 @@ // code generated by the CLI's codegen references them, // but users should not. +#[cfg(test)] +extern crate self as spacetimedb_sdk; + mod callbacks; mod client_cache; mod compression; diff --git a/sdks/rust/src/subscription.rs b/sdks/rust/src/subscription.rs index 2204985ea15..3fb4be3b5b2 100644 --- a/sdks/rust/src/subscription.rs +++ b/sdks/rust/src/subscription.rs @@ -47,23 +47,11 @@ pub(crate) enum PendingUnsubscribeResult { impl SubscriptionManager { pub(crate) fn on_disconnect(&mut self, _ctx: &M::ErrorContext) { - // We need to clear all the subscriptions. - // TODO: is this correct? We don't remove them from the client cache, - // we may want to resume them in the future if we impl reconnecting, - // and users can already register on-disconnect callbacks which will run in this case. - - // NOTE(cloutiertyler) - // This function previously invoke `on_error` for all subscriptions. - // However, this is inconsistent behavior given that `on_disconnect` for - // connections no longer always has an error argument and that the user - // can add an `on_ended` callback when unsubscribing. - // - // We propose instead that `on_ended` be added to the subscription - // builder so that it can be invoked when the subscription is ended - // because of a normal disconnect, but without the user calling - // `unsubscribe_then`. This can be done in a non-breaking way. - // - // For now, we will just do nothing when a subscription ends normally. + // Disconnect does not synthesize on_error/on_ended callbacks. Retained + // handles must nevertheless stop retaining callback captures. + for (_, handle) in std::mem::take(&mut self.subscriptions) { + handle.cancel_pending_callbacks(); + } } /// Register a new subscription. This does not send the subscription to the server. @@ -187,13 +175,17 @@ impl SubscriptionBuilder { self.on_applied, self.on_error, )); - self.conn + if self + .conn .pending_mutations_send .unbounded_send(PendingMutation::Subscribe { query_set_id, handle: handle.clone(), }) - .unwrap(); + .is_err() + { + handle.cancel_pending_callbacks(); + } M::SubscriptionHandle::new(handle) } @@ -471,6 +463,16 @@ impl SubscriptionHandleImpl { } } + pub(crate) fn cancel_pending_callbacks(&self) { + let callbacks = { + let mut inner = self.inner.lock().unwrap(); + inner.status = SubscriptionServerState::Ended; + (inner.on_applied.take(), inner.on_error.take(), inner.on_ended.take()) + }; + // Callback destructors may inspect or unsubscribe this same handle. + drop(callbacks); + } + pub(crate) fn start(&self) -> Option { let mut inner = self.inner.lock().unwrap(); inner.start() From b45c76c023e61ad60c62b698326a9c152a143495 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 21:24:08 -0400 Subject: [PATCH 02/16] Bind managed environment publications to committed operation cursors --- crates/cli/Cargo.toml | 2 +- crates/cli/src/container/publish/client.rs | 73 +++- crates/cli/src/container/publish/journal.rs | 93 +++-- .../container/publish/journal/protected.rs | 91 +++++ .../publish/journal/protected/windows.rs | 334 ++++++++++++++++++ crates/cli/src/container/publish/mod.rs | 5 +- crates/cli/src/container/publish/tests.rs | 37 +- .../publish/tests/environment_tests.rs | 203 +++++++++++ crates/cli/src/subcommands/publish/managed.rs | 59 +++- .../publish/managed/environment_tests.rs | 239 +++++++++++++ crates/client-api/src/routes/database.rs | 7 + crates/core/src/db/container_environment.rs | 12 +- .../src/db/container_environment/tests.rs | 104 ++++++ crates/core/src/db/deployment.rs | 83 +++-- crates/core/src/db/deployment/tests.rs | 172 ++++++++- .../container_environment/durability_tests.rs | 3 + .../host/host_controller/deployment_tests.rs | 36 +- crates/lib/Cargo.toml | 3 +- crates/lib/src/container_environment.rs | 3 + crates/lib/src/deployment.rs | 30 ++ crates/lib/src/deployment/api.rs | 32 +- .../lib/src/deployment/api/request_decode.rs | 86 +++++ .../deployment/api/request_decode/tests.rs | 166 +++++++++ crates/lib/src/deployment/manifest.rs | 1 + crates/lib/src/deployment/tests.rs | 1 + crates/testing/tests/deployment_publish.rs | 17 +- .../00700-environment-variables.md | 2 + .../00100-cli-reference.md | 2 +- 28 files changed, 1810 insertions(+), 86 deletions(-) create mode 100644 crates/cli/src/container/publish/journal/protected.rs create mode 100644 crates/cli/src/container/publish/journal/protected/windows.rs create mode 100644 crates/cli/src/container/publish/tests/environment_tests.rs create mode 100644 crates/cli/src/subcommands/publish/managed/environment_tests.rs create mode 100644 crates/lib/src/deployment/api/request_decode.rs create mode 100644 crates/lib/src/deployment/api/request_decode/tests.rs diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 983b59b2e90..34656eaae3c 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -99,7 +99,7 @@ tikv-jemallocator = { workspace = true } rustix = { version = "1", features = ["fs", "process"] } [target.'cfg(windows)'.dependencies] -windows-sys = { workspace = true, features = ["Win32_System_Console"] } +windows-sys = { workspace = true, features = ["Win32_System_Console", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Threading", "Win32_System_SystemServices"] } [build-dependencies] serde = { workspace = true, features = ["derive"] } diff --git a/crates/cli/src/container/publish/client.rs b/crates/cli/src/container/publish/client.rs index ebea3d20545..4bc85b5206a 100644 --- a/crates/cli/src/container/publish/client.rs +++ b/crates/cli/src/container/publish/client.rs @@ -159,6 +159,71 @@ impl PublisherClient { ) .await } + /// Keep inspects the existing metadata API, not a code-download permission. + /// Identity and program hash bind its declarations to the previously + /// observed deployment; the later revision/operation CAS closes the race. + pub async fn selected_environment( + &self, + prior: &DeploymentStatus, + ) -> Result { + const MAX_SCHEMA_BYTES: usize = 16 * 1024 * 1024; + let mut response = self + .request( + Method::GET, + route( + &self.server, + &["v1", "database", &prior.database_identity.to_hex(), "schema"], + )?, + ) + .query(&[("version", "10")]) + .send() + .await?; + ensure!( + response.status() == StatusCode::OK, + "selected module schema was not authorized or available" + ); + let hash = match &prior.deployment.current().module { + spacetimedb_lib::deployment::ModuleComponent::User(module) => module.program_hash, + spacetimedb_lib::deployment::ModuleComponent::SystemEmpty(module) => module.program_hash, + }; + ensure!( + response + .headers() + .get("x-spacetimedb-database-identity") + .and_then(|h| h.to_str().ok()) + == Some(prior.database_identity.to_hex().as_str()), + "selected module database identity changed" + ); + ensure!( + response + .headers() + .get("x-spacetimedb-module-hash") + .and_then(|h| h.to_str().ok()) + == Some(hash.to_string().as_str()), + "selected module program hash changed" + ); + ensure!( + response + .content_length() + .is_none_or(|length| length <= MAX_SCHEMA_BYTES as u64), + "selected module schema exceeds its bound" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + chunk.len() <= MAX_SCHEMA_BYTES.saturating_sub(bytes.len()), + "selected module schema exceeds its bound" + ); + bytes.extend_from_slice(&chunk); + } + let spacetimedb_lib::sats::serde::SerdeWrapper(raw) = serde_json::from_slice::< + spacetimedb_lib::sats::serde::SerdeWrapper, + >(&bytes) + .map_err(|_| anyhow::anyhow!("invalid selected module schema"))?; + let module = spacetimedb_schema::def::ModuleDef::try_from(spacetimedb_lib::RawModuleDef::V10(raw)) + .map_err(|_| anyhow::anyhow!("invalid selected module schema"))?; + Ok(module.environment().clone()) + } pub async fn reserve(&self, request: &ReserveDatabaseRequest) -> Result { json( self.request( @@ -179,7 +244,11 @@ impl PublisherClient { /// Resume sends these same immutable bytes, without reconstructing a request /// from a changed tag, project configuration, or current deployment. pub async fn submit_bytes(&self, database: Identity, bytes: &[u8]) -> Result { - ensure!(bytes.len() <= 256 * 1024, "publication HTTP body exceeds 256 KiB"); + ensure!( + bytes.len() <= MAX_PUBLISH_REQUEST_BYTES, + "publication HTTP body exceeds its bound" + ); + PublishRequest::decode(bytes)?; json( self.request( Method::PUT, @@ -430,5 +499,5 @@ async fn json(mut response: reqwest::Response, action: &'st ); bytes.extend_from_slice(&chunk); } - serde_json::from_slice(&bytes).with_context(|| format!("invalid {action} response")) + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid {action} response")) } diff --git a/crates/cli/src/container/publish/journal.rs b/crates/cli/src/container/publish/journal.rs index adb8f9521d8..93f61ddcceb 100644 --- a/crates/cli/src/container/publish/journal.rs +++ b/crates/cli/src/container/publish/journal.rs @@ -1,5 +1,6 @@ -//! A publication resume record contains exact request bytes and object -//! descriptors, never credentials or resolved runtime environment values. +//! Progress metadata is separate from the protected immutable submission body. +//! Only that owner-readable body contains the complete resolved environment. +mod protected; use super::client::{ObjectRef, UploadKind, UploadStatus}; use crate::container::{oci, PreparedContainer}; use anyhow::{ensure, Context, Result}; @@ -37,7 +38,9 @@ pub struct Record { pub reservation: Option, pub requested_name: Option, /// UTF-8 JSON sent verbatim on every submission attempt. + #[serde(skip)] pub request_json: String, + /// Private local integrity check only; never included in a wire request. pub request_digest: OciDigest, pub uploads: Vec, pub submitted: bool, @@ -48,14 +51,14 @@ pub struct Record { impl Record { pub fn request(&self) -> Result { ensure!( - self.version == 1 && self.request_json.len() <= 256 * 1024, + self.version == 2 && self.request_json.len() <= MAX_PUBLISH_REQUEST_BYTES, "unsupported or oversized publication resume record" ); ensure!( spacetimedb_oci::sha256(self.request_json.as_bytes()) == self.request_digest, "publication request bytes changed" ); - let request: PublishRequest = serde_json::from_str(&self.request_json)?; + let request = PublishRequest::decode(self.request_json.as_bytes())?; let envelope = &request.manifest.current().envelope; ensure!( envelope.version == PUBLISH_PROTOCOL_VERSION, @@ -96,6 +99,12 @@ impl Record { Some(status.database_identity) == self.database && status.operation_id == envelope.operation_id && status.expected_revision == envelope.expected_revision + && status.expected_last_operation == envelope.expected_last_operation + && status.publication_epoch != 0 + && self + .status + .as_ref() + .is_none_or(|previous| previous.publication_epoch == status.publication_epoch) && status.proposed_revision == request.manifest.current().deployment.revision()?, "publication response belongs to another operation or deployment" ); @@ -105,6 +114,7 @@ impl Record { pub struct Journal { directory: PathBuf, + _parents: Vec, _lock: File, pub record: Record, } @@ -127,17 +137,15 @@ impl Journal { let has_image = image.is_some(); let directory = base.join(request.manifest.current().envelope.operation_id.to_string()); fs::create_dir_all(base)?; - let mut builder = fs::DirBuilder::new(); - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt; - builder.mode(0o700); - } - builder - .create(&directory) - .context("publication resume directory already exists; resume it rather than reusing its operation")?; + let _base_parents = protected::pin_parents(base)?; + protected::create_directory(&directory) + .context("cannot create protected publication directory; resume an existing operation instead")?; let mut incomplete = IncompleteDirectory(Some(directory.clone())); + let parents = protected::pin_parents(&directory)?; let lock = Self::lock(&directory, true)?; + let mut submission = protected::file(&directory.join("submission.json"), true, true)?; + submission.write_all(record.request_json.as_bytes())?; + submission.sync_all()?; if let Some(image) = image { image.persist(&directory.join("image"))?; } @@ -151,6 +159,7 @@ impl Journal { } let journal = Self { directory, + _parents: parents, _lock: lock, record, }; @@ -161,30 +170,37 @@ impl Journal { Ok(journal) } pub fn open(directory: &Path) -> Result { + let parents = protected::pin_parents(directory)?; + protected::directory(directory)?; let directory = directory .canonicalize() .context("publication resume directory not found")?; let lock = Self::lock(&directory, false)?; let path = directory.join("publication.json"); - ensure!( - fs::symlink_metadata(&path)?.is_file(), - "publication resume record must be a regular file" - ); let mut bytes = vec![]; - File::open(path)? + protected::file(&path, false, false)? .take(MAX_RECORD_BYTES as u64 + 1) .read_to_end(&mut bytes)?; ensure!( bytes.len() <= MAX_RECORD_BYTES, "publication resume record is too large" ); - let record: Record = serde_json::from_slice(&bytes)?; - record.request()?; + let mut record: Record = + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid publication progress record"))?; + record.request_json = String::from_utf8(read_submission(&directory)?) + .map_err(|_| anyhow::anyhow!("invalid protected publication body"))?; + let request = record.request()?; + ensure!( + directory.file_name().and_then(|name| name.to_str()) + == Some(request.manifest.current().envelope.operation_id.to_string().as_str()), + "publication directory belongs to another operation" + ); // Creation saves JSON only after artifact flush. Reconfirm the parent // link if its creator stopped before finishing that last barrier. sync_directory_chain(&directory)?; Ok(Self { directory, + _parents: parents, _lock: lock, record, }) @@ -225,11 +241,8 @@ impl Journal { Ok(()) } fn lock(directory: &Path, create: bool) -> Result { - let file = File::options() - .read(true) - .write(true) - .create_new(create) - .open(directory.join("publication.lock"))?; + protected::directory(directory)?; + let file = protected::file(&directory.join("publication.lock"), create, true)?; file.try_lock() .context("another process is using this publication resume directory")?; Ok(file) @@ -240,14 +253,29 @@ impl Journal { pub fn operation(&self) -> Result { Ok(self.record.request()?.manifest.current().envelope.operation_id) } - pub fn save(&self) -> Result<()> { + /// Check the on-disk body even on status-only recovery: missing or altered + /// retained input must never silently become an empty environment. + pub fn submission_bytes(&self) -> Result> { self.record.request()?; + let bytes = read_submission(&self.directory)?; + ensure!( + bytes == self.record.request_json.as_bytes(), + "protected publication body changed" + ); + Ok(bytes) + } + pub fn save(&self) -> Result<()> { + self.submission_bytes()?; let bytes = serde_json::to_vec_pretty(&self.record)?; ensure!( bytes.len() <= MAX_RECORD_BYTES, "publication resume record is too large" ); + #[cfg(not(windows))] let mut file = tempfile::NamedTempFile::new_in(&self.directory)?; + #[cfg(windows)] + let mut file = protected::temporary(&self.directory)?; + protected::verify_file(file.as_file())?; file.write_all(&bytes)?; file.as_file().sync_all()?; file.persist(self.directory.join("publication.json"))?; @@ -334,3 +362,16 @@ fn sync_directory_chain(directory: &Path) -> Result<()> { } Ok(()) } + +fn read_submission(directory: &Path) -> Result> { + protected::directory(directory)?; + let mut bytes = Vec::new(); + protected::file(&directory.join("submission.json"), false, false)? + .take(MAX_PUBLISH_REQUEST_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= MAX_PUBLISH_REQUEST_BYTES, + "protected publication body exceeds its bound" + ); + Ok(bytes) +} diff --git a/crates/cli/src/container/publish/journal/protected.rs b/crates/cli/src/container/publish/journal/protected.rs new file mode 100644 index 00000000000..4df45fe46d1 --- /dev/null +++ b/crates/cli/src/container/publish/journal/protected.rs @@ -0,0 +1,91 @@ +//! Files containing resolved values are opened without following links and +//! checked through the opened handle. Directory ownership is checked before +//! any access; progress rewrites inherit the same private directory policy. +#[cfg(unix)] +use anyhow::{ensure, Result}; +#[cfg(unix)] +use std::{ + fs::{self, File}, + path::Path, +}; + +#[cfg(unix)] +pub(super) fn create_directory(path: &Path) -> Result<()> { + use std::os::unix::fs::DirBuilderExt; + fs::DirBuilder::new().mode(0o700).create(path)?; + directory(path) +} + +#[cfg(unix)] +pub(super) fn directory(path: &Path) -> Result<()> { + let metadata = fs::symlink_metadata(path)?; + ensure!(metadata.is_dir(), "publication directory must be a private directory"); + private_metadata(&metadata) +} + +#[cfg(unix)] +fn private_metadata(metadata: &fs::Metadata) -> Result<()> { + use std::os::unix::fs::MetadataExt; + ensure!( + metadata.uid() == rustix::process::geteuid().as_raw() && metadata.mode() & 0o077 == 0, + "publication storage must be owned by the current user and inaccessible to other users" + ); + Ok(()) +} + +#[cfg(unix)] +pub(super) fn verify_file(file: &File) -> Result<()> { + use std::os::unix::fs::MetadataExt; + let metadata = file.metadata()?; + ensure!( + metadata.is_file() && metadata.nlink() == 1, + "publication storage must be a regular file without additional links" + ); + private_metadata(&metadata) +} + +#[cfg(unix)] +pub(super) fn file(path: &Path, create: bool, write: bool) -> Result { + use std::os::unix::fs::OpenOptionsExt; + let file = File::options() + .read(true) + .write(write) + .create_new(create) + .mode(0o600) + .custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32) + .open(path)?; + verify_file(&file)?; + Ok(file) +} + +/// Reject ancestors through which another user could replace a directory +/// component between checks and access. Root-owned sticky temporary roots are +/// safe: entries below them are themselves required to be root/current-owned. +#[cfg(unix)] +pub(super) fn pin_parents(path: &Path) -> Result> { + use std::os::unix::fs::MetadataExt; + let absolute = std::path::absolute(path)?; + for component in absolute.ancestors() { + let metadata = fs::symlink_metadata(component)?; + ensure!( + metadata.uid() == 0 || metadata.uid() == rustix::process::geteuid().as_raw(), + "publication path has an untrusted owner" + ); + if metadata.is_dir() { + ensure!( + metadata.mode() & 0o022 == 0 || (metadata.uid() == 0 && metadata.mode() & 0o1000 != 0), + "publication path has an untrusted writable ancestor" + ); + } else { + ensure!(metadata.is_symlink(), "invalid publication directory ancestor"); + // Check the resolved target as well as the named ancestor chain. + pin_parents(&component.canonicalize()?)?; + } + } + Ok(Vec::new()) +} + +#[cfg(windows)] +mod windows; +#[cfg(windows)] +pub(super) use windows::{create_directory, directory, file, pin_parents, temporary, verify_file}; diff --git a/crates/cli/src/container/publish/journal/protected/windows.rs b/crates/cli/src/container/publish/journal/protected/windows.rs new file mode 100644 index 00000000000..755f0db3ee7 --- /dev/null +++ b/crates/cli/src/container/publish/journal/protected/windows.rs @@ -0,0 +1,334 @@ +//! Owner-only DACLs are installed atomically at directory creation, without an +//! inheritable parent ACL. Files inherit only that owner ACE. Reopening checks +//! the actual handle's owner, DACL, file type and link count before reading. +use anyhow::{ensure, Result}; +use std::{ + ffi::c_void, + fs::File, + os::windows::{ + ffi::OsStrExt, + fs::OpenOptionsExt, + io::{AsRawHandle, FromRawHandle, OwnedHandle}, + }, + path::Path, + ptr::null_mut, +}; +use windows_sys::Win32::{ + Foundation::{LocalFree, GENERIC_READ, GENERIC_WRITE, INVALID_HANDLE_VALUE}, + Security::{ + Authorization::{ + ConvertSidToStringSidW, ConvertStringSecurityDescriptorToSecurityDescriptorW, GetSecurityInfo, + SDDL_REVISION_1, SE_FILE_OBJECT, + }, + EqualSid, GetAce, GetSecurityDescriptorControl, GetTokenInformation, TokenUser, ACCESS_ALLOWED_ACE, ACE_HEADER, + DACL_SECURITY_INFORMATION, INHERIT_ONLY_ACE, OWNER_SECURITY_INFORMATION, SECURITY_ATTRIBUTES, + SE_DACL_PROTECTED, TOKEN_QUERY, TOKEN_USER, + }, + Storage::FileSystem::{ + CreateDirectoryW, CreateFileW, GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, CREATE_NEW, + FILE_ALL_ACCESS, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, FILE_FLAG_BACKUP_SEMANTICS, + FILE_FLAG_OPEN_REPARSE_POINT, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, READ_CONTROL, + }, + System::{ + SystemServices::ACCESS_ALLOWED_ACE_TYPE, + Threading::{GetCurrentProcess, OpenProcessToken}, + }, +}; + +struct LocalAllocation(*mut c_void); +impl Drop for LocalAllocation { + fn drop(&mut self) { + // SAFETY: this pointer is allocated by a successful Windows security + // descriptor/SID conversion API and is freed exactly once. + unsafe { + LocalFree(self.0); + } + } +} + +fn check(ok: i32) -> Result<()> { + ensure!(ok != 0, "protected publication storage Windows operation failed"); + Ok(()) +} + +/// Aligned backing storage owns the TOKEN_USER and its SID together. +fn user() -> Result> { + unsafe { + let mut raw = null_mut(); + check(OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut raw))?; + let token = OwnedHandle::from_raw_handle(raw); + let mut length = 0; + GetTokenInformation(token.as_raw_handle(), TokenUser, null_mut(), 0, &mut length); + ensure!( + (size_of::()..=1024).contains(&(length as usize)), + "invalid Windows user identity size" + ); + let mut data = vec![0usize; (length as usize).div_ceil(size_of::())]; + check(GetTokenInformation( + token.as_raw_handle(), + TokenUser, + data.as_mut_ptr().cast(), + length, + &mut length, + ))?; + Ok(data) + } +} + +fn owner_descriptor() -> Result { + unsafe { + let user = user()?; + let sid = (*(user.as_ptr().cast::())).User.Sid; + let mut sid_string = null_mut(); + check(ConvertSidToStringSidW(sid, &mut sid_string))?; + let _sid_string = LocalAllocation(sid_string.cast()); + // Windows SID strings are bounded by the SID representation. Read only + // through the terminating NUL in the returned allocation. + let mut length = 0; + while *sid_string.add(length) != 0 { + length += 1; + } + let sid = String::from_utf16(std::slice::from_raw_parts(sid_string, length))?; + let sddl: Vec<_> = format!("O:{sid}D:P(A;OICI;FA;;;{sid})") + .encode_utf16() + .chain([0]) + .collect(); + let mut descriptor = null_mut(); + check(ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl.as_ptr(), + SDDL_REVISION_1, + &mut descriptor, + null_mut(), + ))?; + Ok(LocalAllocation(descriptor)) + } +} + +pub(in super::super) fn create_directory(path: &Path) -> Result<()> { + unsafe { + let descriptor = owner_descriptor()?; + let attributes = SECURITY_ATTRIBUTES { + nLength: size_of::() as u32, + lpSecurityDescriptor: descriptor.0, + bInheritHandle: 0, + }; + let path_wide: Vec<_> = path.as_os_str().encode_wide().chain([0]).collect(); + ensure!( + !path_wide[..path_wide.len() - 1].contains(&0), + "invalid publication path" + ); + check(CreateDirectoryW(path_wide.as_ptr(), &attributes))?; + } + directory(path) +} + +pub(in super::super) fn directory(path: &Path) -> Result<()> { + let file = File::options() + .access_mode(READ_CONTROL) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?; + verify(&file, true) +} + +pub(in super::super) fn file(path: &Path, create: bool, write: bool) -> Result { + let file = if create { + // Explicit owner is necessary even with a private parent: elevated + // Windows tokens can default newly created files to Administrators. + unsafe { + let descriptor = owner_descriptor()?; + let attributes = SECURITY_ATTRIBUTES { + nLength: size_of::() as u32, + lpSecurityDescriptor: descriptor.0, + bInheritHandle: 0, + }; + let path_wide: Vec<_> = path.as_os_str().encode_wide().chain([0]).collect(); + ensure!( + !path_wide[..path_wide.len() - 1].contains(&0), + "invalid publication path" + ); + let raw = CreateFileW( + path_wide.as_ptr(), + GENERIC_READ | if write { GENERIC_WRITE } else { 0 }, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + &attributes, + CREATE_NEW, + FILE_FLAG_OPEN_REPARSE_POINT, + null_mut(), + ); + ensure!(raw != INVALID_HANDLE_VALUE, "cannot create protected publication file"); + File::from_raw_handle(raw) + } + } else { + File::options() + .read(true) + .write(write) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)? + }; + verify_file(&file)?; + Ok(file) +} + +pub(in super::super) fn temporary(directory: &Path) -> Result { + let path = directory.join(format!(".publication-{}.tmp", uuid::Uuid::now_v7())); + let file = file(&path, true, true)?; + Ok(tempfile::NamedTempFile::from_parts( + file, + tempfile::TempPath::from_path(path), + )) +} + +pub(in super::super) fn verify_file(file: &File) -> Result<()> { + verify(file, false) +} + +fn verify(file: &File, directory: bool) -> Result<()> { + unsafe { + let mut info: BY_HANDLE_FILE_INFORMATION = std::mem::zeroed(); + check(GetFileInformationByHandle(file.as_raw_handle(), &mut info))?; + ensure!( + info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT == 0 + && (info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY != 0) == directory + && (directory || info.nNumberOfLinks == 1), + "publication storage must not contain reparse points or additional file links" + ); + let mut owner = null_mut(); + let mut dacl = null_mut(); + let mut descriptor = null_mut(); + ensure!( + GetSecurityInfo( + file.as_raw_handle(), + SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION, + &mut owner, + null_mut(), + &mut dacl, + null_mut(), + &mut descriptor + ) == 0, + "cannot verify protected publication storage permissions" + ); + let _descriptor = LocalAllocation(descriptor); + let user = user()?; + let sid = (*(user.as_ptr().cast::())).User.Sid; + ensure!( + !owner.is_null() && EqualSid(owner, sid) != 0 && !dacl.is_null() && (*dacl).AceCount == 1, + "publication storage must allow only its current owner" + ); + let mut ace = null_mut(); + check(GetAce(dacl, 0, &mut ace))?; + let header = &*ace.cast::(); + ensure!( + header.AceType as u32 == ACCESS_ALLOWED_ACE_TYPE + && header.AceSize as usize >= size_of::(), + "invalid publication access entry" + ); + let ace = &*ace.cast::(); + ensure!( + ace.Header.AceType as u32 == ACCESS_ALLOWED_ACE_TYPE + && ace.Header.AceFlags as u32 & INHERIT_ONLY_ACE == 0 + && ace.Mask == FILE_ALL_ACCESS + && EqualSid((&raw const ace.SidStart).cast_mut().cast(), sid) != 0, + "publication storage must allow only its current owner" + ); + if directory { + let mut control = 0; + let mut revision = 0; + check(GetSecurityDescriptorControl(descriptor, &mut control, &mut revision))?; + ensure!( + control & SE_DACL_PROTECTED != 0, + "publication directory must exclude inherited permissions" + ); + } + } + Ok(()) +} + +/// Deny DELETE sharing for every directory component for the entire journal +/// lifetime. Even a writable parent then cannot rename/replace an opened child +/// while secret files are accessed by path. Reparse ancestors are rejected. +pub(in super::super) fn pin_parents(path: &Path) -> Result> { + use windows_sys::Win32::Storage::FileSystem::FILE_READ_ATTRIBUTES; + let absolute = std::path::absolute(path)?; + let mut files = Vec::new(); + for path in absolute.ancestors().collect::>().into_iter().rev() { + let file = File::options() + .access_mode(FILE_READ_ATTRIBUTES) + .share_mode(FILE_SHARE_READ | FILE_SHARE_WRITE) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?; + unsafe { + let mut info: BY_HANDLE_FILE_INFORMATION = std::mem::zeroed(); + check(GetFileInformationByHandle(file.as_raw_handle(), &mut info))?; + ensure!( + info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT == 0 + && info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY != 0, + "publication directory ancestors must not be reparse points" + ); + } + files.push(file); + } + Ok(files) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn owner_only_directory_and_files_survive_reopen_and_reject_extra_links() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("publication"); + create_directory(&path).unwrap(); + let body = path.join("body"); + drop(file(&body, true, true).unwrap()); + drop(file(&body, false, false).unwrap()); + let progress = temporary(&path).unwrap(); + verify_file(progress.as_file()).unwrap(); + std::fs::hard_link(&body, path.join("extra-link")).unwrap(); + assert!(file(&body, false, false).is_err()); + } + + #[test] + fn reparse_points_are_rejected_before_reading() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("publication"); + create_directory(&path).unwrap(); + let body = path.join("body"); + drop(file(&body, true, true).unwrap()); + let link = path.join("link"); + std::os::windows::fs::symlink_file(&body, &link).unwrap(); + assert!(file(&link, false, false).is_err()); + let link = root.path().join("linked-directory"); + std::os::windows::fs::symlink_dir(&path, &link).unwrap(); + assert!(directory(&link).is_err()); + } + + #[test] + fn opening_a_file_with_a_permissive_dacl_fails_closed() { + use windows_sys::Win32::Security::{Authorization::SetNamedSecurityInfoW, PROTECTED_DACL_SECURITY_INFORMATION}; + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("publication"); + create_directory(&path).unwrap(); + let body = path.join("body"); + drop(file(&body, true, true).unwrap()); + let path_wide: Vec<_> = body.as_os_str().encode_wide().chain([0]).collect(); + // A NULL DACL grants everyone access. This is an owned empty fixture; + // no secret bytes are written before or after changing its ACL. + unsafe { + assert_eq!( + SetNamedSecurityInfoW( + path_wide.as_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION | PROTECTED_DACL_SECURITY_INFORMATION, + null_mut(), + null_mut(), + null_mut(), + null_mut() + ), + 0 + ); + } + assert!(file(&body, false, false).is_err()); + } +} diff --git a/crates/cli/src/container/publish/mod.rs b/crates/cli/src/container/publish/mod.rs index 35ac356322e..0cf686cec9b 100644 --- a/crates/cli/src/container/publish/mod.rs +++ b/crates/cli/src/container/publish/mod.rs @@ -42,6 +42,7 @@ pub async fn run( "resume server differs from the original publication endpoint" ); let artifact = client.artifact_endpoint(&journal.record.artifact_endpoint, approved_artifact)?; + journal.submission_bytes()?; let request = journal.record.request()?; let operation = request.manifest.current().envelope.operation_id; let permission = client.permission().await?; @@ -109,7 +110,7 @@ pub async fn run( journal.record.submitted = true; journal.save()?; let status = client - .submit_bytes(journal.record.database.unwrap(), journal.record.request_json.as_bytes()) + .submit_bytes(journal.record.database.unwrap(), &journal.submission_bytes()?) .await .context("publication outcome is not confirmed; resume the same directory")?; journal.record.check_status(&request, &status)?; @@ -134,7 +135,7 @@ async fn observe_or_replay( .is_some_and(|error| error.status == reqwest::StatusCode::FORBIDDEN) => { client - .submit_bytes(database, journal.record.request_json.as_bytes()) + .submit_bytes(database, &journal.submission_bytes()?) .await .map(Some) .context("exact publication replay was not confirmed; keep the same resume directory") diff --git a/crates/cli/src/container/publish/tests.rs b/crates/cli/src/container/publish/tests.rs index c2b616fb794..e5dab901ece 100644 --- a/crates/cli/src/container/publish/tests.rs +++ b/crates/cli/src/container/publish/tests.rs @@ -1,5 +1,6 @@ //! These fixtures own numeric-loopback sockets and in-memory credentials. They //! never load CLI configuration, environment endpoints, or external builders. +pub(crate) mod environment_tests; use super::*; use axum::{ body::Bytes, @@ -47,6 +48,7 @@ pub(crate) struct Behavior { pub bad_receipt: bool, pub bad_completion: bool, pub bad_status: bool, + pub malformed_status: bool, pub wrong_reservation: bool, pub deny_preflight: bool, pub deny_upload: bool, @@ -54,6 +56,9 @@ pub(crate) struct Behavior { pub fail_naming: bool, pub redirect: Option, pub prior: Option, + pub selected_schema: Option<(spacetimedb_lib::Hash, Vec)>, + pub wrong_module_identity: bool, + pub module_gets: usize, pub uploads: BTreeMap)>, pub submits: Vec>, pub reservations: Vec, @@ -107,6 +112,7 @@ impl Fixture { version: PUBLISH_PROTOCOL_VERSION, operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), expected_revision: None, + expected_last_operation: None, module_action: ModuleAction::Set(UserModule { kind: UserModuleKind::Wasm, program_hash: spacetimedb_lib::hash_bytes(bytes), @@ -115,6 +121,7 @@ impl Fixture { }; let module_artifact = empty_module_artifact(); let request = PublishRequest { + environment: Default::default(), manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { deployment: envelope.resolve(None, &Default::default()).unwrap(), envelope, @@ -131,7 +138,7 @@ impl Fixture { }; let request_json = serde_json::to_string(&request).unwrap(); Record { - version: 1, + version: 2, server: self.endpoint.clone(), artifact_endpoint: self.endpoint.clone(), publisher: publisher(), @@ -251,6 +258,27 @@ async fn handler( .map(|status| Json(status).into_response()) .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); } + if path.ends_with("/schema") && method == Method::GET { + state.module_gets += 1; + assert_eq!(uri.query(), Some("version=10")); + let Some((hash, bytes)) = state.selected_schema.clone() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let identity = if state.wrong_module_identity { + publisher() + } else { + database() + }; + return ( + [ + ("content-type", "application/json".to_owned()), + ("x-spacetimedb-module-hash", hash.to_string()), + ("x-spacetimedb-database-identity", identity.to_hex().to_string()), + ], + bytes, + ) + .into_response(); + } if path.ends_with("/deployment") { if method == Method::GET { return state @@ -278,6 +306,8 @@ async fn handler( operation_id: request.manifest.current().envelope.operation_id, phase: PublicationPhase::Complete, expected_revision: request.manifest.current().envelope.expected_revision, + expected_last_operation: request.manifest.current().envelope.expected_last_operation, + publication_epoch: 1, proposed_revision: if state.bad_status { spacetimedb_lib::Hash::ZERO } else { @@ -286,6 +316,11 @@ async fn handler( error: None, }; state.status = Some(status.clone()); + if state.malformed_status { + let mut value = serde_json::to_value(status).unwrap(); + value["phase"] = json!("secret-body-sentinel"); + return Json(value).into_response(); + } if std::mem::take(&mut state.lose_submit_after_commit) { return StatusCode::SERVICE_UNAVAILABLE.into_response(); } diff --git a/crates/cli/src/container/publish/tests/environment_tests.rs b/crates/cli/src/container/publish/tests/environment_tests.rs new file mode 100644 index 00000000000..ba0d335f5a1 --- /dev/null +++ b/crates/cli/src/container/publish/tests/environment_tests.rs @@ -0,0 +1,203 @@ +use super::*; + +fn set_environment(record: &mut Record, values: BTreeMap) { + let mut request = record.request().unwrap(); + request.environment = values; + record.request_json = serde_json::to_string(&request).unwrap(); + record.request_digest = spacetimedb_oci::sha256(record.request_json.as_bytes()); +} + +#[tokio::test] +async fn complete_large_environment_is_private_and_exact_after_lost_reply_and_revocation() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_after_commit = true; + let temporary = tempfile::tempdir().unwrap(); + let mut record = fixture.record(false, false); + set_environment( + &mut record, + (0..128) + .map(|n| (format!("KEY_{n}"), "secret-body-sentinel".repeat(256))) + .collect(), + ); + assert!(record.request_json.len() > 256 * 1024); + let exact = record.request_json.clone(); + let mut retained = journal(temporary.path(), record); + let path = retained.directory().to_owned(); + let metadata = std::fs::read_to_string(path.join("publication.json")).unwrap(); + assert!(!metadata.contains("secret-body-sentinel") && !metadata.contains("request_json")); + assert_eq!(std::fs::read(path.join("submission.json")).unwrap(), exact.as_bytes()); + assert!(run_now(&fixture.client(), &mut retained).await.is_err()); + drop(retained); + { + let mut state = fixture.state.lock().unwrap(); + state.deny_status = true; + state.permission = false; + state.deny_upload = true; + } + let mut retained = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut retained).await.unwrap(), + Outcome::Complete(_) + )); + assert_eq!( + fixture.state.lock().unwrap().submits, + [exact.as_bytes(), exact.as_bytes()] + ); + drop(retained); + fixture.close().await; +} + +#[tokio::test] +async fn missing_changed_and_malformed_secret_body_never_becomes_empty_or_leaks_in_errors() { + let fixture = Fixture::new().await; + for replacement in [ + None, + Some(b"{}".as_slice()), + Some(br#"{"environment":{"KEY":"secret-body-sentinel", "KEY":17}}"#.as_slice()), + ] { + let temporary = tempfile::tempdir().unwrap(); + let mut record = fixture.record(false, false); + set_environment(&mut record, BTreeMap::from([("KEY".into(), "original-secret".into())])); + let retained = journal(temporary.path(), record); + let path = retained.directory().to_owned(); + if let Some(bytes) = replacement { + std::fs::write(path.join("submission.json"), bytes).unwrap(); + } else { + std::fs::remove_file(path.join("submission.json")).unwrap(); + } + assert!(retained.submission_bytes().is_err()); + drop(retained); + let error = Journal::open(&path).err().unwrap(); + assert!(!format!("{error:#}").contains("secret-body-sentinel")); + } + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} + +#[tokio::test] +async fn status_must_match_previous_operation_and_the_first_confirmed_epoch() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let mut retained = journal(temporary.path(), fixture.record(false, false)); + run_now(&fixture.client(), &mut retained).await.unwrap(); + let request = retained.record.request().unwrap(); + for which in 0..3 { + let mut status = retained.record.status.clone().unwrap(); + match which { + 0 => status.expected_last_operation = Some(Uuid::from_u128(uuid::Uuid::now_v7().as_u128())), + 1 => status.publication_epoch = 0, + _ => status.publication_epoch += 1, + } + assert!(retained.record.check_status(&request, &status).is_err()); + } + fixture.close().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn protected_storage_rejects_public_modes_symlinks_and_hardlinks() { + use std::os::unix::fs::{symlink, PermissionsExt}; + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let retained = journal(temporary.path(), fixture.record(false, false)); + let path = retained.directory().to_owned(); + let body = path.join("submission.json"); + assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o700); + for name in ["submission.json", "publication.json", "publication.lock"] { + assert_eq!( + std::fs::metadata(path.join(name)).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + std::fs::set_permissions(&body, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(retained.submission_bytes().is_err()); + std::fs::set_permissions(&body, std::fs::Permissions::from_mode(0o600)).unwrap(); + let link = temporary.path().join("extra-link"); + std::fs::hard_link(&body, &link).unwrap(); + assert!(retained.submission_bytes().is_err()); + std::fs::remove_file(&link).unwrap(); + std::fs::rename(&body, &link).unwrap(); + symlink(&link, &body).unwrap(); + assert!(retained.submission_bytes().is_err()); + drop(retained); + assert!(Journal::open(&path).is_err()); + std::fs::remove_file(&body).unwrap(); + std::fs::rename(&link, &body).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert!(Journal::open(&path).is_err()); + fixture.close().await; +} + +pub(crate) fn schema_bytes(environment: spacetimedb_lib::environment::EnvironmentSchema) -> Vec { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; + serde_json::to_vec(&spacetimedb_lib::sats::serde::SerdeWrapper(RawModuleDefV10 { + sections: vec![ + RawModuleDefV10Section::Typespace(Default::default()), + RawModuleDefV10Section::Environment(environment.into_declarations()), + ], + })) + .unwrap() +} + +#[tokio::test] +async fn kept_schema_checks_identity_program_hash_and_bounded_v10_metadata() { + let fixture = Fixture::new().await; + let request = fixture.record(false, false).request().unwrap(); + let prior = DeploymentStatus { + database_identity: database(), + revision: Some(request.manifest.current().deployment.revision().unwrap()), + last_operation: Some(request.manifest.current().envelope.operation_id), + deployment: request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: empty_module_artifact().digest, + size_bytes: empty_module_artifact().size_bytes, + }, + }; + let hash = system_empty::empty().descriptor.program_hash; + fixture.state.lock().unwrap().selected_schema = Some((hash, schema_bytes(Default::default()))); + assert_eq!( + fixture.client().selected_environment(&prior).await.unwrap(), + Default::default() + ); + fixture.state.lock().unwrap().wrong_module_identity = true; + assert!(fixture.client().selected_environment(&prior).await.is_err()); + fixture.state.lock().unwrap().wrong_module_identity = false; + for (hash, bytes) in [ + (spacetimedb_lib::Hash::ZERO, schema_bytes(Default::default())), + (hash, b"invalid-private-schema-sentinel".to_vec()), + (hash, vec![0; 16 * 1024 * 1024 + 1]), + ] { + fixture.state.lock().unwrap().selected_schema = Some((hash, bytes)); + let error = fixture.client().selected_environment(&prior).await.unwrap_err(); + assert!(!format!("{error:#}").contains("invalid-private-schema-sentinel")); + } + fixture.close().await; +} + +#[tokio::test] +async fn malformed_server_response_cannot_disclose_echoed_environment_values() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().malformed_status = true; + let temporary = tempfile::tempdir().unwrap(); + let mut retained = journal(temporary.path(), fixture.record(false, false)); + let error = run_now(&fixture.client(), &mut retained).await.unwrap_err(); + assert!(!format!("{error:#}").contains("secret-body-sentinel")); + assert!(retained.record.submitted && retained.record.status.is_none()); + fixture.close().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn untrusted_writable_parent_is_rejected_before_retaining_secrets() { + use std::os::unix::fs::PermissionsExt; + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let base = temporary.path().join("untrusted"); + std::fs::create_dir(&base).unwrap(); + std::fs::set_permissions(&base, std::fs::Permissions::from_mode(0o777)).unwrap(); + let record = fixture.record(false, false); + assert!(Journal::create(&base, record, None, Some(system_empty::empty().bytes.as_ref())).is_err()); + assert_eq!(std::fs::read_dir(&base).unwrap().count(), 0); + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} diff --git a/crates/cli/src/subcommands/publish/managed.rs b/crates/cli/src/subcommands/publish/managed.rs index cecc9e065e3..f08a93d57ca 100644 --- a/crates/cli/src/subcommands/publish/managed.rs +++ b/crates/cli/src/subcommands/publish/managed.rs @@ -1,6 +1,6 @@ //! Managed publication frontend. The resume journal owns every byte needed to //! retry; project configuration and mutable image tags are only read initially. -use super::{confirm_major_version_upgrade, YesFlags}; +use super::{confirm_major_version_upgrade, environment, YesFlags}; use crate::{ common_args::ClearMode, config::Config, @@ -91,7 +91,8 @@ pub(super) fn add_args(mut command: Command) -> Command { Arg::new("publication_state_dir") .long("publication-state-dir") .value_parser(clap::value_parser!(PathBuf)) - .help("Private local directory retaining managed publication bytes and progress"), + .help("Private local directory retaining complete managed publication inputs and progress") + .long_help("Private local directory retaining complete managed publication inputs and progress. The protected submission file includes resolved environment values. Keep this directory private; do not commit it or share it. Resume reuses these exact values without rereading project configuration or shell variables."), ) .arg( Arg::new("resume_publication") @@ -433,6 +434,7 @@ async fn prepare_request( version: deployment::PUBLISH_PROTOCOL_VERSION, operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), expected_revision: prior.and_then(|p| p.revision), + expected_last_operation: prior.and_then(|p| p.last_operation), module_action, container_action, }; @@ -451,6 +453,18 @@ async fn prepare_request( } else { bail!("publication has no selected module artifact") }; + // Resolve a complete replacement without consulting the stored values. + // For Keep, authenticated schema metadata is tied to the observed program; + // a changed committed operation is rejected by the later pair CAS. + let schema = if let Some((kind, bytes)) = &module { + selected_environment(*kind, bytes).await? + } else { + client + .selected_environment(prior.context("publication has no selected module")?) + .await? + }; + let environment = environment::resolve(&schema, target.get_config_value("env"), |key| std::env::var_os(key))?; + print!("{}", environment.display()); let migration_policy = if let Some(prior) = prior { if let Some((kind, bytes)) = &module { migration(client, prior.database_identity, *kind, bytes, target, yes).await? @@ -490,6 +504,7 @@ async fn prepare_request( None }; let request = PublishRequest { + environment: environment.values, manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { envelope, deployment, @@ -543,7 +558,7 @@ async fn prepare_request( None }; let record = Record { - version: 1, + version: 2, server: client.server().to_string(), artifact_endpoint: artifact_endpoint.into(), publisher, @@ -564,6 +579,29 @@ async fn prepare_request( }; Journal::create(base, record, image, module.as_ref().map(|(_, bytes)| bytes.as_slice())) } +async fn selected_environment( + kind: UserModuleKind, + bytes: &[u8], +) -> Result { + // Canonical platform modules can be inspected without executing a helper. + // The verifier regenerates every byte; this is equally valid if a user + // explicitly selected those exact canonical Wasm bytes with --bin-path. + if kind == UserModuleKind::Wasm { + let descriptor = deployment::system_empty::SystemEmptyModule { + version: deployment::system_empty::VERSION, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }; + if let Ok(schema) = deployment::system_empty::verify(&descriptor, bytes) { + return Ok(schema); + } + } + let host_type = match kind { + UserModuleKind::Wasm => "Wasm", + UserModuleKind::Js => "Js", + }; + Ok(environment::inspect(bytes, host_type).await?.environment().clone()) +} + fn tools(args: &ArgMatches) -> Result { let mut tools = BuildTools { buildctl: args.get_one::("buildctl").unwrap().clone(), @@ -703,6 +741,9 @@ async fn execute(client: &PublisherClient, journal: &mut Journal, args: &ArgMatc Ok(()) } +#[cfg(test)] +mod environment_tests; + #[cfg(test)] mod tests { use super::*; @@ -786,6 +827,7 @@ mod tests { let prior_request = fixture.record(false, false).request().unwrap(); let prior = DeploymentStatus { database_identity: database(), + last_operation: Some(Uuid::from_u128(uuid::Uuid::now_v7().as_u128())), revision: Some(prior_request.manifest.current().deployment.revision().unwrap()), deployment: prior_request.manifest.current().deployment.clone(), module_artifact: ArtifactReference { @@ -853,6 +895,7 @@ mod tests { state.deny_preflight = true; state.prior = Some(DeploymentStatus { database_identity: database(), + last_operation: Some(Uuid::from_u128(uuid::Uuid::now_v7().as_u128())), revision: Some(request.manifest.current().deployment.revision().unwrap()), deployment: request.manifest.current().deployment.clone(), module_artifact: ArtifactReference { @@ -922,7 +965,13 @@ mod tests { let fixture = Fixture::new().await; fixture.state.lock().unwrap().lose_submit_before_commit = true; let temporary = tempfile::tempdir().unwrap(); - let record = fixture.record(false, false); + let mut record = fixture.record(false, false); + let mut request = record.request().unwrap(); + request + .environment + .insert("ORIGINAL_ENV".into(), "original-secret-value".into()); + record.request_json = serde_json::to_string(&request).unwrap(); + record.request_digest = spacetimedb_oci::sha256(record.request_json.as_bytes()); let exact = record.request_json.clone(); let mut journal = Journal::create( temporary.path(), @@ -946,7 +995,7 @@ mod tests { std::fs::write(&cli_config, "spacetimedb_token = 'isolated-fixture-credential'\n").unwrap(); let config = Config::load(spacetimedb_paths::cli::CliTomlPath::from_path_unchecked(cli_config)).unwrap(); let changed_project = crate::spacetime_config::LoadedConfig { - config: serde_json::from_value(json!({"database":"different-target", "module_path":"missing-build-source", "server":"must-never-resolve-this-alias"})).unwrap(), + config: serde_json::from_value(json!({"database":"different-target", "module_path":"missing-build-source", "server":"must-never-resolve-this-alias", "env":{"ORIGINAL_ENV":"changed-secret-value"}})).unwrap(), config_dir: temporary.path().into(), loaded_files: vec![], has_dev_file: false, }; let args = super::super::cli() diff --git a/crates/cli/src/subcommands/publish/managed/environment_tests.rs b/crates/cli/src/subcommands/publish/managed/environment_tests.rs new file mode 100644 index 00000000000..1bbad838f07 --- /dev/null +++ b/crates/cli/src/subcommands/publish/managed/environment_tests.rs @@ -0,0 +1,239 @@ +use super::*; +use crate::container::publish::tests::{database, publisher, Fixture}; +use serde_json::json; +use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; +use std::collections::{BTreeMap, HashMap}; + +const KEY: &str = "STDB_MANAGED_FIXTURE_REQUIRED"; + +fn declared_module() -> deployment::system_empty::GeneratedModule { + deployment::system_empty::generate( + &EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: KEY.into(), + constraint: EnvironmentConstraint::OneOf(vec!["first-secret".into(), "second-secret".into()]), + optional: false, + }]) + .unwrap(), + ) + .unwrap() +} + +#[tokio::test] +async fn keep_resolves_program_bound_declarations_and_retains_complete_input() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let module = declared_module(); + let deployment = deployment::DeploymentSpec::V1(deployment::DeploymentSpecV1 { + module: deployment::ModuleComponent::SystemEmpty(module.descriptor), + container: None, + }); + let prior = DeploymentStatus { + database_identity: database(), + revision: Some(deployment.revision().unwrap()), + deployment, + last_operation: Some(Uuid::from_u128(uuid::Uuid::now_v7().as_u128())), + module_artifact: ArtifactReference { + digest: spacetimedb_oci::sha256(&module.bytes), + size_bytes: module.bytes.len() as u64, + }, + }; + fixture.state.lock().unwrap().selected_schema = Some(( + module.descriptor.program_hash, + crate::container::publish::tests::environment_tests::schema_bytes( + deployment::system_empty::verify(&module.descriptor, &module.bytes).unwrap(), + ), + )); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command.try_get_matches_from(["publish", "--remove-container"]).unwrap(); + let target = CommandConfig::new( + &schema, + HashMap::from([("env".into(), json!({KEY:"first-secret"}))]), + &args, + ) + .unwrap(); + let journal = prepare_request( + &fixture.client(), + &target, + temporary.path(), + &temporary.path().join("state"), + Some(&prior), + None, + None, + publisher(), + &fixture.endpoint, + YesFlags::all(), + CancellationToken::new(), + ) + .await + .unwrap(); + let request = journal.record.request().unwrap(); + assert_eq!( + request.environment, + BTreeMap::from([(KEY.into(), "first-secret".into())]) + ); + assert_eq!( + request.manifest.current().envelope.expected_last_operation, + prior.last_operation + ); + assert_eq!(request.manifest.current().envelope.expected_revision, prior.revision); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Keep + )); + assert!(journal.record.uploads.is_empty()); + assert!(!std::fs::read_to_string(journal.directory().join("publication.json")) + .unwrap() + .contains("first-secret")); + let path = journal.directory().to_owned(); + let exact = journal.submission_bytes().unwrap(); + drop(journal); + // Neither the selected program endpoint nor mutable project input is read + // again after creation. The exact protected input drives recovery. + fixture.state.lock().unwrap().selected_schema = None; + std::fs::write( + temporary.path().join("spacetime.json"), + format!(r#"{{"env":{{"{KEY}":"second-secret"}}}}"#), + ) + .unwrap(); + let reopened = Journal::open(&path).unwrap(); + assert_eq!(reopened.submission_bytes().unwrap(), exact); + assert_eq!(fixture.state.lock().unwrap().module_gets, 1); + fixture.close().await; +} + +#[tokio::test] +async fn precompiled_exact_bytes_validate_required_invalid_unknown_and_explicit_values_before_mutation() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let module = declared_module(); + let wasm = temporary.path().join("exact.wasm"); + std::fs::write(&wasm, &module.bytes).unwrap(); + for (n, values, valid) in [ + (0, json!({}), false), + (1, json!({KEY:"secret-invalid-value"}), false), + (2, json!({KEY:"first-secret", "UNKNOWN":"secret-unknown-value"}), false), + (3, json!({KEY:"first-secret"}), true), + ] { + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from(["publish", "--bin-path", wasm.to_str().unwrap()]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::from([("env".into(), values)]), &args).unwrap(); + let result = prepare_request( + &fixture.client(), + &target, + temporary.path(), + &temporary.path().join(format!("state{n}")), + None, + None, + None, + publisher(), + &fixture.endpoint, + YesFlags::all(), + CancellationToken::new(), + ) + .await; + if valid { + let journal = result.unwrap(); + assert_eq!( + std::fs::read(journal.directory().join("module.blob")).unwrap(), + module.bytes.as_ref() + ); + assert_eq!(journal.record.request().unwrap().environment[KEY], "first-secret"); + } else { + let error = format!("{:#}", result.err().unwrap()); + assert!(!error.contains("secret-invalid-value") && !error.contains("secret-unknown-value")); + assert!(!temporary.path().join(format!("state{n}")).exists()); + } + } + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} + +#[tokio::test] +async fn fresh_process_resume_replays_original_values_after_shell_changes() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_before_commit = true; + let temporary = tempfile::tempdir().unwrap(); + let mut record = fixture.record(false, false); + let mut request = record.request().unwrap(); + request.environment.insert(KEY.into(), "first-secret".into()); + record.request_json = serde_json::to_string(&request).unwrap(); + record.request_digest = spacetimedb_oci::sha256(record.request_json.as_bytes()); + let exact = record.request_json.clone(); + let mut journal = Journal::create( + temporary.path(), + record, + None, + Some(deployment::system_empty::empty().bytes.as_ref()), + ) + .unwrap(); + assert!(publish::run( + &fixture.client(), + &mut journal, + None, + Duration::ZERO, + CancellationToken::new() + ) + .await + .is_err()); + let directory = journal.directory().to_owned(); + drop(journal); + let marker = temporary.path().join("child-finished"); + let mut child = tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--ignored", + "--exact", + "subcommands::publish::managed::environment_tests::resume_changed_shell_child", + ]) + .env_clear() + .env("STDB_MANAGED_FIXTURE_RESUME", &directory) + .env("STDB_MANAGED_FIXTURE_MARKER", &marker) + .env(KEY, "second-secret") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let result = tokio::time::timeout(Duration::from_secs(20), child.wait()).await; + let status = match result { + Ok(status) => status.unwrap(), + Err(error) => { + child.start_kill().unwrap(); + child.wait().await.unwrap(); + panic!("resume child exceeded its deadline: {error}"); + } + }; + assert!(status.success()); + assert_eq!(std::fs::read(&marker).unwrap(), b"one fixture completed"); + assert_eq!( + fixture.state.lock().unwrap().submits, + [exact.as_bytes(), exact.as_bytes()] + ); + fixture.close().await; +} + +#[tokio::test] +#[ignore = "invoked only by the owned fresh-process resume fixture"] +async fn resume_changed_shell_child() { + assert_eq!(std::env::var(KEY).unwrap(), "second-secret"); + let directory = PathBuf::from(std::env::var_os("STDB_MANAGED_FIXTURE_RESUME").unwrap()); + let marker = PathBuf::from(std::env::var_os("STDB_MANAGED_FIXTURE_MARKER").unwrap()); + assert!(directory.is_absolute() && marker.is_absolute()); + let mut journal = Journal::open(&directory).unwrap(); + assert!(journal.record.server.starts_with("http://127.0.0.1:")); + let client = PublisherClient::new( + &journal.record.server, + "Bearer isolated-fixture-credential".parse().unwrap(), + ) + .unwrap(); + publish::run(&client, &mut journal, None, Duration::ZERO, CancellationToken::new()) + .await + .unwrap(); + assert_eq!(journal.record.request().unwrap().environment[KEY], "first-secret"); + drop(journal); + std::fs::write(marker, b"one fixture completed").unwrap(); +} diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 412426aeaa5..46c386d225b 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -550,6 +550,13 @@ where Ok(( TypedHeader(SpacetimeIdentity(auth.claims.identity)), TypedHeader(SpacetimeIdentityToken(auth.creds)), + [ + ("x-spacetimedb-module-hash", module.info.module_hash.to_string()), + ( + "x-spacetimedb-database-identity", + module.info.database_identity.to_string(), + ), + ], response_json, )) } diff --git a/crates/core/src/db/container_environment.rs b/crates/core/src/db/container_environment.rs index def8e8569ed..fcf5c0f1fd0 100644 --- a/crates/core/src/db/container_environment.rs +++ b/crates/core/src/db/container_environment.rs @@ -34,7 +34,7 @@ pub enum EnvironmentSnapshotError { InvalidScope, #[error("container environment generation is fenced")] Fenced, - #[error("container environment deployment revision does not match")] + #[error("container environment committed publication does not match")] RevisionConflict, #[error("container environment snapshot belongs to another immutable scope")] ScopeConflict, @@ -116,6 +116,8 @@ fn validate_scope(db: &RelationalDB, scope: &EnvironmentSnapshotScope) -> Result || scope.database_id == 0 || scope.node_id == 0 || scope.generation == 0 + || scope.publication_epoch == 0 + || scope.publication_operation.get_version() != Some(spacetimedb_sats::uuid::Version::V7) || scope.cluster.is_empty() || scope.cluster.len() > 256 || scope.cluster.contains('\0') @@ -162,7 +164,13 @@ fn admitted_spec( let (revision, deployment) = deployment::current_deployment(state) .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? .ok_or(EnvironmentSnapshotError::RevisionConflict)?; - if revision != scope.deployment_revision { + let publication = deployment::current_publication(state) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if revision != scope.deployment_revision + || publication.operation_id != scope.publication_operation + || publication.publication_epoch != scope.publication_epoch + { return Err(EnvironmentSnapshotError::RevisionConflict); } let spec = deployment diff --git a/crates/core/src/db/container_environment/tests.rs b/crates/core/src/db/container_environment/tests.rs index e99e5719a68..ec300aecf80 100644 --- a/crates/core/src/db/container_environment/tests.rs +++ b/crates/core/src/db/container_environment/tests.rs @@ -60,6 +60,7 @@ fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { publication_epoch: 1, publisher: db.owner_identity(), expected_revision: None, + expected_last_operation: None, prepared_manifest_hash: hash_bytes(b"prepared"), deployment: DeploymentSpec::V1(DeploymentSpecV1 { module: ModuleComponent::SystemEmpty(generated.descriptor), @@ -87,6 +88,8 @@ fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { node_incarnation: uuid(), generation: 1, deployment_revision: request.deployment.revision().unwrap(), + publication_operation: request.operation_id, + publication_epoch: request.publication_epoch, start_request: request.operation_id, env_generation: uuid(), env_keys: spec.env_keys, @@ -473,3 +476,104 @@ fn container_environment_concurrent_closure_cannot_reopen_collected_generation() assert_eq!(state.table_row_count(ST_CONTAINER_ENVIRONMENT_ID), Some(0)) }); } + +#[test] +fn snapshots_bind_committed_operation_and_ignore_newer_aborted_attempt_epoch() { + use crate::db::deployment::{abort_deployment_commit, current_publication}; + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let original = setup(&db, vec!["A".into()]); + let attempt = db + .with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + let current = current_publication(tx)?.unwrap(); + let attempt = DeploymentCommit { + operation_id: uuid(), + publication_epoch: current.publication_epoch + 1, + publisher: db.owner_identity(), + expected_revision: Some(current.revision), + expected_last_operation: Some(current.operation_id), + prepared_manifest_hash: hash_bytes(b"aborted"), + deployment: deployment::current_deployment(tx)?.unwrap().1, + }; + install_publication_fence(tx, attempt.publication_epoch, attempt.operation_id)?; + abort_deployment_commit(tx, &attempt)?; + install_container_fence(&db, tx, &self_fence(&db, 2, true))?; + Ok(attempt) + }) + .unwrap(); + let scope = EnvironmentSnapshotScope { + generation: 2, + env_generation: uuid(), + ..original + }; + // Real commitlog replay retains the committed operation and an unrelated + // later closed attempt. A fresh generation still captures the old values. + let db = db.reopen().unwrap(); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let values = db + .with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap(); + assert_eq!(values.selected_values["A"], "before"); + let mut wrong = scope.clone(); + wrong.publication_epoch = attempt.publication_epoch; + assert_eq!( + tx(&db, |tx| capture(&db, tx, &wrong)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + wrong.publication_operation = attempt.operation_id; + assert_eq!( + tx(&db, |tx| capture(&db, tx, &wrong)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + + let accepted = db + .with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + let mut next = attempt.clone(); + next.operation_id = uuid(); + next.publication_epoch += 1; + install_publication_fence(tx, next.publication_epoch, next.operation_id)?; + environment::replace( + &db, + tx, + &environment_schema(&scope.env_keys), + &BTreeMap::from([("A".into(), "new-secret".into())]), + )?; + Ok(record_deployment_commit( + tx, + &next, + Timestamp::now(), + &Default::default(), + )?) + }) + .unwrap(); + assert_eq!(accepted.revision, scope.deployment_revision); + // Keep the generation fence unchanged here to prove the publication cursor + // independently blocks both old capture and old immutable snapshot reads. + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)), + Err(EnvironmentSnapshotError::RevisionConflict) + )); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 3, true))?; + close(&db, tx, &scope, 3)?; + Ok(()) + }) + .unwrap(); + let next_scope = EnvironmentSnapshotScope { + generation: 3, + env_generation: uuid(), + publication_operation: accepted.operation_id, + publication_epoch: accepted.publication_epoch, + ..scope + }; + let receipt = tx(&db, |tx| capture(&db, tx, &next_scope)).unwrap(); + assert_eq!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap() + .selected_values["A"], + "new-secret" + ); +} diff --git a/crates/core/src/db/deployment.rs b/crates/core/src/db/deployment.rs index 2b63611943d..f31753b62c5 100644 --- a/crates/core/src/db/deployment.rs +++ b/crates/core/src/db/deployment.rs @@ -30,7 +30,7 @@ pub enum DeploymentError { UnsupportedHostedModule, #[error("the publication coordinator no longer owns the database fence")] PublicationFenced, - #[error("the expected deployment revision does not match the database")] + #[error("the expected committed publication does not match the database")] RevisionConflict, #[error("operation ID is already bound to a different publication")] OperationConflict, @@ -58,6 +58,7 @@ pub struct DeploymentCommit { pub publication_epoch: u64, pub publisher: Identity, pub expected_revision: Option, + pub expected_last_operation: Option, pub prepared_manifest_hash: Hash, pub deployment: DeploymentSpec, } @@ -69,6 +70,7 @@ pub struct DeploymentCommit { pub struct PublishResult { #[serde(with = "spacetimedb_lib::deployment::uuid_json")] pub operation_id: Uuid, + pub publication_epoch: u64, pub previous_revision: Option, pub revision: Hash, } @@ -179,6 +181,56 @@ pub fn current_deployment(state: &S) -> Result(state: &S) -> Result, DeploymentError> { + let Some(row) = singleton(state, ST_DEPLOYMENT_ID)? else { + return Ok(None); + }; + let row = StDeploymentRow::try_from(row)?; + let spec = DeploymentSpec::decode(&row.payload)?; + let operation = Uuid::from_u128(row.last_operation_id); + let receipt = retained_publication(state, operation)?.ok_or(DeploymentError::CorruptMetadata)?; + if spec.revision()? != row.revision || receipt.result.revision != row.revision { + return Err(DeploymentError::CorruptMetadata); + } + Ok(Some(receipt.result)) +} + +fn require_expected_publication(state: &S, request: &DeploymentCommit) -> Result<(), DeploymentError> { + let current = current_publication(state)?; + if current.as_ref().map(|result| result.revision) != request.expected_revision + || current.as_ref().map(|result| result.operation_id) != request.expected_last_operation + { + return Err(DeploymentError::RevisionConflict); + } + Ok(()) +} + +fn retained_publication(state: &S, operation: Uuid) -> Result, DeploymentError> { + let operation_key = AlgebraicValue::U128(operation.as_u128().into()); + let Some(row) = state + .iter_by_col_eq(ST_DEPLOYMENT_OPERATION_ID, ColId(0), &operation_key)? + .next() + else { + return Ok(None); + }; + let row = StDeploymentOperationRow::try_from(row)?; + let CommitReceipt::V1(receipt) = + bsatn::from_slice(&row.commit_result).map_err(|_| DeploymentError::CorruptMetadata)?; + if operation.get_version() != Some(spacetimedb_sats::uuid::Version::V7) + || row.operation_id != operation.as_u128() + || receipt.result.operation_id != operation + || receipt.result.publication_epoch == 0 + || row.committed_revision != receipt.result.revision + || row.previous_revision != receipt.result.previous_revision + { + return Err(DeploymentError::CorruptMetadata); + } + Ok(Some(receipt)) +} + /// Monotonic compare-and-set, serialized against user-database commits. /// Advancing this epoch does not itself quiesce a container or authorize launch. pub fn install_publication_fence( @@ -235,9 +287,7 @@ pub fn abort_deployment_commit(tx: &mut MutTx, request: &DeploymentCommit) -> Re && (row.operation_id == request.operation_id.as_u128() || row.operation_id == 0) }) .ok_or(DeploymentError::PublicationFenced)?; - if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { - return Err(DeploymentError::RevisionConflict); - } + require_expected_publication(tx, request)?; if fence.operation_id == 0 { return Ok(AbortResult::Aborted { previous_revision: request.expected_revision, @@ -286,6 +336,7 @@ fn request_identity(request: &DeploymentCommit) -> Result<(Hash, Hash), Deployme request.publication_epoch, request.publisher, request.expected_revision, + request.expected_last_operation, request.prepared_manifest_hash, revision, )) @@ -316,9 +367,7 @@ pub fn check_deployment_commit( }) { return Err(DeploymentError::PublicationFenced); } - if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { - return Err(DeploymentError::RevisionConflict); - } + require_expected_publication(tx, request)?; Ok(CommitAdmission::Ready) } @@ -332,22 +381,11 @@ pub fn committed_deployment_operation( request: &DeploymentCommit, ) -> Result, DeploymentError> { let (revision, request_hash) = request_identity(request)?; - let operation_key = AlgebraicValue::U128(request.operation_id.as_u128().into()); - if let Some(row) = state - .iter_by_col_eq(ST_DEPLOYMENT_OPERATION_ID, ColId(0), &operation_key)? - .next() - { - let row = StDeploymentOperationRow::try_from(row)?; - let CommitReceipt::V1(receipt) = - bsatn::from_slice(&row.commit_result).map_err(|_| DeploymentError::CorruptMetadata)?; + if let Some(receipt) = retained_publication(state, request.operation_id)? { if receipt.request_hash != request_hash || receipt.publisher != request.publisher { return Err(DeploymentError::OperationConflict); } - if receipt.result.revision != revision - || row.committed_revision != revision - || row.previous_revision != receipt.result.previous_revision - || receipt.result.operation_id != request.operation_id - { + if receipt.result.revision != revision || receipt.result.publication_epoch != request.publication_epoch { return Err(DeploymentError::CorruptMetadata); } return Ok(Some(receipt.result)); @@ -369,9 +407,7 @@ pub fn deployment_publication_aborted( if !fence.is_some_and(|row| row.publication_epoch == request.publication_epoch && row.operation_id == 0) { return Ok(false); } - if current_deployment(state)?.map(|(revision, _)| revision) != request.expected_revision { - return Err(DeploymentError::RevisionConflict); - } + require_expected_publication(state, request)?; Ok(true) } @@ -390,6 +426,7 @@ pub fn record_deployment_commit( let (spec, revision, request_hash) = normalized_request(request, limits)?; let result = PublishResult { operation_id: request.operation_id, + publication_epoch: request.publication_epoch, previous_revision: request.expected_revision, revision, }; diff --git a/crates/core/src/db/deployment/tests.rs b/crates/core/src/db/deployment/tests.rs index 7a0bed25e14..e031319d575 100644 --- a/crates/core/src/db/deployment/tests.rs +++ b/crates/core/src/db/deployment/tests.rs @@ -9,12 +9,13 @@ use spacetimedb_lib::deployment::{ DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind, PUBLISH_RETRY_WINDOW_MS, }; -fn request(sequence: u64, previous: Option) -> DeploymentCommit { +fn request(sequence: u64, previous: Option<&PublishResult>) -> DeploymentCommit { DeploymentCommit { operation_id: Uuid::from_u128(0x01991ec4000070008000000000000000 | u128::from(sequence)), publication_epoch: sequence, publisher: Identity::from_u256(55u64.into()), - expected_revision: previous, + expected_revision: previous.map(|result| result.revision), + expected_last_operation: previous.map(|result| result.operation_id), prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), deployment: DeploymentSpec::V1(DeploymentSpecV1 { module: ModuleComponent::User(UserModule { @@ -47,7 +48,7 @@ fn deployment_retry_returns_original_result_after_later_publish_without_mutation record_deployment_commit(tx, &first, now(), &limits) }) .unwrap(); - let second = request(2, Some(accepted.revision)); + let second = request(2, Some(&accepted)); let later = transact(&db, |tx| { install_publication_fence(tx, second.publication_epoch, second.operation_id)?; record_deployment_commit(tx, &second, now(), &limits) @@ -66,7 +67,9 @@ fn deployment_retry_returns_original_result_after_later_publish_without_mutation fn deployment_fence_and_revision_conflicts_fail_before_module_execution() { let db = TestDB::in_memory().unwrap(); let first = request(1, None); - let mut second = request(2, Some(hash_bytes(b"not current"))); + let mut second = request(2, None); + second.expected_revision = Some(hash_bytes(b"not current")); + second.expected_last_operation = Some(first.operation_id); let limits = ContainerSpecLimits::default(); transact(&db, |tx| { install_publication_fence(tx, second.publication_epoch, second.operation_id) @@ -86,6 +89,7 @@ fn deployment_fence_and_revision_conflicts_fail_before_module_execution() { Err(DeploymentError::PublicationFenced) )); second.expected_revision = None; + second.expected_last_operation = None; assert!(matches!( check_deployment_commit(tx, &second, now(), &limits)?, CommitAdmission::Ready @@ -191,7 +195,7 @@ fn publication_commit_winning_abort_race_is_never_reported_as_aborted() { record_deployment_commit(tx, &first, now(), &limits) }) .unwrap(); - let second = request(2, Some(committed.revision)); + let second = request(2, Some(&committed)); transact(&db, |tx| { assert_eq!( abort_deployment_commit(tx, &first)?, @@ -249,13 +253,14 @@ fn recovery_of_expired_publication_preserves_commits_and_closes_uncommitted_epoc let expired = Timestamp::from_micros_since_unix_epoch( now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS * 1000) as i64, ); - let second = request(2, Some(committed.revision)); + let second = request(2, Some(&committed)); transact(&db, |tx| { assert!(matches!( check_deployment_commit(tx, &first, expired, &limits), Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) )); assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!(current_publication(tx)?, Some(committed.clone())); assert_eq!( abort_deployment_commit(tx, &first)?, AbortResult::AlreadyCommitted(committed.clone()) @@ -370,7 +375,7 @@ fn publication_recovery_does_not_reapply_tightened_resource_admission() { .unwrap(); let mut tightened = original_limits; tightened.resources.cpu_millicores = 500; - let mut second = request(2, Some(committed.revision)); + let mut second = request(2, Some(&committed)); second.deployment = first.deployment.clone(); transact(&db, |tx| { assert!(first.deployment.clone().normalize(&tightened).is_err()); @@ -631,3 +636,156 @@ fn container_fence_installation_serializes_with_admitted_transactions() { }) .unwrap(); } + +#[test] +fn same_revision_environment_publications_compare_the_last_committed_operation() { + use crate::db::environment; + use spacetimedb_lib::environment::{EnvironmentConstraint, EnvironmentDeclaration, EnvironmentSchema}; + use std::collections::BTreeMap; + use std::sync::{Arc, Barrier}; + + let db = TestDB::in_memory().unwrap(); + let limits = ContainerSpecLimits::default(); + let first = request(1, None); + let schema = EnvironmentSchema::new(vec![EnvironmentDeclaration { + name: "TOKEN".into(), + constraint: EnvironmentConstraint::AnyString, + optional: false, + }]) + .unwrap(); + let accepted = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + environment::replace(&db, tx, &schema, &BTreeMap::from([("TOKEN".into(), "initial".into())])) + .map_err(|error| DeploymentError::Database(DBError::Other(error.into())))?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let mut second = request(2, Some(&accepted)); + second.deployment = first.deployment.clone(); + let mut third = request(3, Some(&accepted)); + third.deployment = first.deployment.clone(); + let barrier = Arc::new(Barrier::new(3)); + let writers: Vec<_> = [(second.clone(), "second"), (third.clone(), "third")] + .into_iter() + .map(|(request, value)| { + let db = db.db.clone(); + let barrier = barrier.clone(); + let schema = schema.clone(); + std::thread::spawn(move || { + barrier.wait(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + check_deployment_commit(tx, &request, now(), &Default::default())?; + environment::replace(&db, tx, &schema, &BTreeMap::from([("TOKEN".into(), value.into())])) + .map_err(|error| DeploymentError::Database(DBError::Other(error.into())))?; + record_deployment_commit(tx, &request, now(), &Default::default()) + }) + }) + }) + .collect(); + barrier.wait(); + // Join both physical writers before asserting, including when one failed. + let outcomes: Vec<_> = writers.into_iter().map(|writer| writer.join()).collect(); + let outcomes: Vec<_> = outcomes.into_iter().map(Result::unwrap).collect(); + assert_eq!(outcomes.iter().filter(|result| result.is_ok()).count(), 1); + let winner = outcomes.into_iter().find_map(Result::ok).unwrap(); + assert_eq!(winner.revision, accepted.revision); + assert_ne!(winner.operation_id, accepted.operation_id); + transact(&db, |tx| { + assert_eq!(current_publication(tx)?, Some(winner.clone())); + let expected = if winner.operation_id == second.operation_id { + "second" + } else { + "third" + }; + assert_eq!( + environment::snapshot(tx).map_err(|error| DeploymentError::Database(DBError::Other(error.into())))? + ["TOKEN"], + expected + ); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(2)); + // A later attempt has the current revision but an obsolete operation. + let mut stale = request(4, Some(&accepted)); + stale.deployment = first.deployment.clone(); + install_publication_fence(tx, stale.publication_epoch, stale.operation_id)?; + assert!(matches!( + check_deployment_commit(tx, &stale, now(), &limits), + Err(DeploymentError::RevisionConflict) + )); + assert!(matches!( + abort_deployment_commit(tx, &stale), + Err(DeploymentError::RevisionConflict) + )); + // The exact current pair can close this attempt; the old pair cannot + // interpret its closed marker as proof that its precondition survived. + let mut current = stale.clone(); + current.expected_last_operation = Some(winner.operation_id); + abort_deployment_commit(tx, ¤t)?; + assert!(deployment_publication_aborted(tx, ¤t)?); + assert!(matches!( + deployment_publication_aborted(tx, &stale), + Err(DeploymentError::RevisionConflict) + )); + assert_eq!(current_publication(tx)?, Some(winner)); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn retained_receipt_binds_prior_operation_epoch_and_current_metadata() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let accepted = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &Default::default()) + }) + .unwrap(); + transact(&db, |tx| { + let mut changed = first.clone(); + changed.expected_last_operation = Some(request(9, None).operation_id); + assert!(matches!( + committed_deployment_operation(tx, &changed), + Err(DeploymentError::OperationConflict) + )); + changed = first.clone(); + changed.publication_epoch += 1; + assert!(matches!( + committed_deployment_operation(tx, &changed), + Err(DeploymentError::OperationConflict) + )); + assert_eq!(current_publication(tx)?, Some(accepted.clone())); + Ok(()) + }) + .unwrap(); + // A corrupted result epoch must fail even when its request hash still + // matches. Do not derive a replacement epoch from the publication fence. + transact(&db, |tx| { + let row = tx.iter(ST_DEPLOYMENT_OPERATION_ID)?.next().unwrap(); + let mut row = StDeploymentOperationRow::try_from(row)?; + let CommitReceipt::V1(mut receipt) = bsatn::from_slice(&row.commit_result).unwrap(); + receipt.result.publication_epoch = 0; + row.commit_result = bsatn::to_vec(&CommitReceipt::V1(receipt)).unwrap().into(); + tx.clear_table(ST_DEPLOYMENT_OPERATION_ID)?; + tx.insert_via_serialize_bsatn(ST_DEPLOYMENT_OPERATION_ID, &row)?; + assert!(matches!(current_publication(tx), Err(DeploymentError::CorruptMetadata))); + assert!(matches!( + committed_deployment_operation(tx, &first), + Err(DeploymentError::CorruptMetadata) + )); + tx.clear_table(ST_DEPLOYMENT_OPERATION_ID)?; + assert!(matches!(current_publication(tx), Err(DeploymentError::CorruptMetadata))); + let next = request(2, Some(&accepted)); + install_publication_fence(tx, next.publication_epoch, next.operation_id)?; + assert!(matches!( + check_deployment_commit(tx, &next, now(), &Default::default()), + Err(DeploymentError::CorruptMetadata) + )); + assert!(matches!( + abort_deployment_commit(tx, &next), + Err(DeploymentError::CorruptMetadata) + )); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/core/src/host/container_environment/durability_tests.rs b/crates/core/src/host/container_environment/durability_tests.rs index 104f4cf09bf..e2e533c78db 100644 --- a/crates/core/src/host/container_environment/durability_tests.rs +++ b/crates/core/src/host/container_environment/durability_tests.rs @@ -66,6 +66,7 @@ fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { publication_epoch: 1, publisher: db.owner_identity(), expected_revision: None, + expected_last_operation: None, prepared_manifest_hash: hash_bytes(b"prepared"), deployment: DeploymentSpec::V1(DeploymentSpecV1 { module: ModuleComponent::SystemEmpty(generated.descriptor), @@ -93,6 +94,8 @@ fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { node_incarnation: uuid(), generation: 1, deployment_revision: request.deployment.revision().unwrap(), + publication_operation: request.operation_id, + publication_epoch: request.publication_epoch, start_request: request.operation_id, env_generation: uuid(), env_keys: spec.env_keys, diff --git a/crates/core/src/host/host_controller/deployment_tests.rs b/crates/core/src/host/host_controller/deployment_tests.rs index efd35023d8e..f853edf9056 100644 --- a/crates/core/src/host/host_controller/deployment_tests.rs +++ b/crates/core/src/host/host_controller/deployment_tests.rs @@ -37,6 +37,7 @@ fn request(program: &Program, sequence: u64, initial: bool) -> DeploymentCommit publication_epoch: sequence, publisher: Identity::ONE, expected_revision: None, + expected_last_operation: None, prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), deployment: DeploymentSpec::V1(DeploymentSpecV1 { module: if initial { @@ -335,6 +336,8 @@ async fn declared_builtin_publishes_complete_environment_atomically_and_reopens( module.relational_db().with_read_only(Workload::Internal, |tx| { assert_eq!(db::environment::snapshot(tx).unwrap(), *expected); assert_eq!(current_deployment(tx).unwrap().unwrap().0, revision); + let cursor = db::deployment::current_publication(tx).unwrap().unwrap(); + assert_eq!(cursor.publication_epoch, receipts); assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(receipts)); }); }; @@ -344,13 +347,15 @@ async fn declared_builtin_publishes_complete_environment_atomically_and_reopens( ("MODE".into(), "paused".into()), ("FIXED".into(), "constant".into()), ]); - let publish_request = |sequence| { + let publish_request = |sequence, previous_operation| { let mut publication = request(&program, sequence, true); publication.deployment = initial.deployment.clone(); publication.expected_revision = Some(revision); + publication.expected_last_operation = Some(previous_operation); publication }; - let publication = publish_request(2); + let publication = publish_request(2, initial.operation_id); + let accepted_operation = publication.operation_id; module .relational_db() .with_auto_commit(Workload::Internal, |tx| { @@ -381,6 +386,29 @@ async fn declared_builtin_publishes_complete_environment_atomically_and_reopens( let module = controller.get_module_host(database.id).await.unwrap(); assert_state(&module, &replacement, 2); + // The same normalized deployment revision does not authorize replacing + // ENV prepared against the earlier committed operation. + let stale = publish_request(3, initial.operation_id); + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, stale.publication_epoch, stale.operation_id) + }) + .unwrap(); + assert!(controller + .update_module_host_with_environment_and_deployment( + database.clone(), + HostType::Wasm, + database.id, + program.bytes.clone(), + MigrationPolicy::Compatible, + initial_values.clone(), + Some(stale), + ) + .await + .is_err()); + assert_state(&controller.get_module_host(database.id).await.unwrap(), &replacement, 2); + let mut invalid_sets = Vec::new(); let mut invalid = replacement.clone(); invalid.remove("REQUIRED"); @@ -391,7 +419,7 @@ async fn declared_builtin_publishes_complete_environment_atomically_and_reopens( invalid_sets.push(invalid); } for (index, invalid) in invalid_sets.into_iter().enumerate() { - let publication = publish_request(3 + index as u64); + let publication = publish_request(4 + index as u64, accepted_operation); module .relational_db() .with_auto_commit(Workload::Internal, |tx| { @@ -417,7 +445,7 @@ async fn declared_builtin_publishes_complete_environment_atomically_and_reopens( // Wasm custom section would leave the extracted declarations unchanged. let mut different_bytes = program.bytes.to_vec(); different_bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); - let publication = publish_request(7); + let publication = publish_request(8, accepted_operation); module .relational_db() .with_auto_commit(Workload::Internal, |tx| { diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index de3ab21159a..13d69140356 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -16,7 +16,7 @@ required-features = ["serde"] [features] default = ["serde", "metrics_impls"] -serde = ["dep:serde", "spacetimedb-sats/serde"] +serde = ["dep:serde", "dep:serde_json", "spacetimedb-sats/serde"] # Allows using `Arbitrary` impls defined in this crate. proptest = ["dep:proptest", "dep:proptest-derive"] # Allows using additional test methods. @@ -42,6 +42,7 @@ thiserror.workspace = true derive_more.workspace = true hex.workspace = true serde = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } blake3.workspace = true enum-map = { workspace = true, optional = true } diff --git a/crates/lib/src/container_environment.rs b/crates/lib/src/container_environment.rs index f2ff648a45e..e33073c2cba 100644 --- a/crates/lib/src/container_environment.rs +++ b/crates/lib/src/container_environment.rs @@ -20,6 +20,9 @@ pub struct EnvironmentSnapshotScope { pub generation: u64, pub deployment_revision: Hash, #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub publication_operation: Uuid, + pub publication_epoch: u64, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] pub start_request: Uuid, #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] pub env_generation: Uuid, diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs index 16b09761845..d5e77ad28d0 100644 --- a/crates/lib/src/deployment.rs +++ b/crates/lib/src/deployment.rs @@ -104,6 +104,10 @@ pub struct PublishEnvelope { /// Exact compare-and-set precondition. None means no deployment record has /// been installed yet, rather than permission to overwrite any revision. pub expected_revision: Option, + /// Last committed operation paired with the revision; ENV-only publications + /// can retain the same revision while changing this cursor. + #[cfg_attr(feature = "serde", serde(with = "option_uuid_json"))] + pub expected_last_operation: Option, #[cfg_attr(feature = "serde", serde(default))] pub module_action: ModuleAction, #[cfg_attr(feature = "serde", serde(default))] @@ -187,6 +191,15 @@ impl PublishEnvelope { if !matches!(self.operation_id.get_version(), Some(Version::V7)) { return Err(DeploymentValidationError::InvalidOperationId); } + if self.expected_revision.is_some() != self.expected_last_operation.is_some() { + return Err(DeploymentValidationError::InvalidEncoding); + } + if self + .expected_last_operation + .is_some_and(|id| id.get_version() != Some(Version::V7)) + { + return Err(DeploymentValidationError::InvalidOperationId); + } let prior = previous.map(DeploymentSpec::current); let module = match &self.module_action { ModuleAction::Keep => prior @@ -238,5 +251,22 @@ pub mod uuid_json { } } +#[cfg(feature = "serde")] +pub mod option_uuid_json { + use super::Uuid; + pub fn serialize(id: &Option, serializer: S) -> Result { + match id { + Some(id) => serializer.serialize_some(&id.to_string()), + None => serializer.serialize_none(), + } + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let value = as serde::Deserialize>::deserialize(deserializer)?; + value + .map(|value| Uuid::parse_str(&value).map_err(serde::de::Error::custom)) + .transpose() + } +} + #[cfg(test)] mod tests; diff --git a/crates/lib/src/deployment/api.rs b/crates/lib/src/deployment/api.rs index c1adcebef6e..f69054661bd 100644 --- a/crates/lib/src/deployment/api.rs +++ b/crates/lib/src/deployment/api.rs @@ -1,9 +1,16 @@ //! HTTP publication messages shared by the CLI, dashboard and Cloud. Artifacts -//! are uploaded separately; no message carries credentials or environment values. +//! are uploaded separately. Complete environment values belong only to the +//! protected publication request; public status and manifests remain value-free. -use super::{manifest::PreparedDeploymentManifest, uuid_json, DeploymentSpec, PUBLISH_PROTOCOL_VERSION}; +use super::{ + manifest::PreparedDeploymentManifest, option_uuid_json, uuid_json, DeploymentSpec, PUBLISH_PROTOCOL_VERSION, +}; use crate::{container::OciDigest, Hash, Identity, Uuid}; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +mod request_decode; +pub use request_decode::{PublishRequestError, MAX_PUBLISH_REQUEST_BYTES}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -12,7 +19,7 @@ pub struct ArtifactReference { pub size_bytes: u64, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct PublishRequest { pub manifest: PreparedDeploymentManifest, @@ -21,6 +28,20 @@ pub struct PublishRequest { /// Original uploaded OCI index or executable manifest. Required for Set. /// Keep uses the prior retained executable manifest; Remove has no image. pub image_source: Option, + /// Complete private values. Omission means an empty replacement, never Keep. + #[serde(default, deserialize_with = "request_decode::deserialize_environment")] + pub environment: BTreeMap, +} + +impl std::fmt::Debug for PublishRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PublishRequest") + .field("manifest", &self.manifest) + .field("creation", &self.creation) + .field("image_source", &self.image_source) + .field("environment", &"[redacted]") + .finish() + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -75,6 +96,9 @@ pub struct PublicationStatus { pub operation_id: Uuid, pub phase: PublicationPhase, pub expected_revision: Option, + #[serde(with = "option_uuid_json")] + pub expected_last_operation: Option, + pub publication_epoch: u64, pub proposed_revision: Hash, pub error: Option, } @@ -85,6 +109,8 @@ pub struct DeploymentStatus { pub database_identity: Identity, /// None means this database has not yet used managed publication. pub revision: Option, + #[serde(with = "option_uuid_json")] + pub last_operation: Option, pub deployment: DeploymentSpec, /// Lets Keep select exactly the currently installed program bytes. pub module_artifact: ArtifactReference, diff --git a/crates/lib/src/deployment/api/request_decode.rs b/crates/lib/src/deployment/api/request_decode.rs new file mode 100644 index 00000000000..9bb6f947841 --- /dev/null +++ b/crates/lib/src/deployment/api/request_decode.rs @@ -0,0 +1,86 @@ +use super::PublishRequest; +use crate::deployment::MAX_DEPLOYMENT_BYTES; +use crate::environment::{validate_key, validate_value, MAX_ENV_KEY_BYTES, MAX_ENV_VALUE_BYTES, MAX_ENV_VARS}; +use serde::de::{MapAccess, Visitor}; +use std::{collections::BTreeMap, fmt}; + +/// Conservative JSON escaping allowance, independent of decoded metadata/map +/// limits. Checked constant arithmetic fails compilation if limits overflow. +pub const MAX_PUBLISH_REQUEST_BYTES: usize = { + let metadata = MAX_DEPLOYMENT_BYTES.checked_mul(6).expect("metadata JSON bound"); + let entry = MAX_ENV_KEY_BYTES + .checked_add(MAX_ENV_VALUE_BYTES) + .expect("environment entry bytes") + .checked_mul(6) + .expect("environment JSON escaping") + .checked_add(8) + .expect("environment JSON framing"); + metadata + .checked_add(MAX_ENV_VARS.checked_mul(entry).expect("environment JSON bound")) + .expect("publication JSON sections") + .checked_add(4096) + .expect("publication JSON framing") +}; + +/// Never retains a value-bearing parser error, source error, input or digest. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid publication request")] +pub struct PublishRequestError; + +impl PublishRequest { + /// Decode protected HTTP/journal input. Routes must also bound streamed + /// bytes and concurrent body ownership before buffering reaches this method. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_PUBLISH_REQUEST_BYTES { + return Err(PublishRequestError); + } + let request: Self = serde_json::from_slice(bytes).map_err(|_| PublishRequestError)?; + request.validate_structure()?; + Ok(request) + } + + /// Structural limits also apply to callers constructing a typed request. + /// Configured resource policy and selected-module declarations are checked + /// at publication admission, not inferred from environment key selection. + pub fn validate_structure(&self) -> Result<(), PublishRequestError> { + self.manifest.encode().map_err(|_| PublishRequestError)?; + if self.environment.len() > MAX_ENV_VARS { + return Err(PublishRequestError); + } + for (key, value) in &self.environment { + validate_key(key).map_err(|_| PublishRequestError)?; + validate_value(value).map_err(|_| PublishRequestError)?; + } + Ok(()) + } +} + +pub(super) fn deserialize_environment<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + struct EnvironmentVisitor; + impl<'de> Visitor<'de> for EnvironmentVisitor { + type Value = BTreeMap; + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a bounded publication environment object") + } + fn visit_map>(self, mut input: M) -> Result { + let mut values = BTreeMap::new(); + while let Some(key) = input.next_key::()? { + if values.len() >= MAX_ENV_VARS || validate_key(&key).is_err() || values.contains_key(&key) { + return Err(serde::de::Error::custom("invalid publication environment")); + } + let value = input.next_value::()?; + if validate_value(&value).is_err() { + return Err(serde::de::Error::custom("invalid publication environment")); + } + values.insert(key, value); + } + Ok(values) + } + } + deserializer.deserialize_map(EnvironmentVisitor) +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/deployment/api/request_decode/tests.rs b/crates/lib/src/deployment/api/request_decode/tests.rs new file mode 100644 index 00000000000..16b4c8b2e5d --- /dev/null +++ b/crates/lib/src/deployment/api/request_decode/tests.rs @@ -0,0 +1,166 @@ +use super::*; +use crate::container::{ContainerAction, OciDigest}; +use crate::deployment::manifest::{ + ModuleArtifact, PreparedDeploymentManifest, PreparedDeploymentManifestV1, PreparedMigrationPolicy, +}; +use crate::deployment::{ + DeploymentSpec, DeploymentSpecV1, ModuleAction, ModuleComponent, PublishEnvelope, PUBLISH_PROTOCOL_VERSION, +}; +use crate::Uuid; + +fn request() -> PublishRequest { + PublishRequest { + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope: PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::parse_str("01991ec4-0000-7000-8000-000000000001").unwrap(), + expected_revision: None, + expected_last_operation: None, + module_action: ModuleAction::Keep, + container_action: ContainerAction::Keep, + }, + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(crate::deployment::system_empty::empty().descriptor), + container: None, + }), + module_artifact: ModuleArtifact { + digest: OciDigest::sha256([17; 32]), + size_bytes: 250, + }, + migration_policy: PreparedMigrationPolicy::Compatible, + }), + creation: None, + image_source: None, + environment: BTreeMap::new(), + } +} + +fn with_raw_environment(raw: &str) -> Vec { + let mut value = serde_json::to_value(request()).unwrap(); + value.as_object_mut().unwrap().remove("environment"); + let mut json = serde_json::to_string(&value).unwrap(); + json.pop(); + format!("{json},\"environment\":{raw}}}").into_bytes() +} + +#[test] +fn protected_environment_decode_is_complete_redacted_and_preserves_strings() { + let mut original = request(); + original.environment = BTreeMap::from([ + ("TOKEN".into(), "private-secret-sentinel".into()), + ("EMPTY".into(), String::new()), + ("NUL".into(), "a\0b".into()), + ("UTF8".into(), "日本語".into()), + ]); + let bytes = serde_json::to_vec(&original).unwrap(); + assert_eq!(PublishRequest::decode(&bytes).unwrap(), original); + let debug = format!("{original:?}"); + assert!(debug.contains("[redacted]")); + for (key, value) in &original.environment { + assert!(!debug.contains(key)); + if !value.is_empty() { + assert!(!debug.contains(value)); + } + } + let mut without = serde_json::to_value(original).unwrap(); + without.as_object_mut().unwrap().remove("environment"); + assert!(PublishRequest::decode(&serde_json::to_vec(&without).unwrap()) + .unwrap() + .environment + .is_empty()); + assert!(PublishRequest::decode(&with_raw_environment("{}")) + .unwrap() + .environment + .is_empty()); +} + +#[test] +fn duplicate_malformed_and_invalid_environment_errors_never_echo_input() { + for raw in [ + r#"{"TOKEN":"private-secret-sentinel","TOKEN":"other"}"#, + r#"{"TOKEN":"private-secret-sentinel","\u0054OKEN":"other"}"#, + r#"{"TOKEN":{"private-secret-sentinel":true}}"#, + r#"{"private-secret-sentinel":"value"}"#, + r#"{"TOKEN":123}"#, + r#"{"TOKEN":"private-secret-sentinel""#, + "null", + "[]", + ] { + let error = PublishRequest::decode(&with_raw_environment(raw)).unwrap_err(); + assert_eq!(error.to_string(), "invalid publication request"); + assert_eq!(format!("{error:?}"), "PublishRequestError"); + } +} + +#[test] +fn environment_and_raw_byte_limits_are_independent_of_manifest_size() { + let mut original = request(); + for index in 0..MAX_ENV_VARS { + let key = format!("K{index:03}{}", "K".repeat(MAX_ENV_KEY_BYTES - 4)); + original.environment.insert(key, "\0".repeat(MAX_ENV_VALUE_BYTES)); + } + let bytes = serde_json::to_vec(&original).unwrap(); + assert!(bytes.len() <= MAX_PUBLISH_REQUEST_BYTES); + assert_eq!(PublishRequest::decode(&bytes).unwrap(), original); + original.environment.insert("EXTRA".into(), String::new()); + assert!(PublishRequest::decode(&serde_json::to_vec(&original).unwrap()).is_err()); + original.environment.clear(); + for (key, value) in [ + ("K".repeat(MAX_ENV_KEY_BYTES + 1), String::new()), + ("K".into(), "é".repeat(MAX_ENV_VALUE_BYTES / 2 + 1)), + ] { + original.environment = BTreeMap::from([(key, value)]); + assert!(original.validate_structure().is_err()); + assert!(PublishRequest::decode(&serde_json::to_vec(&original).unwrap()).is_err()); + } + assert_eq!( + PublishRequest::decode(&vec![b' '; MAX_PUBLISH_REQUEST_BYTES + 1]), + Err(PublishRequestError) + ); + original.environment.clear(); + let PreparedDeploymentManifest::V1(manifest) = &mut original.manifest; + let DeploymentSpec::V1(deployment) = &mut manifest.deployment; + // Metadata retains its own byte cap even with an empty environment. + use crate::container::*; + let container = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["x".repeat(MAX_DEPLOYMENT_BYTES + 1)], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + }; + deployment.container = Some(container); + assert!(PublishRequest::decode(&serde_json::to_vec(&original).unwrap()).is_err()); +} + +#[test] +fn optional_publication_uuid_is_a_string_or_null_and_not_an_integer() { + let mut original = request(); + let PreparedDeploymentManifest::V1(manifest) = &mut original.manifest; + manifest.envelope.expected_last_operation = Some(manifest.envelope.operation_id); + let value = serde_json::to_value(&original).unwrap(); + let cursor = &value["manifest"]["manifest"]["envelope"]["expected_last_operation"]; + assert_eq!(cursor.as_str(), Some("01991ec4-0000-7000-8000-000000000001")); + assert_eq!( + PublishRequest::decode(&serde_json::to_vec(&value).unwrap()).unwrap(), + original + ); + let mut value = value; + value["manifest"]["manifest"]["envelope"]["expected_last_operation"] = 17.into(); + assert!(PublishRequest::decode(&serde_json::to_vec(&value).unwrap()).is_err()); +} diff --git a/crates/lib/src/deployment/manifest.rs b/crates/lib/src/deployment/manifest.rs index d4d7f03c846..683599fd9ab 100644 --- a/crates/lib/src/deployment/manifest.rs +++ b/crates/lib/src/deployment/manifest.rs @@ -122,6 +122,7 @@ mod tests { version: super::super::PUBLISH_PROTOCOL_VERSION, operation_id: crate::Uuid::from_u128(0x01991ec4000070008000000000000001), expected_revision: None, + expected_last_operation: None, module_action: super::super::ModuleAction::Keep, container_action: crate::container::ContainerAction::Keep, }, diff --git a/crates/lib/src/deployment/tests.rs b/crates/lib/src/deployment/tests.rs index e038e0f4860..2918d6c5dfe 100644 --- a/crates/lib/src/deployment/tests.rs +++ b/crates/lib/src/deployment/tests.rs @@ -9,6 +9,7 @@ fn request(module_action: ModuleAction) -> PublishEnvelope { version: PUBLISH_PROTOCOL_VERSION, operation_id: operation_id(), expected_revision: None, + expected_last_operation: None, module_action, container_action: ContainerAction::Keep, } diff --git a/crates/testing/tests/deployment_publish.rs b/crates/testing/tests/deployment_publish.rs index 4ce3c18d1ae..7d894aa91fd 100644 --- a/crates/testing/tests/deployment_publish.rs +++ b/crates/testing/tests/deployment_publish.rs @@ -12,18 +12,19 @@ use spacetimedb_lib::container::{ ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, }; use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; -use spacetimedb_lib::{hash_bytes, sats::product, ConnectionId, Hash, Identity, Uuid}; +use spacetimedb_lib::{hash_bytes, sats::product, ConnectionId, Identity, Uuid}; use spacetimedb_schema::auto_migrate::{MigrationPolicy, MigrationToken}; use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -fn prepared(bytes: &[u8], epoch: u64, previous: Option, command: &str) -> DeploymentCommit { +fn prepared(bytes: &[u8], epoch: u64, previous: Option<&DeploymentCommit>, command: &str) -> DeploymentCommit { let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); DeploymentCommit { operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(epoch)), publication_epoch: epoch, publisher: Identity::ZERO, - expected_revision: previous, + expected_revision: previous.map(|request| request.deployment.revision().unwrap()), + expected_last_operation: previous.map(|request| request.operation_id), prepared_manifest_hash: hash_bytes(epoch.to_le_bytes()), deployment: DeploymentSpec::V1(DeploymentSpecV1 { module: ModuleComponent::User(UserModule { @@ -112,7 +113,7 @@ fn container_only_changes_and_old_retries_preserve_the_current_module() { let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); confirmed(result).await; let first_revision = first.deployment.revision().unwrap(); - let second = prepared(&bytes, 2, Some(first_revision), "/app/second"); + let second = prepared(&bytes, 2, Some(&first), "/app/second"); fence(&host.module().await.unwrap(), &second); confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, second.clone()).await.unwrap()).await; assert_ne!(first_revision, second.deployment.revision().unwrap()); @@ -121,7 +122,7 @@ fn container_only_changes_and_old_retries_preserve_the_current_module() { // its definition. This exercises the actual Wasm migration and swap. let mut newer_bytes = bytes.to_vec(); newer_bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); - let third = prepared(&newer_bytes, 3, Some(second.deployment.revision().unwrap()), "/app/third"); + let third = prepared(&newer_bytes, 3, Some(&second), "/app/third"); fence(&host.module().await.unwrap(), &third); confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, third.clone()).await.unwrap()).await; let newest = third.deployment.revision().unwrap(); @@ -140,10 +141,10 @@ fn container_only_changes_and_old_retries_preserve_the_current_module() { // A stale CAS and a forged association between bytes and declaration // cannot change either component, and rejection leaves service usable. - let stale = prepared(&bytes, 4, Some(first_revision), "/app/stale"); + let stale = prepared(&bytes, 4, Some(&first), "/app/stale"); fence(¤t, &stale); assert!(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, stale).await.is_err()); - let mismatched = prepared(&bytes, 5, Some(newest), "/app/mismatch"); + let mismatched = prepared(&bytes, 5, Some(&third), "/app/mismatch"); fence(¤t, &mismatched); assert!(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, mismatched).await.is_err()); let current = host.module().await.unwrap(); @@ -154,7 +155,7 @@ fn container_only_changes_and_old_retries_preserve_the_current_module() { }); let empty = spacetimedb::host::empty_module::program(spacetimedb::host::empty_module::VERSION).unwrap(); - let mut remove_module = prepared(&empty.bytes, 6, Some(newest), "/app/image-only"); + let mut remove_module = prepared(&empty.bytes, 6, Some(&third), "/app/image-only"); let DeploymentSpec::V1(spec) = &mut remove_module.deployment; spec.module = ModuleComponent::SystemEmpty(spacetimedb_lib::deployment::system_empty::empty().descriptor); // An init request cannot execute this instance's schema while storing diff --git a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md index 34161f9095a..4401be367de 100644 --- a/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md +++ b/docs/docs/00200-core-concepts/00100-databases/00700-environment-variables.md @@ -198,6 +198,8 @@ Previously stored values are **not** defaults for the next publish. Every publis The same rules apply to precompiled modules published with `--bin-path`. The CLI reads declarations from the artifact being published. +Managed publications also retain the complete resolved environment in the operation's private local `submission.json` file. Keep the publication directory private and out of source control. `--resume-publication` sends the original request bytes, including the original values, even if project files or shell variables have changed. Missing or altered retained input causes an error; resuming never substitutes an empty environment. When preserving the current module, the CLI checks its environment declarations against authenticated metadata for the selected database and program before creating this input. + ## Inspect published values The database owner and collaborators with private-table read access can inspect the environment. For the local database above: diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 859a0b0e81b..8f93f856c76 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -154,7 +154,7 @@ Every publish replaces the complete declared environment. Put an env map in spac * `--artifact-endpoint ` — Explicitly authorize this exact artifact URL to receive the publisher credential * `--remove-container` — Explicitly remove the container while preserving the module unless separately changed * `--remove-module` — Replace the module with the versioned empty module after migration preflight -* `--publication-state-dir ` — Private local directory retaining managed publication bytes and progress +* `--publication-state-dir ` — Private local directory retaining complete managed publication inputs and progress. The protected submission file includes resolved environment values. Keep this directory private; do not commit it or share it. Resume reuses these exact values without rereading project configuration or shell variables. * `--resume-publication ` — Resume this operation directory without rebuilding or reading spacetime.json * `--publication-wait ` — Seconds to wait for managed activation; pending operations retain their resume directory From d35d05c01d7b4f3ba8eb7db0ad76cae4788c8533 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 03:26:09 -0400 Subject: [PATCH 03/16] Join native client SDK connection tasks on driver shutdown --- sdks/rust/src/db_connection.rs | 89 +++++++- sdks/rust/src/db_connection/native_tasks.rs | 72 ++++++ .../src/db_connection/native_tasks/tests.rs | 210 ++++++++++++++++++ sdks/rust/src/db_connection/terminal_tests.rs | 1 + 4 files changed, 361 insertions(+), 11 deletions(-) create mode 100644 sdks/rust/src/db_connection/native_tasks.rs create mode 100644 sdks/rust/src/db_connection/native_tasks/tests.rs diff --git a/sdks/rust/src/db_connection.rs b/sdks/rust/src/db_connection.rs index 8d114f4cbce..7f9f131dbaa 100644 --- a/sdks/rust/src/db_connection.rs +++ b/sdks/rust/src/db_connection.rs @@ -31,6 +31,8 @@ use crate::{ websocket::{WsConnection, WsParams}, }; use bytes::Bytes; +#[cfg(not(feature = "browser"))] +use futures::FutureExt; use futures::StreamExt; #[cfg(feature = "browser")] use futures::{pin_mut, FutureExt}; @@ -58,6 +60,11 @@ pub(crate) type SharedCell = Arc>; #[cfg(all(test, not(feature = "browser")))] mod builder_tests; +#[cfg(not(feature = "browser"))] +mod native_tasks; +#[cfg(not(feature = "browser"))] +use native_tasks::NativeTasks; + #[cfg(not(feature = "browser"))] type SharedAsyncCell = Arc>; #[cfg(feature = "browser")] @@ -732,11 +739,9 @@ impl DbContextImpl { #[cfg(not(feature = "browser"))] pub fn run_threaded(&self) -> std::thread::JoinHandle<()> { let this = self.clone(); - std::thread::spawn(move || loop { - match this.advance_one_message_blocking() { - Ok(()) => (), - Err(e) if error_is_normal_disconnect(&e) => return, - Err(e) => panic!("{e:?}"), + std::thread::spawn(move || { + if let Err(e) = this.runtime.block_on(this.run_async()) { + panic!("{e:?}"); } }) } @@ -760,14 +765,66 @@ impl DbContextImpl { /// An async task which does [`Self::advance_one_message_async`] in a loop. /// + /// On native targets, completion also waits for the WebSocket and parser + /// tasks to finish. Dropping this future does not disconnect the connection; + /// call `disconnect` and keep driving it to completion to finish shutdown. + /// A cancelled wait may be resumed by calling this method again. + /// /// Called by the autogenerated `DbConnection` method of the same name. pub async fn run_async(&self) -> crate::Result<()> { - let this = self.clone(); + #[cfg(feature = "browser")] + { + self.run_until_disconnected().await + } + #[cfg(not(feature = "browser"))] + { + // Retain task ownership once per driver, not on every event-context + // clone. Recover a poisoned guard only to obtain the cleanup owner. + let background_tasks = Arc::clone( + &self + .inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .background_tasks, + ); + // Callback panics still propagate, but only after the native tasks + // have relinquished the socket and parser state. Catching an unwind + // outside run_async must not disguise a detached connection. + let result = std::panic::AssertUnwindSafe(self.run_until_disconnected()) + .catch_unwind() + .await; + // An outgoing queue failure can panic while holding this mutex. + // Recover only to close it; poisoning must not bypass task cleanup. + let outgoing = self.send_chan.lock().unwrap_or_else(|error| error.into_inner()).take(); + drop(outgoing); + let mut background_tasks = background_tasks.lock().await; + match &result { + Ok(Err(error)) => background_tasks.record_failure(error.clone()), + Err(_) => { + background_tasks.record_failure(InternalError::new("Connection event processing panicked").into()) + } + Ok(Ok(())) => (), + } + let joined = background_tasks.stop_and_join().await; + match result { + Ok(Ok(())) => joined, + Ok(Err(error)) => Err(error), + Err(panic) => std::panic::resume_unwind(panic), + } + } + } + + async fn run_until_disconnected(&self) -> crate::Result<()> { loop { - match this.advance_one_message_async().await { + match self.advance_one_message_async().await { Ok(()) => (), Err(e) if error_is_normal_disconnect(&e) => return Ok(()), + #[cfg(feature = "browser")] Err(e) => return Err(e), + // Errors applying a message or local mutation must also close + // the connection, including those not routed by process_message. + #[cfg(not(feature = "browser"))] + Err(e) => return Err(self.end_connection(Some(e))), } } } @@ -895,6 +952,8 @@ enum ConnectionLifecycle { /// All the stuff in a [`DbContextImpl`] which can safely be locked while invoking callbacks. pub(crate) struct DbContextImplInner { + #[cfg(not(feature = "browser"))] + background_tasks: SharedAsyncCell, /// `Some` if not within the context of an outer runtime. The `Runtime` must /// then live as long as `Self`. #[allow(unused)] @@ -1090,16 +1149,21 @@ but you must call one of them, or else the connection will never progress. source: InternalError::new("Failed to initiate WebSocket connection").with_cause(source), })?; - let (_websocket_loop_handle, raw_msg_recv, raw_msg_send) = + let (websocket_loop_handle, raw_msg_recv, raw_msg_send) = ws_connection.spawn_message_loop(&handle, extra_logging.clone()); - let (_parse_loop_handle, parsed_recv_chan) = - spawn_parse_loop::(raw_msg_recv, &handle, extra_logging.clone()); + let (parse_loop_handle, parsed_recv_chan) = spawn_parse_loop::(raw_msg_recv, &handle, extra_logging.clone()); let parsed_recv_chan = Arc::new(TokioMutex::new(parsed_recv_chan)); let (pending_mutations_send, pending_mutations_recv) = mpsc::unbounded(); let pending_mutations_recv = Arc::new(TokioMutex::new(pending_mutations_recv)); - let inner_ctx = build_db_ctx_inner(None, self.on_connect, self.on_connect_error, self.on_disconnect); + let inner_ctx = build_db_ctx_inner( + None, + NativeTasks::new(websocket_loop_handle, parse_loop_handle), + self.on_connect, + self.on_connect_error, + self.on_disconnect, + ); Ok(build_db_ctx( handle, inner_ctx, @@ -1297,6 +1361,7 @@ Instead of registering multiple `on_disconnect` callbacks, register a single cal /// Create a [`DbContextImplInner`] wrapped in `Arc>`. fn build_db_ctx_inner( #[cfg(not(feature = "browser"))] runtime: Option, + #[cfg(not(feature = "browser"))] background_tasks: NativeTasks, on_connect_cb: Option>, on_connect_error_cb: Option>, @@ -1305,6 +1370,8 @@ fn build_db_ctx_inner( Arc::new(StdMutex::new(DbContextImplInner { #[cfg(not(feature = "browser"))] runtime, + #[cfg(not(feature = "browser"))] + background_tasks: Arc::new(TokioMutex::new(background_tasks)), db_callbacks: DbCallbacks::default(), reducer_callbacks: ReducerCallbacks::default(), diff --git a/sdks/rust/src/db_connection/native_tasks.rs b/sdks/rust/src/db_connection/native_tasks.rs new file mode 100644 index 00000000000..7671a5dbfda --- /dev/null +++ b/sdks/rust/src/db_connection/native_tasks.rs @@ -0,0 +1,72 @@ +//! Native connection tasks retained through terminal cleanup. + +use crate::__codegen::InternalError; +use tokio::task::JoinHandle; + +#[cfg(test)] +mod tests; + +#[derive(Default)] +pub(super) struct NativeTasks { + websocket: Option>, + parser: Option>, + failure: Option, +} + +impl NativeTasks { + pub fn new(websocket: JoinHandle<()>, parser: JoinHandle<()>) -> Self { + Self { + websocket: Some(websocket), + parser: Some(parser), + failure: None, + } + } + + pub fn record_failure(&mut self, error: crate::Error) { + // A cancelled terminal wait must not discard its original failure. + self.failure.get_or_insert(error); + } + + /// Called only after event processing has terminated. In particular, a + /// parser error must not leave the WebSocket waiting for more input. + pub async fn stop_and_join(&mut self) -> crate::Result<()> { + self.abort(); + // Each handle stays in shared connection state across await. Clear it + // immediately after joining, before the next suspension point, so a + // cancelled caller cannot detach it or poll a completed handle twice. + Self::join(&mut self.websocket, &mut self.failure, "WebSocket").await; + Self::join(&mut self.parser, &mut self.failure, "parser").await; + self.failure.clone().map_or(Ok(()), Err) + } + + async fn join(handle: &mut Option>, failure: &mut Option, name: &str) { + let Some(task) = handle.as_mut() else { return }; + let result = task.await; + *handle = None; + if let Err(error) = result { + // Cancellation was requested above. A task panic is still a failure, + // and repeated terminal waits must continue reporting that failure. + if !error.is_cancelled() && failure.is_none() { + *failure = Some( + InternalError::new(format!("Native {name} task failed")) + .with_cause(error) + .into(), + ); + } + } + } + + fn abort(&self) { + for task in [&self.websocket, &self.parser].into_iter().flatten() { + task.abort(); + } + } +} + +impl Drop for NativeTasks { + fn drop(&mut self) { + // Dropping the last connection is a cancellation request, not a join. + // Owners requiring completed shutdown must finish run_async/run_threaded. + self.abort(); + } +} diff --git a/sdks/rust/src/db_connection/native_tasks/tests.rs b/sdks/rust/src/db_connection/native_tasks/tests.rs new file mode 100644 index 00000000000..1a3906055d6 --- /dev/null +++ b/sdks/rust/src/db_connection/native_tasks/tests.rs @@ -0,0 +1,210 @@ +use super::*; +use crate::db_connection::{ + build_db_ctx, build_db_ctx_inner, terminal_tests::bindings::RemoteModule, DbConnectionBuilder, ParsedMessage, +}; +use futures::{FutureExt, SinkExt, StreamExt}; +use spacetimedb_client_api_messages::websocket::v2 as ws; +use spacetimedb_lib::{bsatn, ConnectionId, Identity, Timestamp}; +use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; +use tokio::{net::TcpListener, sync::oneshot}; +use tokio_tungstenite::tungstenite::{handshake::server::Response, Message}; + +async fn blocked_task() -> (JoinHandle<()>, std::sync::mpsc::Sender<()>) { + let (started, ready) = oneshot::channel(); + let (release, blocked) = std::sync::mpsc::channel(); + let task = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + // Dropping release on a failed assertion also releases this worker. + let _ = blocked.recv(); + }); + ready.await.unwrap(); + (task, release) +} + +#[tokio::test] +async fn cancellation_during_second_join_preserves_owner_for_retry() { + let first = tokio::spawn(std::future::pending()); + let (second, release) = blocked_task().await; + let mut tasks = NativeTasks::new(first, second); + assert!(tokio::time::timeout(Duration::from_millis(30), tasks.stop_and_join()) + .await + .is_err()); + assert!(tasks.websocket.is_none(), "first task should already have been joined"); + assert!(tasks.parser.is_some(), "pending task must still have an owner"); + release.send(()).unwrap(); + tasks.stop_and_join().await.unwrap(); + assert!(tasks.parser.is_none()); + tasks.stop_and_join().await.unwrap(); +} + +#[tokio::test] +async fn task_panic_still_joins_other_task_and_remains_an_error_after_retry() { + let (panicking, ready) = oneshot::channel(); + let first = tokio::spawn(async move { + let _ = panicking.send(()); + panic!("injected connection task panic"); + }); + ready.await.unwrap(); + let (second, release) = blocked_task().await; + let mut tasks = NativeTasks::new(first, second); + assert!(tokio::time::timeout(Duration::from_millis(30), tasks.stop_and_join()) + .await + .is_err()); + assert!(tasks.failure.is_some()); + assert!(tasks.websocket.is_none()); + assert!(tasks.parser.is_some()); + release.send(()).unwrap(); + assert!(tasks.stop_and_join().await.is_err()); + assert!(tasks.parser.is_none()); + assert!(tasks.stop_and_join().await.is_err()); +} + +#[tokio::test] +async fn shutdown_drops_both_task_resources_before_returning() { + let resource = Arc::new(()); + let first_capture = resource.clone(); + let second_capture = resource.clone(); + let first = tokio::spawn(async move { + let _capture = first_capture; + std::future::pending::<()>().await; + }); + let second = tokio::spawn(async move { + let _capture = second_capture; + std::future::pending::<()>().await; + }); + let mut tasks = NativeTasks::new(first, second); + tasks.stop_and_join().await.unwrap(); + assert_eq!(Arc::strong_count(&resource), 1); +} + +#[tokio::test] +async fn cancelled_terminal_driver_preserves_original_processing_error() { + let first = tokio::spawn(std::future::pending()); + let (second, release) = blocked_task().await; + let inner = build_db_ctx_inner::(None, NativeTasks::new(first, second), None, None, None); + let (outgoing, _outgoing_recv) = futures_channel::mpsc::unbounded(); + let (incoming, incoming_recv) = futures_channel::mpsc::unbounded(); + let (pending, pending_recv) = futures_channel::mpsc::unbounded(); + incoming + .unbounded_send(ParsedMessage::Error( + InternalError::new("original processing failure").into(), + )) + .unwrap(); + drop(incoming); + let context = build_db_ctx( + tokio::runtime::Handle::current(), + inner, + outgoing, + Arc::new(tokio::sync::Mutex::new(incoming_recv)), + pending, + Arc::new(tokio::sync::Mutex::new(pending_recv)), + None, + None, + ); + assert!(tokio::time::timeout(Duration::from_millis(30), context.run_async()) + .await + .is_err()); + release.send(()).unwrap(); + let error = context.run_async().await.unwrap_err(); + assert!(error.to_string().contains("original processing failure")); +} + +async fn terminal_connection_joins_tasks(callback_panics: bool, poison_sender: bool) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (closed, closure) = oneshot::channel(); + let peer = tokio::spawn(async move { + let (socket, address) = listener.accept().await.unwrap(); + assert!(address.ip().is_loopback()); + let mut socket = + tokio_tungstenite::accept_hdr_async(socket, |_: &http::Request<()>, mut response: Response| { + response + .headers_mut() + .insert("sec-websocket-protocol", ws::BIN_PROTOCOL.parse().unwrap()); + Ok(response) + }) + .await + .unwrap(); + let message = if callback_panics { + ws::ServerMessage::InitialConnection(ws::InitialConnection { + identity: Identity::from_u256(1u32.into()), + connection_id: ConnectionId::from_u128(1), + token: "disposable-test-token".into(), + }) + } else { + // A well-formed response for a request this client never made is a + // processing error. The peer deliberately keeps its socket open. + ws::ServerMessage::ReducerResult(ws::ReducerResult { + request_id: u32::MAX, + timestamp: Timestamp::UNIX_EPOCH, + result: ws::ReducerOutcome::OkEmpty, + }) + }; + let mut bytes = vec![0]; // No compression. + bytes.extend(bsatn::to_vec(&message).unwrap()); + socket.send(Message::Binary(bytes.into())).await.unwrap(); + while let Some(message) = socket.next().await { + if message.is_err() || matches!(message, Ok(Message::Close(_))) { + break; + } + } + let _ = closed.send(()); + }); + // Keep a positive join even when the test body fails or times out. + let result = AssertUnwindSafe(tokio::time::timeout(Duration::from_secs(3), async { + let context = DbConnectionBuilder::::new() + .with_uri(format!("http://{address}")) + .with_database_name("disposable-native-task-test") + .on_connect(move |_, _, _| { + assert!(!callback_panics, "injected application callback panic"); + }) + .build_native_impl(tokio::runtime::Handle::current()) + .await + .unwrap(); + if poison_sender { + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _outgoing = context.send_chan.lock().unwrap(); + panic!("injected outgoing queue panic while holding its mutex"); + })); + assert!(result.is_err()); + } + let result = AssertUnwindSafe(context.run_async()).catch_unwind().await; + if callback_panics || poison_sender { + assert!(result.is_err(), "callback panic should propagate after cleanup"); + } else { + assert!(result.unwrap().is_err(), "unsolicited reducer result must fail"); + } + assert!(context + .send_chan + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_none()); + let tasks = context.inner.lock().unwrap().background_tasks.clone(); + let tasks = tasks.lock().await; + assert!(tasks.websocket.is_none() && tasks.parser.is_none()); + closure.await.expect("peer disappeared before observing socket closure"); + })) + .catch_unwind() + .await; + peer.abort(); + let joined = peer.await; + if let Err(error) = joined { + assert!(error.is_cancelled(), "test peer failed: {error}"); + } + result.unwrap().expect("connection did not finish native task cleanup"); +} + +#[tokio::test] +async fn processing_error_closes_socket_and_joins_tasks_with_retained_connection() { + terminal_connection_joins_tasks(false, false).await; +} + +#[tokio::test] +async fn callback_panic_joins_tasks_before_propagating_unwind() { + terminal_connection_joins_tasks(true, false).await; +} + +#[tokio::test] +async fn poisoned_outgoing_mutex_does_not_bypass_native_task_join() { + terminal_connection_joins_tasks(false, true).await; +} diff --git a/sdks/rust/src/db_connection/terminal_tests.rs b/sdks/rust/src/db_connection/terminal_tests.rs index 397c36a34ae..b88392dfb3a 100644 --- a/sdks/rust/src/db_connection/terminal_tests.rs +++ b/sdks/rust/src/db_connection/terminal_tests.rs @@ -48,6 +48,7 @@ fn fixture( ) { let inner = build_db_ctx_inner::( None, + NativeTasks::default(), None, None, Some(Box::new(move |_, error| { From f1a474649dc0026f6dab2a45a720ac602d196d8c Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 04:09:42 -0400 Subject: [PATCH 04/16] Define bounded container exec WebSocket messages --- crates/lib/src/container.rs | 2 + crates/lib/src/container/exec.rs | 226 +++++++++++++++++++++++++ crates/lib/src/container/exec/tests.rs | 184 ++++++++++++++++++++ crates/lib/src/container/operations.rs | 2 +- 4 files changed, 413 insertions(+), 1 deletion(-) create mode 100644 crates/lib/src/container/exec.rs create mode 100644 crates/lib/src/container/exec/tests.rs diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs index 02d9c581824..564f5a2cc7a 100644 --- a/crates/lib/src/container.rs +++ b/crates/lib/src/container.rs @@ -10,6 +10,8 @@ use std::{collections::BTreeSet, fmt, str::FromStr}; #[cfg(feature = "serde")] pub mod endpoints; #[cfg(feature = "serde")] +pub mod exec; +#[cfg(feature = "serde")] pub mod operations; /// Version of the normalized deployment encoding, independent of module ABI. diff --git a/crates/lib/src/container/exec.rs b/crates/lib/src/container/exec.rs new file mode 100644 index 00000000000..f517e8a79d6 --- /dev/null +++ b/crates/lib/src/container/exec.rs @@ -0,0 +1,226 @@ +//! Versioned messages for one authenticated WebSocket per container exec. +//! +//! Authenticate the upgrade before accepting a Start message. Start must be the +//! first application message and is accepted only once. A Ready response binds +//! the socket to that exact database, generation and session until Exit/Error. +//! Reconnection or replay must never silently start another process. EOF closes +//! only stdin; closing the socket does not establish process termination. +//! Reject duplicate Start, stdin after EOF, and controls after the terminal +//! response. Output precedes exactly one Exit/Error. Bound queued bytes as well +//! as individual messages. Retain the chosen RuntimeHandle throughout the +//! session, even if another instance starts under the same deployment. +//! +//! These codecs bound and validate messages, not authority or session ordering. +//! The server separately checks Admin permission, the complete current runtime +//! binding and its lease, and validates the inherited environment before exec. + +use super::{ + operations::{decimal_u64, ContainerErrorCode}, + validate_env_key, validate_exec_size, MAX_ARGV_ENTRIES, MAX_ENV_KEYS, +}; +use crate::{deployment::uuid_json, Identity, Uuid}; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, fmt}; + +pub const SUBPROTOCOL: &str = "v1.container.exec.spacetimedb"; +/// JSON escapes may expand a valid 128 KiB argv/environment by up to six times. +pub const MAX_CONTROL_BYTES: usize = 1024 * 1024; +pub const MAX_DATA_BYTES: usize = 64 * 1024; +pub const MAX_BINARY_BYTES: usize = MAX_DATA_BYTES + 1; + +/// Fixed diagnostics never reflect argv, environment values or peer input. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid container exec message")] +pub struct ProtocolError; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TerminalSize { + pub rows: u16, + pub columns: u16, +} +impl TerminalSize { + pub fn validate(self) -> Result<(), ProtocolError> { + if (1..=4096).contains(&self.rows) && (1..=4096).contains(&self.columns) { + Ok(()) + } else { + Err(ProtocolError) + } + } +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExecStart { + /// Exact control generation selected by status, never a floating "current". + #[serde(with = "decimal_u64")] + pub generation: u64, + /// Literal argv; the server never supplies an implicit shell or user override. + pub argv: Vec, + /// None inherits the container's directory. Some must be an absolute path. + pub working_directory: Option, + /// Overrides for this process only; platform keys remain reserved. + pub environment: BTreeMap, + pub stdin: bool, + /// Some allocates a PTY at this initial size; None keeps stdout/stderr separate. + pub terminal: Option, +} +impl fmt::Debug for ExecStart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ExecStart") + .field("generation", &self.generation) + .field("stdin", &self.stdin) + .field("terminal", &self.terminal) + .finish_non_exhaustive() + } +} +impl ExecStart { + /// The backend must also count its inherited environment after applying + /// these overrides. This validation cannot see that immutable launch state. + pub fn validate(&self) -> Result<(), ProtocolError> { + if self.generation == 0 + || self.argv.is_empty() + || self.argv[0].is_empty() + || self.argv.len() > MAX_ARGV_ENTRIES + || self.environment.len() > MAX_ENV_KEYS + || self.environment.keys().any(|key| validate_env_key(key).is_err()) + || self + .working_directory + .as_ref() + .is_some_and(|path| !path.starts_with('/') || path.len() > 4096 || path.contains('\0')) + { + return Err(ProtocolError); + } + if let Some(size) = self.terminal { + size.validate()?; + } + let environment: Vec<_> = self + .environment + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect(); + validate_exec_size(&self.argv, &environment).map_err(|_| ProtocolError) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", content = "data", rename_all = "snake_case", deny_unknown_fields)] +pub enum ClientControl { + Start(ExecStart), + StdinEof, + Resize(TerminalSize), + /// Linux signal number on the supported x86_64/aarch64 guest platforms. + /// Targets this exec process, never the container main process or a host PID. + Signal(u8), +} +impl ClientControl { + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_CONTROL_BYTES { + return Err(ProtocolError); + } + let message: Self = serde_json::from_slice(bytes).map_err(|_| ProtocolError)?; + message.validate()?; + Ok(message) + } + pub fn validate(&self) -> Result<(), ProtocolError> { + match self { + Self::Start(start) => start.validate(), + Self::Resize(size) => size.validate(), + Self::Signal(1..=64) | Self::StdinEof => Ok(()), + Self::Signal(_) => Err(ProtocolError), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExecReady { + pub database_identity: Identity, + #[serde(with = "decimal_u64")] + pub generation: u64, + #[serde(with = "uuid_json")] + pub session_id: Uuid, + pub tty: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", content = "data", rename_all = "snake_case", deny_unknown_fields)] +pub enum ServerControl { + Ready(ExecReady), + /// Follows the final output frame and confirms observed process exit. + Exit { + exit_code: i64, + }, + /// Terminal failure. Does not assert that an already started process exited. + Error { + error: ContainerErrorCode, + }, +} +impl ServerControl { + pub fn decode(bytes: &[u8]) -> Result { + // Responses contain metadata only, so they need much less space than Start. + if bytes.len() > 4096 { + return Err(ProtocolError); + } + let message: Self = serde_json::from_slice(bytes).map_err(|_| ProtocolError)?; + if matches!(&message, Self::Ready(ready) if ready.generation == 0 || ready.session_id.as_u128() == 0) { + return Err(ProtocolError); + } + Ok(message) + } +} + +/// PTY output uses Stdout because the guest PTY combines both output streams. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum Stream { + Stdin = 0, + Stdout = 1, + Stderr = 2, +} + +/// One channel byte followed by arbitrary nonempty process bytes. Binary data +/// need not be UTF-8; empty messages never stand in for an EOF control message. +pub fn encode_data(stream: Stream, bytes: &[u8]) -> Result, ProtocolError> { + if bytes.is_empty() || bytes.len() > MAX_DATA_BYTES { + return Err(ProtocolError); + } + let mut frame = Vec::with_capacity(bytes.len() + 1); + frame.push(stream as u8); + frame.extend_from_slice(bytes); + Ok(frame) +} + +pub fn decode_stdin(frame: &[u8]) -> Result<&[u8], ProtocolError> { + let (stream, data) = decode_data(frame)?; + if stream == Stream::Stdin { + Ok(data) + } else { + Err(ProtocolError) + } +} + +pub fn decode_output(frame: &[u8]) -> Result<(Stream, &[u8]), ProtocolError> { + let (stream, data) = decode_data(frame)?; + if stream == Stream::Stdin { + Err(ProtocolError) + } else { + Ok((stream, data)) + } +} + +fn decode_data(frame: &[u8]) -> Result<(Stream, &[u8]), ProtocolError> { + if frame.len() < 2 || frame.len() > MAX_BINARY_BYTES { + return Err(ProtocolError); + } + let stream = match frame[0] { + 0 => Stream::Stdin, + 1 => Stream::Stdout, + 2 => Stream::Stderr, + _ => return Err(ProtocolError), + }; + Ok((stream, &frame[1..])) +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/container/exec/tests.rs b/crates/lib/src/container/exec/tests.rs new file mode 100644 index 00000000000..e3d1ff496e6 --- /dev/null +++ b/crates/lib/src/container/exec/tests.rs @@ -0,0 +1,184 @@ +use super::*; + +fn start() -> ExecStart { + ExecStart { + generation: u64::MAX, + argv: vec!["/bin/echo".into(), "secret-argv; $(literal)".into()], + working_directory: Some("/secret-directory".into()), + environment: BTreeMap::from([("TOKEN".into(), "secret-value".into())]), + stdin: true, + terminal: None, + } +} + +#[test] +fn start_preserves_literal_arguments_exact_generation_and_redacts_debug() { + let message = ClientControl::Start(start()); + let json = serde_json::to_vec(&message).unwrap(); + assert!(String::from_utf8_lossy(&json).contains("\"generation\":\"18446744073709551615\"")); + assert_eq!(ClientControl::decode(&json).unwrap(), message); + let debug = format!("{message:?}"); + for secret in ["secret-argv", "literal", "secret-directory", "TOKEN", "secret-value"] { + assert!(!debug.contains(secret)); + } +} + +#[test] +fn invalid_start_never_reflects_values_and_rejects_ambiguous_generation() { + let baseline = serde_json::to_value(ClientControl::Start(start())).unwrap(); + for value in [ + serde_json::json!(1), + serde_json::json!("01"), + serde_json::json!("0"), + serde_json::json!("-1"), + ] { + let mut changed = baseline.clone(); + changed["data"]["generation"] = value; + assert!(ClientControl::decode(&serde_json::to_vec(&changed).unwrap()).is_err()); + } + let changes: [fn(&mut ExecStart); 12] = [ + |s| s.argv.clear(), + |s| s.argv[0].clear(), + |s| s.argv.push("secret-value\0".into()), + |s| s.argv = vec!["x".into(); MAX_ARGV_ENTRIES + 1], + |s| s.working_directory = Some("relative-secret-path".into()), + |s| s.working_directory = Some("/secret\0".into()), + |s| s.working_directory = Some(format!("/{}", "x".repeat(4096))), + |s| { + s.environment.insert("SPACETIMEDB_TOKEN".into(), "secret-value".into()); + }, + |s| { + s.environment.insert("INVALID=KEY".into(), "secret-value".into()); + }, + |s| { + s.environment.insert("TOKEN".into(), "secret\0".into()); + }, + |s| s.environment = (0..=MAX_ENV_KEYS).map(|i| (format!("E{i}"), String::new())).collect(), + |s| s.terminal = Some(TerminalSize { rows: 0, columns: 80 }), + ]; + for change in changes { + let mut changed = start(); + change(&mut changed); + let bytes = serde_json::to_vec(&ClientControl::Start(changed)).unwrap(); + let error = ClientControl::decode(&bytes).unwrap_err(); + assert_eq!(error.to_string(), "invalid container exec message"); + assert!(!format!("{error:?}").contains("secret")); + } +} + +#[test] +fn aggregate_budget_counts_overrides_and_json_bound_accepts_escaped_values() { + let mut request = start(); + request.environment.clear(); + request.argv = vec!["program".into()]; + request.argv.extend(vec!["\u{1}".repeat(15_000); 8]); + let bytes = serde_json::to_vec(&ClientControl::Start(request.clone())).unwrap(); + assert!(bytes.len() > 700_000 && bytes.len() < MAX_CONTROL_BYTES); + assert!(ClientControl::decode(&bytes).is_ok()); + request.environment.insert("TOKEN".into(), "x".repeat(15_000)); + assert!(request.validate().is_err()); + request = start(); + request + .environment + .insert("TOKEN".into(), "x".repeat(super::super::MAX_EXEC_STRING_BYTES)); + assert!(request.validate().is_err()); + assert!(ClientControl::decode(&vec![b' '; MAX_CONTROL_BYTES + 1]).is_err()); +} + +#[test] +fn binary_frames_preserve_arbitrary_bytes_enforce_direction_and_bound_size() { + let bytes = [0, 255, 128, b'\n']; + let input = encode_data(Stream::Stdin, &bytes).unwrap(); + assert_eq!(decode_stdin(&input).unwrap(), bytes); + assert!(decode_output(&input).is_err()); + for stream in [Stream::Stdout, Stream::Stderr] { + let output = encode_data(stream, &bytes).unwrap(); + assert_eq!(decode_output(&output).unwrap(), (stream, bytes.as_slice())); + assert!(decode_stdin(&output).is_err()); + } + let max = encode_data(Stream::Stdin, &vec![0xff; MAX_DATA_BYTES]).unwrap(); + assert_eq!(max.len(), MAX_BINARY_BYTES); + assert_eq!(decode_stdin(&max).unwrap().len(), MAX_DATA_BYTES); + for invalid in [vec![], vec![0], vec![3, 1], vec![0; MAX_BINARY_BYTES + 1]] { + assert!(decode_stdin(&invalid).is_err()); + assert!(decode_output(&invalid).is_err()); + } + assert!(encode_data(Stream::Stdin, &[]).is_err()); + assert!(encode_data(Stream::Stdout, &vec![0; MAX_DATA_BYTES + 1]).is_err()); +} + +#[test] +fn eof_resize_and_signal_controls_are_distinct_and_checked() { + let eof = br#"{"type":"stdin_eof"}"#; + assert_eq!(ClientControl::decode(eof).unwrap(), ClientControl::StdinEof); + assert!(decode_stdin(&[0]).is_err()); + for control in [ + ClientControl::Resize(TerminalSize { rows: 24, columns: 80 }), + ClientControl::Signal(1), + ClientControl::Signal(64), + ] { + assert_eq!( + ClientControl::decode(&serde_json::to_vec(&control).unwrap()).unwrap(), + control + ); + } + for control in [ + ClientControl::Resize(TerminalSize { + rows: 4097, + columns: 80, + }), + ClientControl::Signal(0), + ClientControl::Signal(65), + ] { + assert!(ClientControl::decode(&serde_json::to_vec(&control).unwrap()).is_err()); + } +} + +#[test] +fn protocol_rejects_unknown_duplicate_and_trailing_fields_without_echo() { + for bad in [ + br#"{"type":"signal","data":2,"secret":"value"}"#.as_slice(), + br#"{"type":"resize","data":{"rows":24,"columns":80,"secret":"value"}}"#, + br#"{"type":"signal","type":"stdin_eof","data":2}"#, + br#"{"type":"signal","data":2} {"secret":"value"}"#, + br#"{"type":"not-a-command","data":"secret-value"}"#, + ] { + assert_eq!(ClientControl::decode(bad).unwrap_err(), ProtocolError); + } +} + +#[test] +fn server_messages_keep_exact_session_metadata_and_reject_invalid_ready() { + let ready = ExecReady { + database_identity: Identity::from_byte_array([7; 32]), + generation: u64::MAX, + session_id: Uuid::from_u128(42), + tty: true, + }; + for message in [ + ServerControl::Ready(ready.clone()), + ServerControl::Exit { exit_code: 137 }, + ServerControl::Error { + error: ContainerErrorCode::AccessDenied, + }, + ] { + assert_eq!( + ServerControl::decode(&serde_json::to_vec(&message).unwrap()).unwrap(), + message + ); + } + for changed in [ + ExecReady { + generation: 0, + ..ready.clone() + }, + ExecReady { + session_id: Uuid::from_u128(0), + ..ready + }, + ] { + assert!(ServerControl::decode(&serde_json::to_vec(&ServerControl::Ready(changed)).unwrap()).is_err()); + } + assert!(ServerControl::decode(br#"{"type":"exit","data":{"exit_code":0,"secret":"value"}}"#).is_err()); + assert!(ServerControl::decode(&vec![b' '; 4097]).is_err()); +} diff --git a/crates/lib/src/container/operations.rs b/crates/lib/src/container/operations.rs index f6b499f1db8..1f477b229f7 100644 --- a/crates/lib/src/container/operations.rs +++ b/crates/lib/src/container/operations.rs @@ -145,7 +145,7 @@ pub struct ContainerApiError { pub error: ContainerErrorCode, } -mod decimal_u64 { +pub(super) mod decimal_u64 { use serde::{Deserialize, Deserializer, Serializer}; pub fn serialize(number: &u64, serializer: S) -> Result { serializer.collect_str(number) From 34d7dc79bf26c220910d689fcf98dbee2698ea31 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 06:34:57 -0400 Subject: [PATCH 05/16] Add container exec CLI with retained terminal I/O --- crates/cli/Cargo.toml | 5 +- crates/cli/src/subcommands/container.rs | 4 + crates/cli/src/subcommands/container/exec.rs | 118 +++++ .../src/subcommands/container/exec/session.rs | 205 +++++++++ .../subcommands/container/exec/terminal.rs | 404 +++++++++++++++++ .../container/exec/terminal_pty_tests.rs | 163 +++++++ .../src/subcommands/container/exec/tests.rs | 405 ++++++++++++++++++ .../src/subcommands/container/operations.rs | 8 +- .../00100-cli-reference.md | 26 ++ 9 files changed, 1333 insertions(+), 5 deletions(-) create mode 100644 crates/cli/src/subcommands/container/exec.rs create mode 100644 crates/cli/src/subcommands/container/exec/session.rs create mode 100644 crates/cli/src/subcommands/container/exec/terminal.rs create mode 100644 crates/cli/src/subcommands/container/exec/terminal_pty_tests.rs create mode 100644 crates/cli/src/subcommands/container/exec/tests.rs diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 34656eaae3c..d76ebf60559 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -96,7 +96,10 @@ pretty_assertions.workspace = true tikv-jemallocator = { workspace = true } [target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] -rustix = { version = "1", features = ["fs", "process"] } +rustix = { version = "1", features = ["fs", "process", "termios", "event"] } + +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dev-dependencies] +rustix = { version = "1", features = ["pty"] } [target.'cfg(windows)'.dependencies] windows-sys = { workspace = true, features = ["Win32_System_Console", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Threading", "Win32_System_SystemServices"] } diff --git a/crates/cli/src/subcommands/container.rs b/crates/cli/src/subcommands/container.rs index 5e66ef2eeed..92dfff937cc 100644 --- a/crates/cli/src/subcommands/container.rs +++ b/crates/cli/src/subcommands/container.rs @@ -1,5 +1,7 @@ //! Container build and operation commands. Local builds do not read saved //! server credentials; network commands use the explicitly selected server. +#[path = "container/exec.rs"] +mod execute; mod operations; mod url; @@ -17,6 +19,7 @@ pub fn cli() -> Command { .about("Build and manage a database's container") .subcommand_required(true) .subcommand(url::cli()) + .subcommand(execute::cli()) .subcommand( Command::new("build") .about("Prepare verified OCI artifacts locally without publishing") @@ -120,6 +123,7 @@ pub(crate) fn select(config: &SpacetimeConfig, database: Option<&str>) -> Result pub async fn exec(mut config: crate::Config, args: &ArgMatches) -> Result<()> { match args.subcommand().context("missing container command")? { ("build", args) => exec_build(args).await, + ("exec", args) => execute::exec(&mut config, args).await, ("url", args) => url::exec(&config, args).await, (name @ ("status" | "start" | "stop" | "restart"), args) => operations::exec(&mut config, name, args).await, _ => anyhow::bail!("unsupported container command"), diff --git a/crates/cli/src/subcommands/container/exec.rs b/crates/cli/src/subcommands/container/exec.rs new file mode 100644 index 00000000000..8dfd976b547 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec.rs @@ -0,0 +1,118 @@ +//! One invocation owns one exec socket. A lost connection is never replayed. +#[path = "exec/session.rs"] +mod session; +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[path = "exec/terminal.rs"] +mod terminal; +#[cfg(test)] +#[path = "exec/tests.rs"] +mod tests; + +use anyhow::{ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_lib::container::{ + exec::ExecStart, + operations::{ContainerStatus, DesiredState, ObservedState}, +}; +use std::collections::BTreeMap; + +pub(super) fn cli() -> Command { + Command::new("exec") + .about("Run a literal command in the current running container") + .after_help("Requires database Admin permission. No shell, container start, or reconnect is implicit. Use -- before COMMAND; for a shell, name its executable explicitly. Linux and macOS terminals are currently supported. A lost connection does not establish that the process exited.") + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg(crate::common_args::server()) + .arg(crate::common_args::yes()) + .arg(Arg::new("interactive").short('i').long("interactive").action(ArgAction::SetTrue).help("Forward stdin and send EOF when it closes")) + .arg(Arg::new("tty").short('t').long("tty").requires("interactive").action(ArgAction::SetTrue).help("Allocate a PTY using this foreground terminal's dimensions")) + .arg(Arg::new("workdir").long("workdir").help("Absolute working directory inside the container")) + .arg(Arg::new("environment").short('e').long("env").action(ArgAction::Append).value_name("NAME=VALUE").help("Override a process environment variable; platform keys are reserved")) + .arg(Arg::new("argv").required(true).num_args(1..).last(true).value_name("COMMAND").help("Executable and literal arguments; no shell expansion")) +} + +fn start(args: &ArgMatches, generation: u64) -> Result { + let mut environment = BTreeMap::new(); + for value in args.get_many::("environment").into_iter().flatten() { + let (key, value) = value + .split_once('=') + .context("exec environment overrides require NAME=VALUE")?; + ensure!( + environment.insert(key.to_owned(), value.to_owned()).is_none(), + "duplicate exec environment override" + ); + } + let start = ExecStart { + generation, + argv: args + .get_many::("argv") + .context("exec command is required")? + .cloned() + .collect(), + working_directory: args.get_one::("workdir").cloned(), + environment, + stdin: args.get_flag("interactive"), + terminal: None, + }; + start + .validate() + .context("invalid exec command, directory, or environment override")?; + Ok(start) +} + +fn running_generation(status: &ContainerStatus) -> Result { + ensure!(status.published, "database has no published container"); + let operation = status.operational.as_ref().context("container is not running")?; + let instance = operation + .current_instance + .as_ref() + .context("container is not running")?; + ensure!( + operation.desired_state == DesiredState::Running + && operation.generation != 0 + && instance.generation == operation.generation + && matches!(instance.state, ObservedState::Running | ObservedState::Ready), + "container is not running at the current generation" + ); + Ok(operation.generation) +} + +pub(super) async fn exec(config: &mut crate::Config, args: &ArgMatches) -> Result<()> { + // Reject unsupported terminal implementations before login, status, or a socket. + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = (config, args); + anyhow::bail!("container exec terminal support is not implemented on this host platform"); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + let mut start = start(args, 1)?; + if args.get_flag("tty") { + start.terminal = Some(terminal::dimensions()?); + } + let selection = args.get_one::("server").map(String::as_str); + let origin = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; + let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; + let mut auth = auth.to_header().context("container exec requires a login")?; + auth.set_sensitive(true); + let client = super::operations::ContainerClient::new(origin, auth.clone())?; + let status = client + .status(args.get_one::("database").context("database is required")?) + .await?; + start.generation = running_generation(&status)?; + let url = client.url(status.database_identity.to_hex().as_ref(), "exec")?; + let signals = terminal::signals(start.terminal.is_some())?; + let (mut terminal, mut io) = terminal::Terminal::stdio(start.stdin, start.terminal.is_some())?; + let result = session::run(url, auth, status.database_identity, start, &mut io, signals).await; + // Always join and restore the terminal, including a failed handshake. + let cleanup = terminal.finish(); + let code = match (result, cleanup) { + (Ok(code), Ok(())) => code, + (Err(error), Ok(())) | (Ok(_), Err(error)) => return Err(error), + (Err(error), Err(cleanup)) => return Err(error.context(format!("terminal cleanup also failed: {cleanup}"))), + }; + if code != 0 { + return Err(crate::ExitWithCode(std::process::ExitCode::from(code)).into()); + } + Ok(()) + } +} diff --git a/crates/cli/src/subcommands/container/exec/session.rs b/crates/cli/src/subcommands/container/exec/session.rs new file mode 100644 index 00000000000..fa2204c3445 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/session.rs @@ -0,0 +1,205 @@ +use anyhow::{bail, ensure, Context, Result}; +use futures::{SinkExt, Stream, StreamExt}; +use reqwest::{ + header::{HeaderValue, AUTHORIZATION, SEC_WEBSOCKET_PROTOCOL}, + Url, +}; +use spacetimedb_lib::{container::exec::*, Identity}; +use std::time::Duration; +use tokio::sync::{mpsc, oneshot}; +use tokio_tungstenite::{ + connect_async_with_config, + tungstenite::{client::IntoClientRequest, protocol::WebSocketConfig, Message}, +}; + +const NETWORK_WAIT: Duration = Duration::from_secs(10); +pub(super) enum Input { + Data(Vec), + Eof, +} +pub(super) struct Output { + pub stream: OutputStream, + pub bytes: Vec, + pub completed: oneshot::Sender>, +} +pub(super) type OutputStream = spacetimedb_lib::container::exec::Stream; +pub(super) struct OutputSender { + pub sender: mpsc::Sender, + pub wake: Option>, +} +impl OutputSender { + async fn send(&self, value: Output) -> Result<()> { + self.sender.send(value).await.context("terminal output stopped")?; + if let Some(wake) = &self.wake { + wake(); + } + Ok(()) + } +} +pub(super) struct Io { + pub input: mpsc::Receiver, + pub output: OutputSender, + pub completion: oneshot::Receiver>, +} + +pub(super) async fn run( + mut url: Url, + authorization: HeaderValue, + identity: Identity, + start: ExecStart, + io: &mut Io, + mut signals: S, +) -> Result +where + S: Stream> + Unpin, +{ + start.validate()?; + ensure!( + url.username().is_empty() && url.password().is_none() && url.query().is_none() && url.fragment().is_none(), + "invalid exec endpoint" + ); + let scheme = match url.scheme() { + "http" => "ws", + "https" => "wss", + _ => bail!("invalid exec endpoint"), + }; + url.set_scheme(scheme) + .map_err(|_| anyhow::anyhow!("invalid exec endpoint"))?; + url.query_pairs_mut() + .append_pair("generation", &start.generation.to_string()); + let mut request = url + .as_str() + .into_client_request() + .map_err(|_| anyhow::anyhow!("invalid exec endpoint"))?; + request.headers_mut().insert(AUTHORIZATION, authorization); + request + .headers_mut() + .insert(SEC_WEBSOCKET_PROTOCOL, HeaderValue::from_static(SUBPROTOCOL)); + let config = WebSocketConfig::default() + .max_message_size(Some(MAX_CONTROL_BYTES)) + .max_frame_size(Some(MAX_CONTROL_BYTES)) + .write_buffer_size(0) + .max_write_buffer_size(MAX_CONTROL_BYTES + MAX_BINARY_BYTES + 1024); + let (mut socket, response) = + tokio::time::timeout(NETWORK_WAIT, connect_async_with_config(request, Some(config), true)) + .await + .map_err(|_| anyhow::anyhow!("exec connection timed out; the command was not replayed"))? + .map_err(|_| anyhow::anyhow!("exec connection was rejected or failed"))?; + ensure!( + response.headers().get_all(SEC_WEBSOCKET_PROTOCOL).iter().count() == 1 + && response + .headers() + .get(SEC_WEBSOCKET_PROTOCOL) + .is_some_and(|value| value == SUBPROTOCOL), + "server did not negotiate the exec protocol" + ); + send(&mut socket, text(&ClientControl::Start(start.clone()))?).await?; + tokio::time::timeout(NETWORK_WAIT, async { + loop { + match socket + .next() + .await + .context("exec closed before Ready; process outcome is unknown")? + .map_err(|_| anyhow::anyhow!("exec transport failed before Ready; process outcome is unknown"))? + { + Message::Text(value) => match ServerControl::decode(value.as_bytes())? { + ServerControl::Ready(ready) => { + ensure!( + ready.database_identity == identity + && ready.generation == start.generation + && ready.tty == start.terminal.is_some(), + "exec Ready did not match the selected instance" + ); + return Ok(()); + } + ServerControl::Error { .. } => bail!("container exec was rejected"), + _ => bail!("invalid exec response before Ready"), + }, + Message::Ping(bytes) => send(&mut socket, Message::Pong(bytes)).await?, + Message::Pong(_) => {} + _ => bail!("invalid exec response before Ready"), + } + } + }) + .await + .map_err(|_| anyhow::anyhow!("exec Ready timed out; process outcome is unknown"))??; + + let (mut sink, mut source) = socket.split(); + let (pong_tx, mut pong_rx) = mpsc::channel(1); + let input = &mut io.input; + let output = &io.output; + let reader = async { + while let Some(message) = source.next().await { + match message.map_err(|_| anyhow::anyhow!("exec transport failed; process outcome is unknown"))? { + Message::Binary(frame) => { + let (stream, bytes) = decode_output(&frame)?; + ensure!( + start.terminal.is_none() || stream == OutputStream::Stdout, + "unexpected PTY stderr channel" + ); + let (completed, written) = oneshot::channel(); + output + .send(Output { + stream, + bytes: bytes.to_vec(), + completed, + }) + .await + .context("terminal output stopped")?; + written.await.context("terminal output stopped")??; + } + Message::Text(value) => match ServerControl::decode(value.as_bytes())? { + ServerControl::Exit { exit_code } => { + return u8::try_from(exit_code).context("invalid observed exec exit code") + } + ServerControl::Error { .. } => bail!("container exec failed; process outcome is unknown"), + ServerControl::Ready(_) => bail!("duplicate exec Ready"), + }, + Message::Ping(bytes) => pong_tx.send(bytes).await.context("exec writer stopped")?, + Message::Pong(_) => {} + _ => bail!("exec closed without observed exit; process outcome is unknown"), + } + } + bail!("exec closed without observed exit; process outcome is unknown") + }; + let writer = async { + let mut stdin_open = start.stdin; + loop { + let message = tokio::select! { + value = input.recv(), if stdin_open => match value.context("terminal input stopped")? { + Input::Data(bytes) => Message::Binary(encode_data(OutputStream::Stdin, &bytes)?.into()), + Input::Eof => { stdin_open = false; text(&ClientControl::StdinEof)? }, + }, + value = signals.next() => text(&value.context("terminal signals stopped")??)?, + Some(bytes) = pong_rx.recv() => Message::Pong(bytes), + }; + send(&mut sink, message).await?; + } + #[allow(unreachable_code)] + Ok::(0) + }; + // These are owned, inline futures, not spawned tasks. The losing future and + // both socket halves are dropped before the caller joins its terminal worker. + tokio::select! { + result = reader => result, + result = writer => result, + result = &mut io.completion => { + result.context("terminal worker stopped")??; + bail!("terminal worker stopped before observed exec exit") + } + } +} + +fn text(value: &ClientControl) -> Result { + value.validate()?; + Ok(Message::Text(serde_json::to_string(value)?.into())) +} +async fn send(sink: &mut S, message: Message) -> Result<()> +where + S: futures::Sink + Unpin, +{ + tokio::time::timeout(NETWORK_WAIT, sink.send(message)) + .await + .map_err(|_| anyhow::anyhow!("exec write timed out; process outcome is unknown"))? + .map_err(|_| anyhow::anyhow!("exec write failed; process outcome is unknown")) +} diff --git a/crates/cli/src/subcommands/container/exec/terminal.rs b/crates/cli/src/subcommands/container/exec/terminal.rs new file mode 100644 index 00000000000..b0fa92e61ec --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal.rs @@ -0,0 +1,404 @@ +//! Standard descriptors share open-file descriptions with their duplicates. +//! Save all flags before changing any, and restore them only after the one I/O +//! worker has joined. Nonblocking writes are necessary even after select: another +//! writer can fill a pipe between readiness and write. Never use Tokio stdin's +//! uncancellable blocking worker here. +use super::session::{Input, Io, Output, OutputSender, OutputStream}; +use anyhow::{bail, ensure, Context, Result}; +use futures::{stream, Stream, StreamExt}; +use rustix::{ + event::{fd_set_insert, fd_set_num_elements, select, FdSetElement, Timespec}, + fd::{AsRawFd, OwnedFd}, + fs::{fcntl_getfl, fcntl_setfl, OFlags}, + io::{read, write, Errno}, + termios::{tcgetattr, tcgetpgrp, tcgetwinsize, tcsetattr, OptionalActions, Termios}, +}; +use spacetimedb_lib::container::exec::{ClientControl, TerminalSize, MAX_DATA_BYTES}; +use std::{ + os::unix::net::UnixStream, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread::JoinHandle, +}; +use tokio::sync::{mpsc, oneshot}; + +pub(super) fn dimensions() -> Result { + let stdin = std::io::stdin(); + let stdout = std::io::stdout(); + ensure!( + tcgetpgrp(&stdin).ok() == Some(rustix::process::getpgrp()) + && tcgetpgrp(&stdout).ok() == Some(rustix::process::getpgrp()), + "PTY requires a foreground-owned stdin and stdout terminal" + ); + let size = tcgetwinsize(&stdout).context("cannot read terminal dimensions")?; + let size = TerminalSize { + rows: size.ws_row, + columns: size.ws_col, + }; + size.validate().context("invalid terminal dimensions")?; + Ok(size) +} + +type Signals = Pin> + Send>>; +pub(super) fn signals(tty: bool) -> Result { + use tokio::signal::unix::{signal, SignalKind}; + let mappings = [ + (SignalKind::hangup(), 1), + (SignalKind::interrupt(), 2), + (SignalKind::quit(), 3), + (SignalKind::terminate(), 15), + (SignalKind::user_defined1(), 10), + (SignalKind::user_defined2(), 12), + ]; + let mut streams: Vec = Vec::new(); + for (kind, linux) in mappings { + let signal = signal(kind).context("cannot install exec signal handler")?; + streams.push(Box::pin(stream::unfold(signal, move |mut signal| async move { + signal.recv().await.map(|()| (Ok(ClientControl::Signal(linux)), signal)) + }))); + } + if tty { + let signal = signal(SignalKind::window_change()).context("cannot install terminal resize handler")?; + streams.push(Box::pin(stream::unfold(signal, |mut signal| async move { + signal + .recv() + .await + .map(|()| (dimensions().map(ClientControl::Resize), signal)) + }))); + } + Ok(stream::select_all(streams).boxed()) +} + +struct Descriptors { + files: [OwnedFd; 3], + flags: [OFlags; 3], + terminal: Option, + restored: bool, +} +impl Descriptors { + fn prepare(files: [OwnedFd; 3], tty: bool) -> Result { + let flags = [ + fcntl_getfl(&files[0])?, + fcntl_getfl(&files[1])?, + fcntl_getfl(&files[2])?, + ]; + let mut this = Self { + files, + flags, + terminal: None, + restored: false, + }; + if tty { + ensure!( + tcgetpgrp(&this.files[0]).ok() == Some(rustix::process::getpgrp()) + && tcgetpgrp(&this.files[1]).ok() == Some(rustix::process::getpgrp()), + "PTY requires a foreground-owned terminal" + ); + let original = tcgetattr(&this.files[0])?; + let mut raw = original.clone(); + raw.make_raw(); + this.terminal = Some(original); + tcsetattr(&this.files[0], OptionalActions::Now, &raw)?; + } + for (fd, flags) in this.files.iter().zip(this.flags) { + fcntl_setfl(fd, flags | OFlags::NONBLOCK).context("cannot enable cancellable terminal I/O")?; + } + Ok(this) + } + fn restore(&mut self) -> Result<()> { + if self.restored { + return Ok(()); + } + let mut failed = false; + if let Some(terminal) = &self.terminal { + failed |= tcsetattr(&self.files[0], OptionalActions::Now, terminal).is_err(); + } + for (fd, flags) in self.files.iter().zip(self.flags) { + failed |= fcntl_setfl(fd, flags).is_err(); + } + self.restored = !failed; + ensure!(!failed, "could not restore terminal settings or descriptor flags"); + Ok(()) + } +} +impl Drop for Descriptors { + fn drop(&mut self) { + let _ = self.restore(); + } +} + +pub(super) struct Terminal { + stop: Arc, + wake: Arc, + worker: Option>>, +} +impl Terminal { + pub(super) fn stdio(stdin: bool, tty: bool) -> Result<(Self, Io)> { + Self::open( + [ + rustix::io::fcntl_dupfd_cloexec(std::io::stdin(), 3)?, + rustix::io::fcntl_dupfd_cloexec(std::io::stdout(), 3)?, + rustix::io::fcntl_dupfd_cloexec(std::io::stderr(), 3)?, + ], + stdin, + tty, + ) + } + fn open(files: [OwnedFd; 3], stdin: bool, tty: bool) -> Result<(Self, Io)> { + let mut descriptors = Descriptors::prepare(files, tty)?; + let (wake, wake_reader) = UnixStream::pair()?; + wake.set_nonblocking(true)?; + wake_reader.set_nonblocking(true)?; + let wake = Arc::new(wake); + let stop = Arc::new(AtomicBool::new(false)); + let (input_tx, input) = mpsc::channel(1); + let (output_tx, output) = mpsc::channel(1); + let (completed, completion) = oneshot::channel(); + let stopped = stop.clone(); + let worker = std::thread::Builder::new() + .name("container-exec-terminal".into()) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + pump(&descriptors.files, &wake_reader, stopped, stdin, input_tx, output) + })) + .unwrap_or_else(|_| Err(anyhow::anyhow!("terminal I/O worker panicked"))); + let restored = descriptors.restore(); + let result = match (result, restored) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(error), Err(restored)) => { + Err(error.context(format!("terminal restoration also failed: {restored}"))) + } + }; + let _ = completed.send( + result + .as_ref() + .map(|_| ()) + .map_err(|_| anyhow::anyhow!("terminal I/O failed")), + ); + result + }) + .context("cannot start terminal worker")?; + let notification = wake.clone(); + let output = OutputSender { + sender: output_tx, + wake: Some(Arc::new(move || { + let _ = write(&*notification, &[1]); + })), + }; + Ok(( + Self { + stop, + wake, + worker: Some(worker), + }, + Io { + input, + output, + completion, + }, + )) + } + pub(super) fn finish(&mut self) -> Result<()> { + self.stop.store(true, Ordering::Release); + let _ = write(&*self.wake, &[1]); + if let Some(worker) = self.worker.take() { + worker + .join() + .map_err(|_| anyhow::anyhow!("terminal worker join failed"))??; + } + Ok(()) + } +} +impl Drop for Terminal { + fn drop(&mut self) { + let _ = self.finish(); + } +} + +fn pump( + files: &[OwnedFd; 3], + wake: &UnixStream, + stop: Arc, + mut stdin: bool, + input: mpsc::Sender, + mut output: mpsc::Receiver, +) -> Result<()> { + let mut incoming = None; + let mut outgoing: Option<(Output, usize)> = None; + let mut bytes = vec![0; MAX_DATA_BYTES]; + while !stop.load(Ordering::Acquire) { + if outgoing.is_none() { + outgoing = output.try_recv().ok().map(|value| (value, 0)); + } + let mut progress = false; + if let Some((value, offset)) = &mut outgoing { + let fd = match value.stream { + OutputStream::Stdout => &files[1], + OutputStream::Stderr => &files[2], + _ => bail!("invalid terminal output channel"), + }; + match write(fd, &value.bytes[*offset..]) { + Ok(0) => bail!("terminal output closed"), + Ok(count) => { + *offset += count; + progress = true; + } + Err(Errno::AGAIN | Errno::INTR) => {} + Err(_) => bail!("terminal output failed"), + } + if *offset == value.bytes.len() { + let (value, _) = outgoing.take().unwrap(); + let _ = value.completed.send(Ok(())); + } + } + if stdin && incoming.is_none() { + match read(&files[0], &mut bytes) { + Ok(0) => { + incoming = Some(Input::Eof); + stdin = false; + } + Ok(count) => { + incoming = Some(Input::Data(bytes[..count].to_vec())); + progress = true; + } + Err(Errno::AGAIN | Errno::INTR) => {} + Err(_) => bail!("terminal input failed"), + } + } + if let Some(value) = incoming.take() { + match input.try_send(value) { + Ok(()) => progress = true, + Err(mpsc::error::TrySendError::Full(value)) => incoming = Some(value), + Err(mpsc::error::TrySendError::Closed(_)) => bail!("terminal input consumer stopped"), + } + } + if progress { + continue; + } + let write_fd = outgoing + .as_ref() + .map(|(value, _)| files[if value.stream == OutputStream::Stdout { 1 } else { 2 }].as_raw_fd()); + let read_fd = (stdin && incoming.is_none()).then(|| files[0].as_raw_fd()); + let max = [Some(wake.as_raw_fd()), read_fd, write_fd] + .into_iter() + .flatten() + .max() + .unwrap() + + 1; + let elements = fd_set_num_elements(max as usize, 3); + let mut reads = vec![FdSetElement::default(); elements]; + let mut writes = reads.clone(); + fd_set_insert(&mut reads, wake.as_raw_fd()); + if let Some(fd) = read_fd { + fd_set_insert(&mut reads, fd); + } + if let Some(fd) = write_fd { + fd_set_insert(&mut writes, fd); + } + // select supports macOS terminals, unlike poll. Every registered FD is + // owned for the worker's entire lifetime; the sets use its exact bound. + let result = unsafe { + select( + max, + Some(&mut reads), + Some(&mut writes), + None, + Some(&Timespec { + tv_sec: 0, + tv_nsec: 20_000_000, + }), + ) + }; + if let Err(error) = result + && error != Errno::INTR + { + bail!("terminal readiness failed"); + } + let mut notifications = [0; 64]; + while read(wake, &mut notifications).is_ok_and(|count| count != 0) {} + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{Duration, Instant}; + + #[tokio::test] + async fn cancelled_backpressured_terminal_joins_and_restores_shared_descriptor_flags() { + let (input, _input_peer) = UnixStream::pair().unwrap(); + let (output, _output_peer) = UnixStream::pair().unwrap(); + output.set_nonblocking(true).unwrap(); + let block = [0; 8192]; + while write(&output, &block).is_ok() {} + output.set_nonblocking(false).unwrap(); + let input_flags = fcntl_getfl(&input).unwrap(); + let output_flags = fcntl_getfl(&output).unwrap(); + let (mut terminal, io) = Terminal::open( + [ + rustix::io::dup(&input).unwrap(), + rustix::io::dup(&output).unwrap(), + rustix::io::dup(&output).unwrap(), + ], + true, + false, + ) + .unwrap(); + assert!(fcntl_getfl(&output).unwrap().contains(OFlags::NONBLOCK)); + let (completed, mut written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stdout, + bytes: vec![42; MAX_DATA_BYTES], + completed, + }) + .await + .unwrap(); + if let Some(wake) = &io.output.wake { + wake(); + } + assert!(tokio::time::timeout(Duration::from_millis(30), &mut written) + .await + .is_err()); + let started = Instant::now(); + terminal.finish().unwrap(); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(terminal.worker.is_none()); + assert_eq!(fcntl_getfl(&input).unwrap(), input_flags); + assert_eq!(fcntl_getfl(&output).unwrap(), output_flags); + io.completion.await.unwrap().unwrap(); + assert!(written.await.is_err()); + terminal.finish().unwrap(); + } + + #[tokio::test] + async fn terminal_drop_joins_idle_input_and_refuses_unowned_pty_without_flag_changes() { + let (input, _input_peer) = UnixStream::pair().unwrap(); + let (output, _output_peer) = UnixStream::pair().unwrap(); + let flags = fcntl_getfl(&input).unwrap(); + let files = || { + [ + rustix::io::dup(&input).unwrap(), + rustix::io::dup(&output).unwrap(), + rustix::io::dup(&output).unwrap(), + ] + }; + assert!(Terminal::open(files(), true, true).is_err()); + assert_eq!(fcntl_getfl(&input).unwrap(), flags); + let (terminal, io) = Terminal::open(files(), true, false).unwrap(); + drop(terminal); + io.completion.await.unwrap().unwrap(); + assert_eq!(fcntl_getfl(&input).unwrap(), flags); + assert!(!fcntl_getfl(&output).unwrap().contains(OFlags::NONBLOCK)); + } +} + +#[cfg(test)] +#[path = "terminal_pty_tests.rs"] +mod pty_tests; diff --git a/crates/cli/src/subcommands/container/exec/terminal_pty_tests.rs b/crates/cli/src/subcommands/container/exec/terminal_pty_tests.rs new file mode 100644 index 00000000000..c9aa52e60b4 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_pty_tests.rs @@ -0,0 +1,163 @@ +//! A child session owns its own controlling PTY; the test runner's terminal and +//! process group are never modified. The parent retains and waits every child. +use super::*; +use rustix::{ + fs::{open, Mode}, + pty::{grantpt, openpt, ptsname, unlockpt, OpenptFlags}, + termios::{tcsetwinsize, LocalModes, Winsize}, +}; +use std::{ + fs::File, + process::{Child, Command, Stdio}, + time::{Duration, Instant}, +}; + +struct ChildOwner(Child); +impl Drop for ChildOwner { + fn drop(&mut self) { + if self.0.try_wait().ok().flatten().is_none() { + let _ = self.0.kill(); + } + let _ = self.0.wait(); + } +} + +#[test] +fn foreground_pty_restores_after_normal_error_and_cancel() { + for mode in ["normal", "error", "cancel"] { + let master = openpt(OpenptFlags::RDWR | OpenptFlags::NOCTTY).unwrap(); + rustix::io::fcntl_setfd(&master, rustix::io::FdFlags::CLOEXEC).unwrap(); + grantpt(&master).unwrap(); + unlockpt(&master).unwrap(); + let path = ptsname(&master, Vec::new()).unwrap(); + let slave = open(&path, OFlags::RDWR | OFlags::NOCTTY | OFlags::CLOEXEC, Mode::empty()).unwrap(); + tcsetwinsize( + &slave, + Winsize { + ws_row: 48, + ws_col: 120, + ws_xpixel: 0, + ws_ypixel: 0, + }, + ) + .unwrap(); + // Prime Darwin's kernel FWASWRITTEN state before the snapshot; + // the child test harness writes to this same open-file description. + assert_eq!(write(&slave, b"owned PTY fixture\n").unwrap(), 18); + let before = configuration(tcgetattr(&slave).unwrap()); + let flags = fcntl_getfl(&slave).unwrap(); + let root = tempfile::tempdir().unwrap(); + let file: File = rustix::io::dup(&slave).unwrap().into(); + let child = Command::new(std::env::current_exe().unwrap()) + .args([ + "subcommands::container::execute::terminal::pty_tests::foreground_pty_child", + "--exact", + "--ignored", + "--nocapture", + "--test-threads=1", + ]) + .env_clear() + .env("SPACETIMEDB_EXEC_PTY_CHILD", mode) + .env("TMPDIR", root.path()) + .current_dir(root.path()) + .stdin(Stdio::from(file.try_clone().unwrap())) + .stdout(Stdio::from(file.try_clone().unwrap())) + .stderr(Stdio::from(file)) + .spawn() + .unwrap(); + let mut child = ChildOwner(child); + fcntl_setfl(&master, fcntl_getfl(&master).unwrap() | OFlags::NONBLOCK).unwrap(); + let deadline = Instant::now() + Duration::from_secs(10); + let mut captured = Vec::new(); + let mut verified = false; + let mut buffer = [0; 4096]; + let status = loop { + while let Ok(count) = read(&master, &mut buffer) { + if count == 0 { + break; + } + captured.extend_from_slice(&buffer[..count]); + assert!(captured.len() < 64 * 1024); + } + if !verified + && captured + .windows(b"PTY_RESTORED".len()) + .any(|text| text == b"PTY_RESTORED") + { + assert_eq!(configuration(tcgetattr(&slave).unwrap()), before, "mode {mode}"); + assert_eq!(fcntl_getfl(&slave).unwrap(), flags, "mode {mode}"); + assert_eq!(write(&master, b"y\n").unwrap(), 2); + verified = true; + } + if let Some(status) = child.0.try_wait().unwrap() { + break status; + } + assert!(Instant::now() < deadline, "owned PTY child timed out"); + std::thread::sleep(Duration::from_millis(5)); + }; + // Keep the session leader alive for the independent slave inspection: + // macOS revokes that slave when the controlling session exits. + // Then positively wait, even after try_wait. + child.0.wait().unwrap(); + assert!(status.success(), "mode {mode}: {}", String::from_utf8_lossy(&captured)); + assert!(verified, "mode {mode} did not acknowledge restored state"); + assert_eq!(fcntl_getfl(&slave).unwrap(), flags, "mode {mode}"); + } +} + +#[tokio::test] +#[ignore = "owned child of foreground_pty_restores_after_normal_error_and_cancel"] +async fn foreground_pty_child() { + let mode = std::env::var("SPACETIMEDB_EXEC_PTY_CHILD").expect("requires owned PTY parent"); + assert!(["normal", "error", "cancel"].contains(&mode.as_str())); + rustix::process::setsid().unwrap(); + rustix::process::ioctl_tiocsctty(std::io::stdin()).unwrap(); + rustix::termios::tcsetpgrp(std::io::stdin(), rustix::process::getpgrp()).unwrap(); + assert_eq!(dimensions().unwrap(), TerminalSize { rows: 48, columns: 120 }); + let before = configuration(tcgetattr(std::io::stdin()).unwrap()); + let flags = fcntl_getfl(std::io::stdin()).unwrap(); + let (mut terminal, io) = Terminal::stdio(true, true).unwrap(); + assert!(!tcgetattr(std::io::stdin()) + .unwrap() + .local_modes + .intersects(LocalModes::ICANON | LocalModes::ECHO)); + match mode.as_str() { + "normal" => terminal.finish().unwrap(), + "cancel" => drop(terminal), + "error" => { + let (completed, _written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stdin, + bytes: vec![1], + completed, + }) + .await + .unwrap(); + if let Some(wake) = &io.output.wake { + wake(); + } + assert!(io.completion.await.unwrap().is_err()); + assert!(terminal.finish().is_err()); + } + _ => unreachable!(), + } + assert_eq!(configuration(tcgetattr(std::io::stdin()).unwrap()), before); + assert_eq!(fcntl_getfl(std::io::stdin()).unwrap(), flags); + use std::io::{Read, Write}; + println!("PTY_RESTORED"); + std::io::stdout().flush().unwrap(); + let mut ack = [0]; + std::io::stdin().read_exact(&mut ack).unwrap(); + assert_eq!(ack, [b'y']); +} + +fn configuration(mut termios: Termios) -> String { + // PENDIN is queued-input state, not a configured terminal mode. macOS + // sets it when ICANON is restored, even for an empty queue. Do not flush + // user input or change production restoration to erase this kernel state. + // https://github.com/apple-oss-distributions/xnu/blob/main/bsd/kern/tty.c + termios.local_modes.remove(LocalModes::PENDIN); + format!("{termios:?}") +} diff --git a/crates/cli/src/subcommands/container/exec/tests.rs b/crates/cli/src/subcommands/container/exec/tests.rs new file mode 100644 index 00000000000..f962848cf25 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/tests.rs @@ -0,0 +1,405 @@ +use super::session::{Input, Io, OutputSender}; +use super::*; +use futures::{SinkExt, StreamExt}; +use reqwest::{header::HeaderValue, Url}; +use spacetimedb_lib::{ + container::exec::{self as wire, ClientControl, ExecReady, ServerControl}, + Identity, Uuid, +}; +use tokio::{ + net::TcpListener, + sync::{mpsc, oneshot}, +}; +use tokio_tungstenite::tungstenite::{ + handshake::server::{Request, Response}, + Message, +}; + +fn arguments(values: &[&str]) -> Result { + cli().try_get_matches_from(values) +} +#[test] +fn literal_arguments_and_environment_are_bounded_and_redacted() { + cli().help_expected(true).debug_assert(); + let args = arguments(&[ + "exec", + "db", + "-i", + "--workdir", + "/a b", + "-e", + "UNICODE=🦀=literal", + "--", + "/bin/echo", + "$(no-shell)", + "--flag", + ]) + .unwrap(); + let value = start(&args, 42).unwrap(); + assert_eq!(value.argv, ["/bin/echo", "$(no-shell)", "--flag"]); + assert_eq!(value.environment["UNICODE"], "🦀=literal"); + assert_eq!(value.working_directory.as_deref(), Some("/a b")); + for invalid in ["SPACETIMEDB_TOKEN=secret-value", "BAD-NAME=secret-value", "NO_EQUALS"] { + let args = arguments(&["exec", "db", "-e", invalid, "--", "true"]).unwrap(); + let error = start(&args, 1).unwrap_err().to_string(); + assert!(!error.contains("secret-value")); + } + let args = arguments(&["exec", "db", "-e", "A=1", "-e", "A=2", "--", "true"]).unwrap(); + assert!(start(&args, 1).is_err()); + assert!(arguments(&["exec", "db", "-t", "--", "true"]).is_err()); + let args = arguments(&["exec", "db", "--workdir", "relative", "--", "true"]).unwrap(); + assert!(start(&args, 1).is_err()); +} + +#[tokio::test] +async fn real_loopback_socket_preserves_binary_eof_controls_and_exit() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!( + "http://{}/v1/database/{}/container/exec", + listener.local_addr().unwrap(), + Identity::ZERO.to_hex() + ); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_hdr_async(stream, |request: &Request, mut response: Response| { + assert_eq!(request.uri().query(), Some("generation=42")); + assert_eq!(request.headers()["authorization"], "Bearer owned-loopback-fixture"); + assert_eq!(request.headers()["sec-websocket-protocol"], wire::SUBPROTOCOL); + assert!(!request.uri().to_string().contains("owned-loopback-fixture")); + response + .headers_mut() + .insert("sec-websocket-protocol", HeaderValue::from_static(wire::SUBPROTOCOL)); + Ok(response) + }) + .await + .unwrap(); + let first = socket.next().await.unwrap().unwrap(); + let ClientControl::Start(start) = ClientControl::decode(first.into_data().as_ref()).unwrap() else { + panic!("missing Start") + }; + assert_eq!(start.argv, ["/bin/echo", "literal"]); + let ready = ServerControl::Ready(ExecReady { + database_identity: Identity::ZERO, + generation: 42, + session_id: Uuid::from_u128(7), + tty: false, + }); + socket + .send(Message::Text(serde_json::to_string(&ready).unwrap().into())) + .await + .unwrap(); + let mut data = false; + let mut eof = false; + let mut signal = false; + let mut resize = false; + while !(data && eof && signal && resize) { + match socket.next().await.unwrap().unwrap() { + Message::Binary(value) => { + assert_eq!(wire::decode_stdin(&value).unwrap(), &[0, 255, 1]); + data = true; + } + Message::Text(value) => match ClientControl::decode(value.as_bytes()).unwrap() { + ClientControl::StdinEof => { + assert!(!eof); + eof = true; + } + ClientControl::Signal(10) => signal = true, + ClientControl::Resize(wire::TerminalSize { rows: 24, columns: 80 }) => resize = true, + _ => panic!("unexpected control"), + }, + _ => panic!("unexpected frame"), + } + } + for stream in [wire::Stream::Stdout, wire::Stream::Stderr] { + socket + .send(Message::Binary( + wire::encode_data(stream, &[255, 0, 128]).unwrap().into(), + )) + .await + .unwrap(); + } + socket + .send(Message::Text( + serde_json::to_string(&ServerControl::Exit { exit_code: 37 }) + .unwrap() + .into(), + )) + .await + .unwrap(); + }); + let (input_tx, input) = mpsc::channel(2); + input_tx.send(Input::Data(vec![0, 255, 1])).await.unwrap(); + input_tx.send(Input::Eof).await.unwrap(); + let (output_tx, mut output) = mpsc::channel::(1); + let (_completed, completion) = oneshot::channel(); + let mut io = Io { + input, + output: OutputSender { + sender: output_tx, + wake: None, + }, + completion, + }; + let options = start( + &arguments(&["exec", "db", "-i", "--", "/bin/echo", "literal"]).unwrap(), + 42, + ) + .unwrap(); + let signals = futures::stream::iter([ + Ok(ClientControl::Signal(10)), + Ok(ClientControl::Resize(wire::TerminalSize { rows: 24, columns: 80 })), + ]) + .chain(futures::stream::pending()); + let client = session::run( + Url::parse(&origin).unwrap(), + HeaderValue::from_static("Bearer owned-loopback-fixture"), + Identity::ZERO, + options, + &mut io, + signals, + ); + let output_reader = async { + let mut seen = Vec::new(); + for _ in 0..2 { + let value = output.recv().await.unwrap(); + assert_eq!(value.bytes, [255, 0, 128]); + seen.push(value.stream); + value.completed.send(Ok(())).unwrap(); + } + assert_eq!(seen, [wire::Stream::Stdout, wire::Stream::Stderr]); + }; + let results = tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::join!(client, output_reader) + }) + .await; + if results.is_err() { + server.abort(); + } + let joined = server.await; + assert!(joined.is_ok()); + assert_eq!(results.unwrap().0.unwrap(), 37); +} + +#[test] +fn status_requires_the_exact_running_generation_without_http_readiness() { + use spacetimedb_lib::container::operations::*; + let mut status = ContainerStatus { + database_identity: Identity::ZERO, + published: true, + endpoints: EndpointStatus::Pending, + operational: Some(OperationalState { + desired_revision: spacetimedb_lib::Hash::from_byte_array([0; 32]), + desired_state: DesiredState::Running, + generation: 42, + condition: Condition::None, + restart_pending: false, + restart_attempt: 0, + restart_not_before_ms: 0, + current_instance: Some(CurrentInstance { + generation: 42, + state: ObservedState::Running, + observed_revision: None, + applied_env_generation: None, + exit_code: None, + oom_killed: false, + condition: Condition::None, + }), + }), + }; + assert_eq!(running_generation(&status).unwrap(), 42); + for state in [ + ObservedState::Pending, + ObservedState::Starting, + ObservedState::Draining, + ObservedState::Stopped, + ObservedState::Completed, + ObservedState::Failed, + ] { + status + .operational + .as_mut() + .unwrap() + .current_instance + .as_mut() + .unwrap() + .state = state; + assert!(running_generation(&status).is_err()); + } + status + .operational + .as_mut() + .unwrap() + .current_instance + .as_mut() + .unwrap() + .state = ObservedState::Ready; + assert_eq!(running_generation(&status).unwrap(), 42); + status + .operational + .as_mut() + .unwrap() + .current_instance + .as_mut() + .unwrap() + .generation = 41; + assert!(running_generation(&status).is_err()); + status + .operational + .as_mut() + .unwrap() + .current_instance + .as_mut() + .unwrap() + .generation = 42; + status.operational.as_mut().unwrap().desired_state = DesiredState::Stopped; + assert!(running_generation(&status).is_err()); +} + +#[tokio::test] +async fn real_loopback_rejects_bad_bindings_order_and_unobserved_exit_without_replay() { + for case in 0..8 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = Url::parse(&format!( + "http://{}/v1/database/{}/container/exec", + listener.local_addr().unwrap(), + Identity::ZERO.to_hex() + )) + .unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_hdr_async(stream, move |_: &Request, mut response: Response| { + if case != 0 { + response + .headers_mut() + .insert("sec-websocket-protocol", HeaderValue::from_static(wire::SUBPROTOCOL)); + } + Ok(response) + }) + .await + .unwrap(); + if case != 0 { + let first = socket.next().await.unwrap().unwrap(); + assert!(matches!( + ClientControl::decode(first.into_data().as_ref()).unwrap(), + ClientControl::Start(_) + )); + let ready = ServerControl::Ready(ExecReady { + database_identity: if case == 1 { + Identity::from_byte_array([1; 32]) + } else { + Identity::ZERO + }, + generation: if case == 2 { 43 } else { 42 }, + session_id: Uuid::from_u128(if case == 3 { 0 } else { 7 }), + tty: case == 4, + }); + if case == 5 { + socket + .send(Message::Binary( + wire::encode_data(wire::Stream::Stdout, b"premature").unwrap().into(), + )) + .await + .unwrap(); + } else { + socket + .send(Message::Text(serde_json::to_string(&ready).unwrap().into())) + .await + .unwrap(); + if case == 6 { + socket + .send(Message::Text(serde_json::to_string(&ready).unwrap().into())) + .await + .unwrap(); + } + if case == 7 { + socket.close(None).await.unwrap(); + } + } + } + // The client must close this exact connection and never dial a retry. + let closed = tokio::time::timeout(std::time::Duration::from_secs(2), socket.next()).await; + assert!(matches!(closed, Ok(None | Some(Err(_)) | Some(Ok(Message::Close(_)))))); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), listener.accept()) + .await + .is_err() + ); + }); + let (_input_tx, input) = mpsc::channel(1); + let (output_tx, _output) = mpsc::channel(1); + let (_completed, completion) = oneshot::channel(); + let mut io = Io { + input, + output: OutputSender { + sender: output_tx, + wake: None, + }, + completion, + }; + let options = start(&arguments(&["exec", "db", "--", "true"]).unwrap(), 42).unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(4), + session::run( + url, + HeaderValue::from_static("Bearer owned-loopback-fixture"), + Identity::ZERO, + options, + &mut io, + futures::stream::pending(), + ), + ) + .await; + if result.is_err() { + server.abort(); + } + let joined = server.await; + assert!(joined.is_ok(), "case {case}"); + assert!(result.unwrap().is_err(), "case {case}"); + } +} + +#[tokio::test] +async fn status_uses_authenticated_explicit_loopback_and_pins_the_resolved_identity() { + use spacetimedb_lib::container::operations::EndpointStatus; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = Url::parse(&format!("http://{}", listener.local_addr().unwrap())).unwrap(); + let identity = Identity::from_byte_array([7; 32]); + let status = ContainerStatus { + database_identity: identity, + published: true, + operational: None, + endpoints: EndpointStatus::Pending, + }; + let router = axum::Router::new().route( + "/v1/database/owned-test-db/container/status", + axum::routing::get(move |headers: axum::http::HeaderMap| async move { + assert_eq!(headers["authorization"], "Bearer owned-loopback-fixture"); + axum::Json(status) + }), + ); + let (stop, stopped) = oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + let client = super::super::operations::ContainerClient::new( + origin, + HeaderValue::from_static("Bearer owned-loopback-fixture"), + ) + .unwrap(); + let result = client.status("owned-test-db").await; + let _ = stop.send(()); + let joined = server.await.unwrap(); + joined.unwrap(); + let status = result.unwrap(); + assert_eq!(status.database_identity, identity); + let endpoint = client.url(status.database_identity.to_hex().as_ref(), "exec").unwrap(); + assert_eq!( + endpoint.path(), + format!("/v1/database/{}/container/exec", identity.to_hex()) + ); + assert!(running_generation(&status).is_err()); +} diff --git a/crates/cli/src/subcommands/container/operations.rs b/crates/cli/src/subcommands/container/operations.rs index 3236baca42d..7f12a4ade6a 100644 --- a/crates/cli/src/subcommands/container/operations.rs +++ b/crates/cli/src/subcommands/container/operations.rs @@ -120,13 +120,13 @@ pub(super) async fn exec(config: &mut crate::Config, name: &str, args: &ArgMatch Ok(()) } -struct ContainerClient { +pub(super) struct ContainerClient { http: Client, server: Url, authorization: HeaderValue, } impl ContainerClient { - fn new(server: Url, mut authorization: HeaderValue) -> Result { + pub(super) fn new(server: Url, mut authorization: HeaderValue) -> Result { ensure!( authorization.as_bytes().starts_with(b"Bearer "), "container operations require ordinary Bearer authentication" @@ -144,7 +144,7 @@ impl ContainerClient { authorization, }) } - fn url(&self, database: &str, operation: &str) -> Result { + pub(super) fn url(&self, database: &str, operation: &str) -> Result { validate_database(database)?; let mut url = self.server.clone(); url.path_segments_mut() @@ -153,7 +153,7 @@ impl ContainerClient { .extend(["v1", "database", database, "container", operation]); Ok(url) } - async fn status(&self, database: &str) -> Result { + pub(super) async fn status(&self, database: &str) -> Result { let response = self .http .get(self.url(database, "status")?) diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 7be9b57817d..101edcf2e64 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -31,6 +31,7 @@ This document contains the help content for the `spacetime` command-line program * [`spacetime build`↴](#spacetime-build) * [`spacetime container`↴](#spacetime-container) * [`spacetime container url`↴](#spacetime-container-url) +* [`spacetime container exec`↴](#spacetime-container-exec) * [`spacetime container build`↴](#spacetime-container-build) * [`spacetime container status`↴](#spacetime-container-status) * [`spacetime container start`↴](#spacetime-container-start) @@ -594,6 +595,7 @@ Build and manage a database's container ###### **Subcommands:** * `url` — Print a container's published HTTPS URL +* `exec` — Run a literal command in the current running container * `build` — Prepare verified OCI artifacts locally without publishing * `status` — Inspect container control state without opening its database * `start` — Request container execution @@ -621,6 +623,30 @@ Discovery does not start the container or wait for readiness. No login is requir +## `spacetime container exec` + +Run a literal command in the current running container + +**Usage:** `spacetime container exec [OPTIONS] -- ...` + +Requires database Admin permission. No shell, container start, or reconnect is implicit. Use -- before COMMAND; for a shell, name its executable explicitly. Linux and macOS terminals are currently supported. A lost connection does not establish that the process exited. + +###### **Arguments:** + +* `` — Database name or Identity +* `` — Executable and literal arguments; no shell expansion + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `-i`, `--interactive` — Forward stdin and send EOF when it closes +* `-t`, `--tty` — Allocate a PTY using this foreground terminal's dimensions +* `--workdir ` — Absolute working directory inside the container +* `-e`, `--env ` — Override a process environment variable; platform keys are reserved + + + ## `spacetime container build` Prepare verified OCI artifacts locally without publishing From cadac7192aaa96372b3c92f88e9bc2527076d7da Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 07:32:53 -0400 Subject: [PATCH 06/16] Support container exec on Windows with owned terminal I/O --- .github/workflows/ci.yml | 14 + crates/cli/Cargo.toml | 5 +- crates/cli/src/subcommands/container/exec.rs | 18 +- .../src/subcommands/container/exec/session.rs | 36 +- .../container/exec/terminal_windows.rs | 373 ++++++++++++++ .../exec/terminal_windows/console_tests.rs | 370 ++++++++++++++ .../exec/terminal_windows/control.rs | 97 ++++ .../container/exec/terminal_windows/io.rs | 344 +++++++++++++ .../container/exec/terminal_windows/tests.rs | 471 ++++++++++++++++++ .../src/subcommands/container/exec/tests.rs | 82 +++ .../00100-cli-reference.md | 2 +- tools/ci/windows-exec-tests.ps1 | 192 +++++++ 12 files changed, 1992 insertions(+), 12 deletions(-) create mode 100644 crates/cli/src/subcommands/container/exec/terminal_windows.rs create mode 100644 crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs create mode 100644 crates/cli/src/subcommands/container/exec/terminal_windows/control.rs create mode 100644 crates/cli/src/subcommands/container/exec/terminal_windows/io.rs create mode 100644 crates/cli/src/subcommands/container/exec/terminal_windows/tests.rs create mode 100644 tools/ci/windows-exec-tests.ps1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f7a1d9ee0c..2de05d409cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,6 +231,20 @@ jobs: mkdir -p "${openssl_cache_dir}" cp -R "${openssl_installs[0]}/." "${openssl_cache_dir}/" + - name: Test Windows container exec + if: runner.os == 'Windows' + shell: pwsh + run: ./tools/ci/windows-exec-tests.ps1 + + - name: Upload Windows container exec test results + if: always() && runner.os == 'Windows' && env.WINDOWS_EXEC_RESULTS != '' + uses: actions/upload-artifact@v4 + with: + name: windows-container-exec-results + path: ${{ env.WINDOWS_EXEC_RESULTS }} + if-no-files-found: error + retention-days: 14 + - name: Package build artifacts shell: bash run: | diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d76ebf60559..6f951afd920 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -102,7 +102,10 @@ rustix = { version = "1", features = ["fs", "process", "termios", "event"] } rustix = { version = "1", features = ["pty"] } [target.'cfg(windows)'.dependencies] -windows-sys = { workspace = true, features = ["Win32_System_Console", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Threading", "Win32_System_SystemServices"] } +windows-sys = { workspace = true, features = ["Win32_System_Console", "Win32_Security_Authorization", "Win32_Storage_FileSystem", "Win32_System_Threading", "Win32_System_SystemServices", "Win32_System_IO", "Wdk_Storage_FileSystem"] } + +[target.'cfg(windows)'.dev-dependencies] +windows-sys = { workspace = true, features = ["Win32_System_Pipes"] } [build-dependencies] serde = { workspace = true, features = ["derive"] } diff --git a/crates/cli/src/subcommands/container/exec.rs b/crates/cli/src/subcommands/container/exec.rs index 8dfd976b547..71f8768812b 100644 --- a/crates/cli/src/subcommands/container/exec.rs +++ b/crates/cli/src/subcommands/container/exec.rs @@ -4,6 +4,9 @@ mod session; #[cfg(any(target_os = "linux", target_os = "macos"))] #[path = "exec/terminal.rs"] mod terminal; +#[cfg(windows)] +#[path = "exec/terminal_windows.rs"] +mod terminal; #[cfg(test)] #[path = "exec/tests.rs"] mod tests; @@ -19,7 +22,7 @@ use std::collections::BTreeMap; pub(super) fn cli() -> Command { Command::new("exec") .about("Run a literal command in the current running container") - .after_help("Requires database Admin permission. No shell, container start, or reconnect is implicit. Use -- before COMMAND; for a shell, name its executable explicitly. Linux and macOS terminals are currently supported. A lost connection does not establish that the process exited.") + .after_help("Requires database Admin permission. No shell, container start, or reconnect is implicit. Use -- before COMMAND; for a shell, name its executable explicitly. Linux, macOS, and Windows terminals are supported. Windows requires an attached VT-capable console for --tty; inherited asynchronous seekable files are unsupported. A lost connection does not establish that the process exited.") .arg(Arg::new("database").required(true).help("Database name or Identity")) .arg(crate::common_args::server()) .arg(crate::common_args::yes()) @@ -78,17 +81,20 @@ fn running_generation(status: &ContainerStatus) -> Result { pub(super) async fn exec(config: &mut crate::Config, args: &ArgMatches) -> Result<()> { // Reject unsupported terminal implementations before login, status, or a socket. - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] { let _ = (config, args); anyhow::bail!("container exec terminal support is not implemented on this host platform"); } - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "linux", target_os = "macos", windows))] { let mut start = start(args, 1)?; if args.get_flag("tty") { start.terminal = Some(terminal::dimensions()?); } + // Classify inherited Windows handles before any login or network I/O. + #[cfg(windows)] + let prepared = terminal::Prepared::stdio(start.stdin, start.terminal.is_some())?; let selection = args.get_one::("server").map(String::as_str); let origin = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; @@ -100,8 +106,14 @@ pub(super) async fn exec(config: &mut crate::Config, args: &ArgMatches) -> Resul .await?; start.generation = running_generation(&status)?; let url = client.url(status.database_identity.to_hex().as_ref(), "exec")?; + #[cfg(not(windows))] let signals = terminal::signals(start.terminal.is_some())?; + #[cfg(not(windows))] let (mut terminal, mut io) = terminal::Terminal::stdio(start.stdin, start.terminal.is_some())?; + #[cfg(windows)] + let (mut terminal, mut io) = prepared.start()?; + #[cfg(windows)] + let signals = terminal.signals()?; let result = session::run(url, auth, status.database_identity, start, &mut io, signals).await; // Always join and restore the terminal, including a failed handshake. let cleanup = terminal.finish(); diff --git a/crates/cli/src/subcommands/container/exec/session.rs b/crates/cli/src/subcommands/container/exec/session.rs index fa2204c3445..dc590e15a37 100644 --- a/crates/cli/src/subcommands/container/exec/session.rs +++ b/crates/cli/src/subcommands/container/exec/session.rs @@ -43,11 +43,39 @@ pub(super) struct Io { } pub(super) async fn run( - mut url: Url, + url: Url, authorization: HeaderValue, identity: Identity, start: ExecStart, io: &mut Io, + signals: S, +) -> Result +where + S: Stream> + Unpin, +{ + // Terminal failure/cancellation also owns the connection and Ready phases. + // The inline socket future drops before the caller joins native I/O workers. + let Io { + input, + output, + completion, + } = io; + tokio::select! { + result = run_socket(url, authorization, identity, start, input, output, signals) => result, + result = completion => { + result.context("terminal worker stopped")??; + bail!("terminal worker stopped before observed exec exit") + } + } +} + +async fn run_socket( + mut url: Url, + authorization: HeaderValue, + identity: Identity, + start: ExecStart, + input: &mut mpsc::Receiver, + output: &OutputSender, mut signals: S, ) -> Result where @@ -126,8 +154,6 @@ where let (mut sink, mut source) = socket.split(); let (pong_tx, mut pong_rx) = mpsc::channel(1); - let input = &mut io.input; - let output = &io.output; let reader = async { while let Some(message) = source.next().await { match message.map_err(|_| anyhow::anyhow!("exec transport failed; process outcome is unknown"))? { @@ -183,10 +209,6 @@ where tokio::select! { result = reader => result, result = writer => result, - result = &mut io.completion => { - result.context("terminal worker stopped")??; - bail!("terminal worker stopped before observed exec exit") - } } } diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows.rs b/crates/cli/src/subcommands/container/exec/terminal_windows.rs new file mode 100644 index 00000000000..dcff29de6fe --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows.rs @@ -0,0 +1,373 @@ +//! Windows stdio workers remain owned through cancellation and console restoration. +//! Inherited synchronous handles cannot be converted to overlapped handles by dup. +#[path = "terminal_windows/control.rs"] +mod control; +#[path = "terminal_windows/io.rs"] +mod io; +#[cfg(test)] +#[path = "terminal_windows/tests.rs"] +mod tests; + +use super::session::{Input, Io, Output, OutputSender, OutputStream}; +use anyhow::{bail, ensure, Context, Result}; +use futures::{stream, Stream, StreamExt}; +use io::{ConsoleState, Event, File, Kind}; +use spacetimedb_lib::container::exec::{ClientControl, TerminalSize, MAX_DATA_BYTES}; +use std::{ + os::windows::io::AsRawHandle, + pin::Pin, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Condvar, Mutex, MutexGuard, + }, + thread::JoinHandle, + time::Duration, +}; +use tokio::sync::{mpsc, oneshot}; +use windows_sys::Win32::{ + Foundation::*, + System::{Console::*, IO::CancelSynchronousIo}, +}; + +fn check(ok: BOOL) -> Result<()> { + ensure!(ok != 0, "Windows terminal operation failed"); + Ok(()) +} +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|error| error.into_inner()) +} + +struct Shared { + stop: AtomicBool, + wake: Event, + restored: Event, + failure: Mutex>>>, + controls: mpsc::Sender>, + callbacks: Mutex, + callbacks_done: Condvar, +} +impl Shared { + fn stopped(&self) -> bool { + self.stop.load(Ordering::Acquire) + } + fn stop(&self) { + self.stop.store(true, Ordering::Release); + self.wake.set(); + } + fn fail(&self) { + self.stop(); + if let Some(sender) = lock(&self.failure).take() { + let _ = sender.send(Err(anyhow::anyhow!("Windows terminal stopped"))); + } + } +} + +pub(super) fn dimensions() -> Result { + let mut input_mode = 0; + let mut info: CONSOLE_SCREEN_BUFFER_INFO = unsafe { std::mem::zeroed() }; + unsafe { + check(GetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), &mut input_mode)) + .context("PTY requires attached console input")?; + check(GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &mut info)) + .context("PTY requires attached console output")?; + } + let size = TerminalSize { + rows: u16::try_from(i32::from(info.srWindow.Bottom) - i32::from(info.srWindow.Top) + 1)?, + columns: u16::try_from(i32::from(info.srWindow.Right) - i32::from(info.srWindow.Left) + 1)?, + }; + size.validate().context("invalid console dimensions")?; + Ok(size) +} + +pub(super) struct Prepared { + files: [Option; 3], + tty: bool, +} +impl Prepared { + pub(super) fn stdio(stdin: bool, tty: bool) -> Result { + let handles = unsafe { + [ + GetStdHandle(STD_INPUT_HANDLE), + GetStdHandle(STD_OUTPUT_HANDLE), + GetStdHandle(STD_ERROR_HANDLE), + ] + }; + Self::new( + [ + stdin.then(|| File::duplicate(handles[0])).transpose()?, + Some(File::duplicate(handles[1])?), + (!tty).then(|| File::duplicate(handles[2])).transpose()?, + ], + tty, + ) + } + fn new(files: [Option; 3], tty: bool) -> Result { + ensure!(files[1].is_some(), "terminal output is required"); + ensure!( + !tty || files[..2] + .iter() + .all(|file| file.as_ref().is_some_and(|file| file.kind == Kind::Console)), + "PTY requires attached console input and output" + ); + Ok(Self { files, tty }) + } + pub(super) fn start(self) -> Result<(Terminal, Io)> { + let (input_sender, input) = mpsc::channel(1); + let (output_sender, output) = mpsc::channel(1); + let (completed, completion) = oneshot::channel(); + let (control_sender, controls) = mpsc::channel(8); + let shared = Arc::new(Shared { + stop: AtomicBool::new(false), + wake: Event::new()?, + restored: Event::new()?, + failure: Mutex::new(Some(completed)), + controls: control_sender, + callbacks: Mutex::new(0), + callbacks_done: Condvar::new(), + }); + // Reserve callback/console ownership before any shared console mutation. + let registration = control::Registration::new(shared.clone())?; + let mut terminal = Terminal { + shared: shared.clone(), + workers: Vec::new(), + console: None, + registration: Some(registration), + controls: Some(controls), + tty: self.tty, + finished: false, + failed: false, + }; + terminal.console = Some(ConsoleState::prepare(&self.files, self.tty)?); + let [stdin, stdout, stderr] = self.files; + if let Some(stdin) = stdin { + terminal.spawn("container-exec-input", move |shared| { + read_input(stdin, input_sender, shared) + })?; + } + terminal.spawn("container-exec-output", move |shared| { + write_output(stdout.unwrap(), stderr, output, shared) + })?; + let notification = shared.clone(); + Ok(( + terminal, + Io { + input, + output: OutputSender { + sender: output_sender, + wake: Some(Arc::new(move || notification.wake.set())), + }, + completion, + }, + )) + } +} + +pub(super) struct Terminal { + shared: Arc, + workers: Vec>>, + console: Option, + registration: Option, + controls: Option>>, + tty: bool, + finished: bool, + failed: bool, +} +type Signals = Pin> + Send>>; +impl Terminal { + fn spawn(&mut self, name: &str, work: impl FnOnce(&Shared) -> Result<()> + Send + 'static) -> Result<()> { + let shared = self.shared.clone(); + self.workers.push( + std::thread::Builder::new() + .name(name.into()) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| work(&shared))) + .unwrap_or_else(|_| Err(anyhow::anyhow!("Windows terminal worker panicked"))); + if result.is_err() { + shared.fail(); + } + result + }) + .context("cannot start Windows terminal worker")?, + ); + Ok(()) + } + pub(super) fn signals(&mut self) -> Result { + let receiver = self.controls.take().context("terminal controls already consumed")?; + let mut streams: Vec = vec![Box::pin(stream::unfold(receiver, |mut receiver| async move { + receiver.recv().await.map(|control| (control, receiver)) + }))]; + if self.tty { + let initial = dimensions()?; + let interval = tokio::time::interval(Duration::from_millis(200)); + streams.push(Box::pin(stream::unfold( + (interval, initial), + |(mut interval, mut previous)| async move { + loop { + interval.tick().await; + match dimensions() { + Ok(size) if size == previous => {} + Ok(size) => { + previous = size; + return Some((Ok(ClientControl::Resize(size)), (interval, previous))); + } + Err(error) => return Some((Err(error), (interval, previous))), + } + } + }, + ))); + } + Ok(stream::select_all(streams).boxed()) + } + pub(super) fn finish(&mut self) -> Result<()> { + if self.finished { + ensure!(!self.failed, "Windows terminal cleanup previously failed"); + return Ok(()); + } + let mut errors = Vec::new(); + if let Some(registration) = &mut self.registration + && let Err(error) = registration.seal() + { + errors.push(error); + } + self.shared.stop(); + // Repeat cancellation until actual completion, including registration races. + while self.workers.iter().any(|worker| !worker.is_finished()) { + for worker in self.workers.iter().filter(|worker| !worker.is_finished()) { + unsafe { + CancelSynchronousIo(worker.as_raw_handle()); + } + } + std::thread::sleep(Duration::from_millis(10)); + } + for worker in self.workers.drain(..) { + match worker.join() { + Ok(Ok(())) => {} + Ok(Err(error)) => errors.push(error), + Err(_) => errors.push(anyhow::anyhow!("Windows terminal worker join failed")), + } + } + if let Some(console) = &mut self.console + && let Err(error) = console.restore() + { + errors.push(error); + } + // A close callback waits on this event. Signal before waiting for it. + self.shared.restored.set(); + let mut count = lock(&self.shared.callbacks); + while *count != 0 { + count = self + .shared + .callbacks_done + .wait(count) + .unwrap_or_else(|error| error.into_inner()); + } + drop(count); + self.registration.take(); + self.finished = true; + self.failed = !errors.is_empty(); + if !errors.is_empty() { + let error = errors.remove(0); + return Err(error.context(format!( + "Windows terminal cleanup failed ({} additional failures)", + errors.len() + ))); + } + Ok(()) + } +} +impl Drop for Terminal { + fn drop(&mut self) { + let _ = self.finish(); + } +} + +fn send_input(sender: &mpsc::Sender, mut value: Input, shared: &Shared) -> Result<()> { + while !shared.stopped() { + match sender.try_send(value) { + Ok(()) => return Ok(()), + Err(mpsc::error::TrySendError::Closed(_)) => bail!("terminal input consumer stopped"), + Err(mpsc::error::TrySendError::Full(returned)) => value = returned, + } + // No blocking queue send can prevent cancellation. Input consumption has + // no native wake hook, so this one bounded pending chunk polls at 20ms. + std::thread::sleep(Duration::from_millis(20)); + } + Ok(()) +} +fn read_input(file: File, sender: mpsc::Sender, shared: &Shared) -> Result<()> { + let mut bytes = vec![0u8; MAX_DATA_BYTES]; + let mut units = vec![0u16; MAX_DATA_BYTES / 4]; + let mut high_surrogate = None; + while !shared.stopped() { + let size = if file.kind == Kind::Console { + let count = file.read_console(&mut units, shared)?; + if shared.stopped() { + return Ok(()); + } + if count == 0 { + 0 + } else { + let text = decode_console(&units[..count], &mut high_surrogate)?; + bytes[..text.len()].copy_from_slice(text.as_bytes()); + if text.is_empty() { + continue; + } + text.len() + } + } else { + file.read(&mut bytes, shared)? + }; + if shared.stopped() { + return Ok(()); + } + if size == 0 { + ensure!(high_surrogate.is_none(), "incomplete console Unicode input"); + return send_input(&sender, Input::Eof, shared); + } + send_input(&sender, Input::Data(bytes[..size].to_vec()), shared)?; + } + Ok(()) +} +fn decode_console(units: &[u16], high: &mut Option) -> Result { + let mut combined = Vec::with_capacity(units.len() + 1); + combined.extend(high.take()); + combined.extend_from_slice(units); + if combined.last().is_some_and(|unit| (0xd800..=0xdbff).contains(unit)) { + *high = combined.pop(); + } + String::from_utf16(&combined).context("invalid console Unicode input") +} +fn write_output( + stdout: File, + stderr: Option, + mut receiver: mpsc::Receiver, + shared: &Shared, +) -> Result<()> { + while !shared.stopped() { + shared.wake.reset(); + match receiver.try_recv() { + Ok(value) => { + let file = match value.stream { + OutputStream::Stdout => &stdout, + OutputStream::Stderr => stderr.as_ref().context("terminal stderr is unavailable")?, + _ => bail!("invalid terminal output channel"), + }; + let mut offset = 0; + while offset < value.bytes.len() && !shared.stopped() { + let count = file.write(&value.bytes[offset..], shared)?; + if shared.stopped() { + return Ok(()); + } + ensure!(count != 0, "terminal output closed"); + offset += count; + } + if !shared.stopped() { + let _ = value.completed.send(Ok(())); + } + } + Err(mpsc::error::TryRecvError::Empty) => shared.wake.wait(20), + Err(mpsc::error::TryRecvError::Disconnected) => return Ok(()), + } + } + Ok(()) +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs new file mode 100644 index 00000000000..b6346bd0d40 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs @@ -0,0 +1,370 @@ +//! The parent owns a ConPTY, an exact test child and a separately drained output +//! pipe. No desktop, inherited account configuration, or user console is needed. +use super::*; +use std::{ + ffi::OsStr, + os::windows::ffi::OsStrExt, + path::{Path, PathBuf}, +}; + +fn wide(value: &OsStr) -> Vec { + value.encode_wide().chain([0]).collect() +} +struct Child { + process: Option, + console: HPCON, + input: Option, + startup_pipes: Option<(OwnedHandle, OwnedHandle)>, + reader: Option>>>, + waited: bool, +} +impl Child { + fn start(root: &Path, mode: &str) -> Result { + let (input_read, input) = pipe(); + let (output_read, output_write) = pipe(); + let mut this = Self { + process: None, + console: 0, + input: Some(input), + startup_pipes: Some((input_read, output_write)), + reader: None, + waited: false, + }; + ensure!( + unsafe { + CreatePseudoConsole( + COORD { X: 120, Y: 48 }, + this.startup_pipes.as_ref().unwrap().0.as_raw_handle(), + this.startup_pipes.as_ref().unwrap().1.as_raw_handle(), + 0, + &mut this.console, + ) + } == 0, + "cannot create owned pseudoconsole" + ); + // Drain concurrently before launching a process that can write console output. + this.reader = Some(std::thread::spawn(move || { + let mut all = Vec::new(); + let mut bytes = [0; 4096]; + loop { + let mut count = 0; + let ok = unsafe { + ReadFile( + output_read.as_raw_handle(), + bytes.as_mut_ptr(), + bytes.len() as u32, + &mut count, + null_mut(), + ) + }; + if ok == 0 { + ensure!( + unsafe { GetLastError() } == ERROR_BROKEN_PIPE, + "owned console drain failed" + ); + break; + } + if count == 0 { + break; + } + if all.len() < 1024 * 1024 { + all.extend_from_slice(&bytes[..count as usize]); + } + } + Ok(all) + })); + let mut length = 0; + unsafe { + InitializeProcThreadAttributeList(null_mut(), 1, 0, &mut length); + } + ensure!(length > 0 && length <= 4096, "invalid owned process attribute size"); + let mut storage = vec![0usize; length.div_ceil(size_of::())]; + let attributes = storage.as_mut_ptr().cast(); + unsafe { + check(InitializeProcThreadAttributeList(attributes, 1, 0, &mut length))?; + } + struct Attributes(windows_sys::Win32::System::Threading::LPPROC_THREAD_ATTRIBUTE_LIST); + impl Drop for Attributes { + fn drop(&mut self) { + unsafe { + DeleteProcThreadAttributeList(self.0); + } + } + } + let attributes = Attributes(attributes); + unsafe { + check(UpdateProcThreadAttribute( + attributes.0, + 0, + PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE as usize, + this.console as *const _, + size_of::(), + null_mut(), + null(), + ))?; + } + let executable = std::env::current_exe()?; + let module = module_path!().split_once("::").unwrap().1; + let selected = format!("{module}::windows_console_child"); + let mut command = wide(OsStr::new(&format!( + "\"{}\" --exact {selected} --ignored --nocapture --test-threads=1", + executable.display() + ))); + // Inherit no endpoint, auth, proxy or CLI settings. SystemRoot is only + // the Windows DLL/runtime base; all writable locations belong to root. + let system = std::env::var_os("SystemRoot").context("missing Windows system directory")?; + let mut environment = Vec::new(); + for (key, value) in [ + ("STDB_EXEC_CONSOLE_MODE", OsStr::new(mode)), + ("STDB_EXEC_CONSOLE_ROOT", root.as_os_str()), + ("SystemRoot", system.as_os_str()), + ("TEMP", root.as_os_str()), + ("TMP", root.as_os_str()), + ] { + environment.extend(format!("{key}=").encode_utf16()); + environment.extend(value.encode_wide()); + environment.push(0); + } + environment.push(0); + let mut startup: STARTUPINFOEXW = unsafe { std::mem::zeroed() }; + startup.StartupInfo.cb = size_of::() as u32; + startup.lpAttributeList = attributes.0; + let mut process: PROCESS_INFORMATION = unsafe { std::mem::zeroed() }; + unsafe { + check(CreateProcessW( + wide(executable.as_os_str()).as_ptr(), + command.as_mut_ptr(), + null(), + null(), + 0, + EXTENDED_STARTUPINFO_PRESENT | CREATE_UNICODE_ENVIRONMENT, + environment.as_ptr().cast(), + wide(root.as_os_str()).as_ptr(), + &startup.StartupInfo, + &mut process, + ))?; + } + this.process = Some(unsafe { OwnedHandle::from_raw_handle(process.hProcess) }); + drop(unsafe { OwnedHandle::from_raw_handle(process.hThread) }); + this.startup_pipes.take(); + Ok(this) + } + fn complete(&mut self) -> Result<()> { + let process = self.process.as_ref().context("owned child not started")?; + let completed = unsafe { WaitForSingleObject(process.as_raw_handle(), 10_000) } == WAIT_OBJECT_0; + ensure!(completed, "owned console child timed out"); + self.waited = true; + let mut code = 1; + unsafe { + check(GetExitCodeProcess(process.as_raw_handle(), &mut code))?; + } + ensure!(code == 0, "owned console child failed"); + Ok(()) + } + fn close(&mut self) -> Result<()> { + let mut errors = Vec::new(); + if let Some(process) = &self.process + && !self.waited + { + errors.push(anyhow::anyhow!("owned console child required forced cleanup")); + unsafe { + if let Err(error) = check(TerminateProcess(process.as_raw_handle(), 1)) { + errors.push(error.context("cannot terminate owned console child")); + } + self.waited = WaitForSingleObject(process.as_raw_handle(), INFINITE) == WAIT_OBJECT_0; + } + if !self.waited { + errors.push(anyhow::anyhow!("owned console child wait failed")); + } + } + self.input.take(); + // Also close parent's startup copies on every partial-start failure; + // retaining output_write would keep the drain waiting after ConPTY close. + self.startup_pipes.take(); + // Keep the drain alive while closing ConPTY; close can emit final output. + if self.console != 0 { + unsafe { + ClosePseudoConsole(self.console); + } + self.console = 0; + } + if let Some(reader) = self.reader.take() { + match reader.join() { + Ok(Ok(_)) => (), + Ok(Err(error)) => errors.push(error), + Err(_) => errors.push(anyhow::anyhow!("owned console drain panicked")), + } + } + ensure!( + errors.is_empty(), + "{}", + errors + .iter() + .map(|error| format!("{error:#}")) + .collect::>() + .join("; ") + ); + Ok(()) + } +} +impl Drop for Child { + fn drop(&mut self) { + if let Err(error) = self.close() { + eprintln!("owned console cleanup failed: {error:#}"); + } + } +} + +#[test] +fn native_windows_conpty_normal_error_cancel_restore_console() { + let _owner = lock(&TEST_OWNER); + for mode in ["normal", "error", "cancel"] { + let root = tempfile::tempdir().unwrap(); + let mut child = Child::start(root.path(), mode).unwrap(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let deadline = Instant::now() + Duration::from_secs(10); + while !root.path().join("ready").exists() { + assert!(Instant::now() < deadline, "owned console child did not become ready"); + std::thread::sleep(Duration::from_millis(10)); + } + write(child.input.as_ref().unwrap(), "🦀\u{1b}[A\u{3}".as_bytes()); + assert_eq!( + unsafe { ResizePseudoConsole(child.console, COORD { X: 100, Y: 40 }) }, + 0 + ); + child.complete()?; + assert_eq!( + std::fs::read(root.path().join("restored")).unwrap(), + b"all console settings restored" + ); + Ok(()) + })); + let cleanup = child.close(); + finish(result, [cleanup]).unwrap(); + } +} + +#[test] +#[ignore = "exact owned ConPTY child, invoked by native_windows_conpty_normal_error_cancel_restore_console"] +fn windows_console_child() { + let root = PathBuf::from(std::env::var_os("STDB_EXEC_CONSOLE_ROOT").expect("owned console child only")); + let mode = std::env::var("STDB_EXEC_CONSOLE_MODE").unwrap(); + assert!(matches!(mode.as_str(), "normal" | "error" | "cancel")); + let handles = unsafe { + [ + GetStdHandle(STD_INPUT_HANDLE), + GetStdHandle(STD_OUTPUT_HANDLE), + GetStdHandle(STD_ERROR_HANDLE), + ] + }; + let original: Vec<_> = handles + .iter() + .map(|handle| { + let mut value = 0; + unsafe { + check(GetConsoleMode(*handle, &mut value)).unwrap(); + } + value + }) + .collect(); + let code_page = unsafe { GetConsoleOutputCP() }; + let initial = dimensions().unwrap(); + assert_eq!((initial.rows, initial.columns), (48, 120)); + let (mut terminal, mut io) = Prepared::stdio(true, true).unwrap().start().unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut controls = { + let _entered = runtime.enter(); + terminal.signals().unwrap() + }; + let mut raw = 0; + unsafe { + check(GetConsoleMode(handles[0], &mut raw)).unwrap(); + } + assert_eq!( + raw & (ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT), + 0 + ); + assert_ne!(raw & ENABLE_VIRTUAL_TERMINAL_INPUT, 0); + std::fs::write(root.join("ready"), b"ready").unwrap(); + let result = runtime.block_on(async { + tokio::time::timeout(Duration::from_secs(5), async { + let expected = "🦀\u{1b}[A\u{3}".as_bytes(); + let mut received = Vec::new(); + while received.len() < expected.len() { + let Input::Data(bytes) = io.input.recv().await.context("missing console input")? else { + bail!("early console EOF") + }; + received.extend(bytes); + } + ensure!(received == expected, "console UTF-8/VT input differs"); + loop { + if let Some(Ok(ClientControl::Resize(size))) = controls.next().await + && (size.rows, size.columns) == (40, 100) + { + break; + } + } + let (completed, written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: if mode == "error" { + OutputStream::Stdin + } else { + OutputStream::Stdout + }, + bytes: "owned 🦀 output\r\n".as_bytes().to_vec(), + completed, + }) + .await?; + terminal.shared.wake.set(); + if mode == "error" { + ensure!(written.await.is_err(), "invalid output channel accepted"); + } else { + written.await??; + } + Ok::<(), anyhow::Error>(()) + }) + .await? + }); + let cleanup = if mode == "cancel" { + drop(terminal); + Ok(()) + } else { + terminal.finish() + }; + result.unwrap(); + assert_eq!(cleanup.is_err(), mode == "error"); + for (handle, previous) in handles.iter().zip(original) { + let mut value = 0; + unsafe { + check(GetConsoleMode(*handle, &mut value)).unwrap(); + } + assert_eq!(value, previous); + } + assert_eq!(unsafe { GetConsoleOutputCP() }, code_page); + std::fs::write(root.join("restored"), b"all console settings restored").unwrap(); +} + +#[test] +fn native_windows_console_cleanup_reports_drain_failure() { + for panic in [false, true] { + let mut child = Child { + process: None, + console: 0, + input: None, + startup_pipes: None, + reader: Some(std::thread::spawn(move || { + assert!(!panic, "owned drain panic"); + bail!("owned drain failure") + })), + waited: false, + }; + let error = child.close().unwrap_err(); + assert!(error.to_string().contains(if panic { "panicked" } else { "failure" })); + assert!(child.reader.is_none()); + } +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/control.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/control.rs new file mode 100644 index 00000000000..b7a09d5d68d --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/control.rs @@ -0,0 +1,97 @@ +use super::{check, lock, ClientControl, Shared}; +use anyhow::{ensure, Result}; +use std::sync::{Arc, Mutex}; +use windows_sys::Win32::{ + Foundation::{BOOL, FALSE, TRUE}, + System::{Console::*, Threading::INFINITE}, +}; + +static ACTIVE: Mutex, bool)>> = Mutex::new(None); +pub(super) struct Registration { + shared: Arc, + sealed: bool, +} +impl Registration { + pub fn new(shared: Arc) -> Result { + let mut active = lock(&ACTIVE); + ensure!(active.is_none(), "another Windows terminal owner is active"); + unsafe { + check(SetConsoleCtrlHandler(Some(handler), TRUE))?; + } + *active = Some((shared.clone(), true)); + Ok(Self { shared, sealed: false }) + } + pub fn seal(&mut self) -> Result<()> { + if self.sealed { + return Ok(()); + } + let mut active = lock(&ACTIVE); + if let Some((shared, admitted)) = active.as_mut() + && Arc::ptr_eq(shared, &self.shared) + { + *admitted = false; + } + self.sealed = true; + unsafe { check(SetConsoleCtrlHandler(Some(handler), FALSE)) } + } +} +impl Drop for Registration { + fn drop(&mut self) { + let _ = self.seal(); + let mut active = lock(&ACTIVE); + if active + .as_ref() + .is_some_and(|(shared, _)| Arc::ptr_eq(shared, &self.shared)) + { + *active = None; + } + } +} + +struct Callback(Arc); +impl Drop for Callback { + fn drop(&mut self) { + let mut count = lock(&self.0.callbacks); + *count -= 1; + self.0.callbacks_done.notify_all(); + } +} +unsafe extern "system" fn handler(event: u32) -> BOOL { + let callback = { + let active = lock(&ACTIVE); + let Some((shared, true)) = active.as_ref() else { + return FALSE; + }; + *lock(&shared.callbacks) += 1; + Callback(shared.clone()) + }; + // The OS owns this callback thread. Never unwind across the ABI boundary. + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dispatch(event, &callback.0))).unwrap_or_else(|_| { + callback.0.fail(); + TRUE + }) +} +fn dispatch(event: u32, shared: &Shared) -> BOOL { + match event { + CTRL_C_EVENT | CTRL_BREAK_EVENT => { + let signal = if event == CTRL_C_EVENT { 2 } else { 3 }; + if shared.controls.try_send(Ok(ClientControl::Signal(signal))).is_err() { + shared.fail(); + } + TRUE + } + CTRL_CLOSE_EVENT | CTRL_LOGOFF_EVENT | CTRL_SHUTDOWN_EVENT => { + shared.fail(); + // Returning allows OS termination. Only acknowledge after joined I/O + // and restoration; an OS forced timeout cannot count as completion. + shared.restored.wait(INFINITE); + TRUE + } + _ => FALSE, + } +} + +#[cfg(test)] +pub(super) fn invoke(event: u32) -> BOOL { + unsafe { handler(event) } +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/io.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/io.rs new file mode 100644 index 00000000000..20e30e1c949 --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/io.rs @@ -0,0 +1,344 @@ +//! Buffers and OVERLAPPED storage outlive the exact operation, including cancellation. +use super::{check, Shared}; +use anyhow::{bail, ensure, Result}; +use std::{ + os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}, + ptr::{null, null_mut}, + sync::Arc, +}; +use windows_sys::{ + Wdk::Storage::FileSystem::{ + FileModeInformation, NtQueryInformationFile, FILE_MODE_INFORMATION, FILE_SYNCHRONOUS_IO_ALERT, + FILE_SYNCHRONOUS_IO_NONALERT, + }, + Win32::{ + Foundation::*, + Storage::FileSystem::*, + System::{Console::*, Threading::*, IO::*}, + }, +}; + +pub(super) struct Event(OwnedHandle); +impl Event { + pub fn new() -> Result { + // SAFETY: no security attributes, name, or inherited handle; owned once. + let raw = unsafe { CreateEventW(null(), 1, 0, null()) }; + ensure!(!raw.is_null(), "cannot create terminal event"); + Ok(Self(unsafe { OwnedHandle::from_raw_handle(raw) })) + } + pub fn raw(&self) -> HANDLE { + self.0.as_raw_handle() + } + pub fn set(&self) { + unsafe { + SetEvent(self.raw()); + } + } + pub fn reset(&self) { + unsafe { + ResetEvent(self.raw()); + } + } + pub fn wait(&self, millis: u32) { + unsafe { + WaitForSingleObject(self.raw(), millis); + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Kind { + Console, + Synchronous, + OverlappedPipe, +} +pub(super) struct File { + pub handle: Arc, + pub kind: Kind, + pipe: bool, +} +impl File { + pub fn duplicate(raw: HANDLE) -> Result { + ensure!(!raw.is_null() && raw != INVALID_HANDLE_VALUE, "missing terminal handle"); + let mut duplicate = null_mut(); + // SAFETY: duplicate this process's borrowed standard handle, non-inheritable. + unsafe { + check(DuplicateHandle( + GetCurrentProcess(), + raw, + GetCurrentProcess(), + &mut duplicate, + 0, + 0, + DUPLICATE_SAME_ACCESS, + ))?; + } + Self::owned(unsafe { OwnedHandle::from_raw_handle(duplicate) }) + } + pub fn owned(handle: OwnedHandle) -> Result { + let raw = handle.as_raw_handle(); + let file_type = unsafe { GetFileType(raw) }; + let mut mode = 0; + let kind = if unsafe { GetConsoleMode(raw, &mut mode) } != 0 { + Kind::Console + } else { + ensure!( + matches!(file_type, FILE_TYPE_DISK | FILE_TYPE_PIPE | FILE_TYPE_CHAR), + "unsupported terminal handle type" + ); + // This class-specific query follows .NET 10 SafeFileHandle.GetFileOptions: + // it returns the file object's mode directly and accepts only SUCCESS. + // Do not generalize this to other information classes or infer request + // completion by waiting on an inherited file's shared event. + // https://github.com/dotnet/runtime/blob/60629d14374c56f1cb51819049ad1fa529307f8d/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs#L182-L205 + let mut status: IO_STATUS_BLOCK = unsafe { std::mem::zeroed() }; + let mut info: FILE_MODE_INFORMATION = unsafe { std::mem::zeroed() }; + let result = unsafe { + NtQueryInformationFile( + raw, + &mut status, + (&mut info as *mut FILE_MODE_INFORMATION).cast(), + size_of::() as u32, + FileModeInformation, + ) + }; + ensure!(result == STATUS_SUCCESS, "cannot classify inherited terminal handle"); + if info.Mode & (FILE_SYNCHRONOUS_IO_ALERT | FILE_SYNCHRONOUS_IO_NONALERT) != 0 { + Kind::Synchronous + } else { + ensure!( + file_type == FILE_TYPE_PIPE, + "inherited asynchronous seekable files are unsupported for container exec" + ); + Kind::OverlappedPipe + } + }; + Ok(Self { + handle: Arc::new(handle), + kind, + pipe: file_type == FILE_TYPE_PIPE, + }) + } + pub fn raw(&self) -> HANDLE { + self.handle.as_raw_handle() + } + + pub fn read(&self, bytes: &mut [u8], shared: &Shared) -> Result { + if self.kind == Kind::Console { + bail!("console input requires Unicode decoding"); + } + loop { + match self.transfer(bytes.as_mut_ptr(), bytes.len(), false, shared)? { + // A zero-byte pipe message is not EOF. Only a broken pipe is. + Some(0) if self.pipe => continue, + Some(count) => return Ok(count), + None => return Ok(0), + } + } + } + pub fn write(&self, bytes: &[u8], shared: &Shared) -> Result { + Ok(self + .transfer(bytes.as_ptr().cast_mut(), bytes.len(), true, shared)? + .unwrap_or(0)) + } + fn transfer(&self, bytes: *mut u8, length: usize, write: bool, shared: &Shared) -> Result> { + ensure!(length <= super::MAX_DATA_BYTES, "terminal I/O exceeds bound"); + if shared.stopped() { + return Ok(None); + } + let mut count = 0; + let mut pending = if self.kind == Kind::OverlappedPipe { + Some(Pending::new(self.raw())?) + } else { + None + }; + let overlap = pending.as_mut().map_or(null_mut(), |value| &mut *value.overlap); + // SAFETY: the caller owns this bounded buffer until final completion; + // Pending retains its stable OVERLAPPED/event and drains on unwind. + let ok = unsafe { + if write { + WriteFile(self.raw(), bytes, length as u32, &mut count, overlap) + } else { + ReadFile(self.raw(), bytes, length as u32, &mut count, overlap) + } + }; + let mut error = if ok == 0 { + unsafe { GetLastError() } + } else { + ERROR_SUCCESS + }; + if error == ERROR_IO_PENDING { + let operation = pending.as_mut().expect("overlapped operation must own completion"); + operation.active = true; + // Cancellation may have arrived before ReadFile/WriteFile registered. + // The issuing owner rechecks it and cancels the exact operation. + loop { + if shared.stopped() { + operation.cancel(); + } + let done = unsafe { GetOverlappedResult(self.raw(), &*operation.overlap, &mut count, 0) }; + error = if done != 0 { + ERROR_SUCCESS + } else { + unsafe { GetLastError() } + }; + if error != ERROR_IO_INCOMPLETE { + operation.active = false; + break; + } + operation.event.wait(20); + } + } + if shared.stopped() { + return Ok(None); + } + if !write && matches!(error, ERROR_BROKEN_PIPE | ERROR_HANDLE_EOF) { + return Ok(None); + } + ensure!( + error == ERROR_SUCCESS || (!write && error == ERROR_MORE_DATA), + "terminal I/O failed" + ); + ensure!(count as usize <= length, "invalid terminal I/O count"); + Ok(Some(count as usize)) + } + pub fn read_console(&self, units: &mut [u16], shared: &Shared) -> Result { + loop { + let mut count = 0; + if shared.stopped() { + return Ok(0); + } + let ok = unsafe { + SetLastError(ERROR_SUCCESS); + ReadConsoleW( + self.raw(), + units.as_mut_ptr().cast(), + units.len() as u32, + &mut count, + null(), + ) + }; + if shared.stopped() { + return Ok(0); + } + // Cooked console Ctrl+C/Break is forwarded by our control handler. + // It must interrupt and resume this read, not turn into EOF/failure. + if count == 0 && unsafe { GetLastError() } == ERROR_OPERATION_ABORTED { + continue; + } + check(ok)?; + ensure!(count as usize <= units.len(), "invalid console input count"); + return Ok(count as usize); + } + } +} + +struct Pending { + handle: HANDLE, + overlap: Box, + event: Event, + active: bool, +} +impl Pending { + fn new(handle: HANDLE) -> Result { + let event = Event::new()?; + let mut overlap: Box = Box::new(unsafe { std::mem::zeroed() }); + // The inherited file may belong to another owner's completion port. + // The low bit suppresses IOCP packets for this exact operation without + // changing the shared file's modes. Wait/close uses the unflagged event. + overlap.hEvent = event.raw().map_addr(|address| address | 1); + Ok(Self { + handle, + overlap, + event, + active: false, + }) + } + fn cancel(&self) { + unsafe { + CancelIoEx(self.handle, &*self.overlap); + } + } +} +impl Drop for Pending { + fn drop(&mut self) { + if self.active { + self.cancel(); + let mut count = 0; + // An unwind never abandons an accepted I/O operation or its memory. + unsafe { + GetOverlappedResult(self.handle, &*self.overlap, &mut count, 1); + } + } + } +} + +pub(super) struct ConsoleState { + files: Vec<(Arc, u32)>, + code_page: Option, + restored: bool, +} +impl ConsoleState { + pub fn prepare(files: &[Option; 3], tty: bool) -> Result { + let mut this = Self { + files: Vec::new(), + code_page: None, + restored: false, + }; + // Capture every alias before mutating any shared console buffer. + for file in files.iter().flatten().filter(|file| file.kind == Kind::Console) { + let mut mode = 0; + unsafe { + check(GetConsoleMode(file.raw(), &mut mode))?; + } + this.files.push((file.handle.clone(), mode)); + } + if files[1..].iter().flatten().any(|file| file.kind == Kind::Console) { + let original = unsafe { GetConsoleOutputCP() }; + ensure!(original != 0, "cannot read console output code page"); + this.code_page = Some(original); + unsafe { + check(SetConsoleOutputCP(65001))?; + } + } + for (index, file) in files.iter().enumerate().filter_map(|(i, f)| f.as_ref().map(|f| (i, f))) { + if file.kind != Kind::Console { + continue; + } + let mut mode = 0; + unsafe { + check(GetConsoleMode(file.raw(), &mut mode))?; + } + if index == 0 && tty { + mode = (mode | ENABLE_VIRTUAL_TERMINAL_INPUT | ENABLE_EXTENDED_FLAGS) + & !(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT | ENABLE_QUICK_EDIT_MODE); + } else if index != 0 { + mode |= ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING; + } + unsafe { + check(SetConsoleMode(file.raw(), mode))?; + } + } + Ok(this) + } + pub fn restore(&mut self) -> Result<()> { + if self.restored { + return Ok(()); + } + let mut failed = false; + for (file, mode) in &self.files { + failed |= unsafe { SetConsoleMode(file.as_raw_handle(), *mode) } == 0; + } + if let Some(code_page) = self.code_page { + failed |= unsafe { SetConsoleOutputCP(code_page) } == 0; + } + self.restored = !failed; + ensure!(!failed, "could not restore console settings"); + Ok(()) + } +} +impl Drop for ConsoleState { + fn drop(&mut self) { + let _ = self.restore(); + } +} diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/tests.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/tests.rs new file mode 100644 index 00000000000..afe90e294ac --- /dev/null +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/tests.rs @@ -0,0 +1,471 @@ +// The process-wide console handler must stay exclusively owned across awaits. +#![allow(clippy::await_holding_lock)] +use super::*; +use std::{ + fs::OpenOptions, + os::windows::{ + fs::OpenOptionsExt, + io::{FromRawHandle, OwnedHandle}, + }, + ptr::{null, null_mut}, + time::Instant, +}; +use windows_sys::Win32::{ + Storage::FileSystem::*, + System::{Pipes::*, Threading::*}, +}; + +#[path = "console_tests.rs"] +mod console; + +// The real console child has a separate process. These pipe fixtures serialize +// only our removable process-wide Ctrl handler, never the user's console modes. +static TEST_OWNER: Mutex<()> = Mutex::new(()); + +fn finish(primary: std::thread::Result>, cleanup: impl IntoIterator>) -> Result { + let errors: Vec<_> = cleanup.into_iter().filter_map(Result::err).collect(); + let details = errors + .iter() + .map(|error| format!("{error:#}")) + .collect::>() + .join("; "); + match primary { + Ok(Ok(value)) if errors.is_empty() => Ok(value), + Ok(Ok(_)) => bail!("owned Windows fixture cleanup failed: {details}"), + Ok(Err(error)) if errors.is_empty() => Err(error), + Ok(Err(error)) => Err(error.context(format!("owned Windows fixture cleanup also failed: {details}"))), + Err(panic) => { + if !errors.is_empty() { + eprintln!("owned Windows fixture cleanup also failed: {details}"); + } + std::panic::resume_unwind(panic) + } + } +} +fn pipe() -> (OwnedHandle, OwnedHandle) { + let (mut read, mut write) = (null_mut(), null_mut()); + unsafe { + check(CreatePipe(&mut read, &mut write, null(), 4096)).unwrap(); + (OwnedHandle::from_raw_handle(read), OwnedHandle::from_raw_handle(write)) + } +} +fn fixture() -> (Terminal, Io, OwnedHandle, OwnedHandle, OwnedHandle) { + let (input, input_writer) = pipe(); + let (output_reader, output) = pipe(); + let (error_reader, error) = pipe(); + let (terminal, io) = Prepared::new( + [ + Some(File::owned(input).unwrap()), + Some(File::owned(output).unwrap()), + Some(File::owned(error).unwrap()), + ], + false, + ) + .unwrap() + .start() + .unwrap(); + (terminal, io, input_writer, output_reader, error_reader) +} +fn receive(handle: &OwnedHandle, size: usize) -> Vec { + let deadline = Instant::now() + Duration::from_secs(5); + let mut received = Vec::new(); + while received.len() < size { + let mut available = 0; + unsafe { + check(PeekNamedPipe( + handle.as_raw_handle(), + null_mut(), + 0, + null_mut(), + &mut available, + null_mut(), + )) + .unwrap(); + } + assert!(Instant::now() < deadline, "owned pipe output timed out"); + if available == 0 { + std::thread::sleep(Duration::from_millis(5)); + continue; + } + let mut bytes = vec![0; (available as usize).min(size - received.len())]; + let mut count = 0; + unsafe { + check(ReadFile( + handle.as_raw_handle(), + bytes.as_mut_ptr(), + bytes.len() as u32, + &mut count, + null_mut(), + )) + .unwrap(); + } + received.extend_from_slice(&bytes[..count as usize]); + } + received +} +fn write(handle: &OwnedHandle, bytes: &[u8]) { + let mut count = 0; + unsafe { + check(WriteFile( + handle.as_raw_handle(), + bytes.as_ptr(), + bytes.len() as u32, + &mut count, + null_mut(), + )) + .unwrap(); + } + assert_eq!(count as usize, bytes.len()); +} + +#[test] +fn native_windows_pipe_backpressure_cancellation_joins_both_workers() { + let _owner = lock(&TEST_OWNER); + let (mut terminal, io, _input_writer, _output_reader, _error_reader) = fixture(); + let (ack, mut completed) = oneshot::channel(); + io.output + .sender + .try_send(Output { + stream: OutputStream::Stdout, + bytes: vec![42; MAX_DATA_BYTES], + completed: ack, + }) + .unwrap(); + terminal.shared.wake.set(); + std::thread::sleep(Duration::from_millis(50)); + assert!(completed.try_recv().is_err()); + let started = Instant::now(); + terminal.finish().unwrap(); + assert!(started.elapsed() < Duration::from_secs(3)); + assert!(terminal.workers.is_empty()); + terminal.finish().unwrap(); + assert!(completed.blocking_recv().is_err()); +} + +#[test] +fn native_windows_cancel_before_io_registration_and_worker_panic_are_joined() { + let _owner = lock(&TEST_OWNER); + for panic in [false, true] { + let (mut terminal, _io, _writer, _reader, _error_reader) = fixture(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + terminal + .spawn("owned-race", move |shared| { + entered_tx.send(()).unwrap(); + while !shared.stopped() { + std::thread::yield_now(); + } + if panic { + panic!("owned terminal panic"); + } + // Intentionally issue a synchronous call after cancellation was + // requested. Repeated thread cancellation must catch registration. + let (read, _write) = pipe(); + let mut byte = 0; + let mut count = 0; + unsafe { + ReadFile(read.as_raw_handle(), &mut byte, 1, &mut count, null_mut()); + } + Ok(()) + }) + .unwrap(); + entered.recv_timeout(Duration::from_secs(2)).unwrap(); + let result = terminal.finish(); + assert_eq!(result.is_err(), panic); + assert!(terminal.workers.is_empty()); + } +} + +#[tokio::test] +async fn native_windows_pipe_bytes_eof_and_signal_numbers() { + let _owner = lock(&TEST_OWNER); + let (mut terminal, mut io, writer, reader, error_reader) = fixture(); + write(&writer, &[0, 255, 1, 13, 10]); + drop(writer); + let Input::Data(bytes) = io.input.recv().await.unwrap() else { + panic!("missing input bytes") + }; + assert_eq!(bytes, [0, 255, 1, 13, 10]); + assert!(matches!(io.input.recv().await, Some(Input::Eof))); + let (completed, written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stdout, + bytes: vec![255, 0, 13, 10], + completed, + }) + .await + .unwrap(); + terminal.shared.wake.set(); + written.await.unwrap().unwrap(); + assert_eq!(receive(&reader, 4), [255, 0, 13, 10]); + let (completed, written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stderr, + bytes: vec![0, 254, 128], + completed, + }) + .await + .unwrap(); + terminal.shared.wake.set(); + written.await.unwrap().unwrap(); + assert_eq!(receive(&error_reader, 3), [0, 254, 128]); + let mut signals = terminal.signals().unwrap(); + assert_eq!(control::invoke(CTRL_C_EVENT), 1); + assert!(matches!(signals.next().await, Some(Ok(ClientControl::Signal(2))))); + assert_eq!(control::invoke(CTRL_BREAK_EVENT), 1); + assert!(matches!(signals.next().await, Some(Ok(ClientControl::Signal(3))))); + terminal.finish().unwrap(); +} + +#[test] +fn native_windows_synchronous_file_offset_and_asynchronous_file_rejection() { + let _owner = lock(&TEST_OWNER); + use std::io::{Read, Seek, SeekFrom, Write}; + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("owned-redirection"); + let mut file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .unwrap(); + file.write_all(b"abcdef").unwrap(); + file.seek(SeekFrom::Start(2)).unwrap(); + let classified = File::duplicate(file.as_raw_handle()).unwrap(); + assert_eq!(classified.kind, Kind::Synchronous); + let (mut terminal, _io, _writer, _reader, _error_reader) = fixture(); + let mut bytes = [0; 2]; + assert_eq!(classified.read(&mut bytes, &terminal.shared).unwrap(), 2); + assert_eq!(&bytes, b"cd"); + let mut remainder = Vec::new(); + file.read_to_end(&mut remainder).unwrap(); + assert_eq!(&remainder, b"ef"); + let asynchronous = OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_OVERLAPPED) + .open(path) + .unwrap(); + assert!(File::duplicate(asynchronous.as_raw_handle()) + .err() + .unwrap() + .to_string() + .contains("asynchronous seekable")); + terminal.finish().unwrap(); +} + +fn overlapped_pipe(output: bool) -> (OwnedHandle, OwnedHandle) { + let unique = tempfile::tempdir().unwrap(); + let name: Vec = format!( + r"\\.\pipe\spacetimedb-exec-{}-{}", + std::process::id(), + unique.path().file_name().unwrap().to_string_lossy() + ) + .encode_utf16() + .chain([0]) + .collect(); + let access = if output { + PIPE_ACCESS_OUTBOUND + } else { + PIPE_ACCESS_INBOUND + }; + let raw = unsafe { + CreateNamedPipeW( + name.as_ptr(), + access | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE, + PIPE_TYPE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, + 1, + 4096, + 4096, + 0, + null(), + ) + }; + assert_ne!(raw, INVALID_HANDLE_VALUE); + let server = unsafe { OwnedHandle::from_raw_handle(raw) }; + let access = if output { GENERIC_READ } else { GENERIC_WRITE }; + let raw = unsafe { CreateFileW(name.as_ptr(), access, 0, null(), OPEN_EXISTING, 0, null_mut()) }; + assert_ne!(raw, INVALID_HANDLE_VALUE); + let client = unsafe { OwnedHandle::from_raw_handle(raw) }; + let mut connected: windows_sys::Win32::System::IO::OVERLAPPED = unsafe { std::mem::zeroed() }; + // The exact client has already connected; no asynchronous accept remains. + assert_eq!(unsafe { ConnectNamedPipe(server.as_raw_handle(), &mut connected) }, 0); + assert_eq!(unsafe { GetLastError() }, ERROR_PIPE_CONNECTED); + (server, client) +} + +#[tokio::test] +async fn native_windows_overlapped_pipe_cancel_observes_exact_completion() { + let _owner = lock(&TEST_OWNER); + for output in [false, true] { + let (server, client) = overlapped_pipe(output); + let file = File::owned(server).unwrap(); + assert_eq!(file.kind, Kind::OverlappedPipe); + let (unused_reader, ordinary_output) = pipe(); + let files = if output { + [None, Some(file), None] + } else { + [Some(file), Some(File::owned(ordinary_output).unwrap()), None] + }; + let (mut terminal, mut io) = Prepared::new(files, false).unwrap().start().unwrap(); + let mut ack = None; + if output { + let (completed, written) = oneshot::channel(); + io.output + .sender + .send(Output { + stream: OutputStream::Stdout, + bytes: vec![0x81; MAX_DATA_BYTES], + completed, + }) + .await + .unwrap(); + terminal.shared.wake.set(); + ack = Some(written); + } else { + write(&client, &[]); + write(&client, &[0, 255, 13, 10]); + let Input::Data(bytes) = tokio::time::timeout(Duration::from_secs(2), io.input.recv()) + .await + .unwrap() + .unwrap() + else { + panic!("zero pipe message became EOF") + }; + assert_eq!(bytes, [0, 255, 13, 10]); + } + tokio::time::sleep(Duration::from_millis(30)).await; + let started = Instant::now(); + terminal.finish().unwrap(); + assert!(started.elapsed() < Duration::from_secs(3)); + assert!(terminal.workers.is_empty()); + if let Some(written) = ack { + assert!(written.await.is_err()); + } + drop(unused_reader); + } +} + +#[tokio::test] +async fn native_windows_close_before_ready_restores_then_releases_callback() { + let _owner = lock(&TEST_OWNER); + use futures::{FutureExt, StreamExt}; + let (mut terminal, mut io, _writer, _reader, _error_reader) = fixture(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!( + "http://{}/v1/database/{}/container/exec", + listener.local_addr().unwrap(), + spacetimedb_lib::Identity::ZERO.to_hex() + ); + let (started, start) = std::sync::mpsc::channel(); + let callback = std::thread::spawn(move || { + start.recv_timeout(Duration::from_secs(5)).unwrap(); + control::invoke(CTRL_CLOSE_EVENT) + }); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = tokio_tungstenite::accept_hdr_async( + socket, + |_request: &tokio_tungstenite::tungstenite::handshake::server::Request, + mut response: tokio_tungstenite::tungstenite::handshake::server::Response| { + response.headers_mut().insert( + "sec-websocket-protocol", + spacetimedb_lib::container::exec::SUBPROTOCOL.parse().unwrap(), + ); + Ok(response) + }, + ) + .await + .unwrap(); + assert!(socket.next().await.is_some()); + started.send(()).unwrap(); + assert!(socket.next().await.is_none_or(|value| value.is_err())); + }); + let start = spacetimedb_lib::container::exec::ExecStart { + generation: 1, + argv: vec!["literal-command".into()], + working_directory: None, + environment: Default::default(), + stdin: true, + terminal: None, + }; + let result = std::panic::AssertUnwindSafe(async { + let result = tokio::time::timeout( + Duration::from_secs(3), + super::super::session::run( + origin.parse().unwrap(), + "Bearer owned-loopback-fixture".parse().unwrap(), + spacetimedb_lib::Identity::ZERO, + start, + &mut io, + futures::stream::pending(), + ), + ) + .await?; + ensure!(result.is_err(), "close was reported as a guest exit"); + Ok(()) + }) + .catch_unwind() + .await; + // Join and release the handler even if the assertion below will fail. + let terminal_result = terminal.finish(); + let callback_result = callback + .join() + .map_err(|_| anyhow::anyhow!("close callback panicked")) + .and_then(|result| { + ensure!(result == 1, "close callback was not handled"); + Ok(()) + }); + let server_result = server.await.context("owned close fixture server failed"); + finish(result, [terminal_result, callback_result, server_result]).unwrap(); + assert_eq!(*lock(&terminal.shared.callbacks), 0); +} + +#[test] +fn native_windows_console_unicode_surrogates_are_not_split() { + let mut pending = None; + assert_eq!(decode_console(&[0xd83e], &mut pending).unwrap(), ""); + assert_eq!( + decode_console(&[0xdd80, 27, 91, 65], &mut pending).unwrap(), + "🦀\u{1b}[A" + ); + assert!(decode_console(&[0xdc00], &mut pending).is_err()); +} + +#[test] +fn native_windows_overlapped_pipe_suppresses_inherited_completion_port_packets() { + let _owner = lock(&TEST_OWNER); + use windows_sys::Win32::System::IO::{CreateIoCompletionPort, GetQueuedCompletionStatus}; + for output in [false, true] { + let (server, client) = overlapped_pipe(output); + let raw = unsafe { CreateIoCompletionPort(server.as_raw_handle(), null_mut(), 123, 1) }; + assert!(!raw.is_null()); + let port = unsafe { OwnedHandle::from_raw_handle(raw) }; + // Dup preserves the existing IOCP association. No shared completion mode + // is changed by classification or the adapter's per-operation events. + let file = File::duplicate(server.as_raw_handle()).unwrap(); + let (mut terminal, _io, _writer, _reader, _error_reader) = fixture(); + let primary = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + if output { + assert_eq!(file.write(&[0, 255, 7], &terminal.shared)?, 3); + assert_eq!(receive(&client, 3), [0, 255, 7]); + } else { + write(&client, &[0, 255, 8]); + let mut bytes = [0; 3]; + assert_eq!(file.read(&mut bytes, &terminal.shared)?, 3); + assert_eq!(bytes, [0, 255, 8]); + } + let (mut bytes, mut key, mut overlap) = (0, 0, null_mut()); + let result = + unsafe { GetQueuedCompletionStatus(port.as_raw_handle(), &mut bytes, &mut key, &mut overlap, 50) }; + assert_eq!(result, 0); + assert_eq!(unsafe { GetLastError() }, WAIT_TIMEOUT); + assert!(overlap.is_null(), "adapter operation escaped to inherited IOCP"); + Ok(()) + })); + let cleanup = terminal.finish(); + finish(primary, [cleanup]).unwrap(); + } +} diff --git a/crates/cli/src/subcommands/container/exec/tests.rs b/crates/cli/src/subcommands/container/exec/tests.rs index f962848cf25..bb8b2f7f2ac 100644 --- a/crates/cli/src/subcommands/container/exec/tests.rs +++ b/crates/cli/src/subcommands/container/exec/tests.rs @@ -403,3 +403,85 @@ async fn status_uses_authenticated_explicit_loopback_and_pins_the_resolved_ident ); assert!(running_generation(&status).is_err()); } + +#[tokio::test] +async fn terminal_cancellation_closes_pending_handshake_and_ready_without_replay() { + use tokio::io::AsyncReadExt; + for handshake in [true, false] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = Url::parse(&format!( + "http://{}/v1/database/{}/container/exec", + listener.local_addr().unwrap(), + Identity::ZERO.to_hex() + )) + .unwrap(); + let (_input_tx, input) = mpsc::channel(1); + let (output_tx, _output) = mpsc::channel(1); + let (cancel, completion) = oneshot::channel(); + let mut io = Io { + input, + output: OutputSender { + sender: output_tx, + wake: None, + }, + completion, + }; + let mut server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + if handshake { + let mut byte = [0]; + assert_eq!(socket.read(&mut byte).await.unwrap(), 1); + let _ = cancel.send(Err(anyhow::anyhow!("owned terminal cancellation"))); + let mut pending_request = Vec::new(); + socket.read_to_end(&mut pending_request).await.unwrap(); + } else { + let mut socket = + tokio_tungstenite::accept_hdr_async(socket, |_request: &Request, mut response: Response| { + response + .headers_mut() + .insert("sec-websocket-protocol", HeaderValue::from_static(wire::SUBPROTOCOL)); + Ok(response) + }) + .await + .unwrap(); + assert!(matches!(socket.next().await, Some(Ok(Message::Text(_))))); + let _ = cancel.send(Err(anyhow::anyhow!("owned terminal cancellation"))); + assert!(matches!( + socket.next().await, + None | Some(Err(_)) | Some(Ok(Message::Close(_))) + )); + } + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), listener.accept()) + .await + .is_err() + ); + }); + let options = start(&arguments(&["exec", "db", "--", "true"]).unwrap(), 42).unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_secs(2), + session::run( + url, + HeaderValue::from_static("Bearer owned-loopback-fixture"), + Identity::ZERO, + options, + &mut io, + futures::stream::pending(), + ), + ) + .await; + let joined = tokio::time::timeout(std::time::Duration::from_secs(2), &mut server).await; + let forced = joined.is_err(); + if forced { + server.abort(); + let _ = server.await; + } + assert!(!forced, "owned cancellation server failed to stop"); + joined.unwrap().unwrap(); + assert!(result + .unwrap() + .unwrap_err() + .to_string() + .contains("owned terminal cancellation")); + } +} diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index 101edcf2e64..bc2d13c3b7f 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -629,7 +629,7 @@ Run a literal command in the current running container **Usage:** `spacetime container exec [OPTIONS] -- ...` -Requires database Admin permission. No shell, container start, or reconnect is implicit. Use -- before COMMAND; for a shell, name its executable explicitly. Linux and macOS terminals are currently supported. A lost connection does not establish that the process exited. +Requires database Admin permission. No shell, container start, or reconnect is implicit. Use -- before COMMAND; for a shell, name its executable explicitly. Linux, macOS, and Windows terminals are supported. Windows requires an attached VT-capable console for --tty; inherited asynchronous seekable files are unsupported. A lost connection does not establish that the process exited. ###### **Arguments:** diff --git a/tools/ci/windows-exec-tests.ps1 b/tools/ci/windows-exec-tests.ps1 new file mode 100644 index 00000000000..eb561733310 --- /dev/null +++ b/tools/ci/windows-exec-tests.ps1 @@ -0,0 +1,192 @@ +# Compile the actual CLI libtest, then run only owned local exec fixtures. +# A forced cleanup is a failed gate, never evidence that the fixture joined. +# These receipts cover direct children. Each native fixture owns and joins its +# own children; the wrapper does not claim independent descendant retirement. +$ErrorActionPreference = 'Stop' +if (-not $IsWindows -or $env:RUNNER_OS -ne 'Windows') { + throw 'This gate requires the native Windows CI runner.' +} + +$workspace = (Resolve-Path (Join-Path $PSScriptRoot '../..')).Path +$results = Join-Path $env:RUNNER_TEMP ('container-exec-' + [Guid]::NewGuid().ToString('N')) +if (Test-Path $results) { throw 'The owned result directory already exists.' } +$null = New-Item -ItemType Directory -Path $results +$runtime = Join-Path $results 'runtime' +$null = New-Item -ItemType Directory -Path $runtime +"WINDOWS_EXEC_RESULTS=$results" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append +$windows = [Environment]::GetFolderPath([Environment+SpecialFolder]::Windows) +if (-not [IO.Path]::IsPathFullyQualified($windows) -or -not (Test-Path $windows -PathType Container)) { + throw 'The Windows system directory is unavailable.' +} +$system32 = Join-Path $windows 'System32' +$report = [ordered]@{ + state = 'running' + windows = [Environment]::OSVersion.VersionString + processes = [Collections.Generic.List[object]]::new() +} + +function Invoke-OwnedProcess { + param([string]$Executable, [string[]]$Arguments, [string]$Name, + [int]$TimeoutSeconds, [switch]$Sterile) + $stdoutPath = Join-Path $results "$Name.stdout" + $stderrPath = Join-Path $results "$Name.stderr" + $start = [Diagnostics.ProcessStartInfo]::new() + $start.FileName = $Executable + $start.UseShellExecute = $false + $start.WorkingDirectory = if ($Sterile) { $runtime } else { $workspace } + $start.RedirectStandardInput = $true + $start.RedirectStandardOutput = $true + $start.RedirectStandardError = $true + foreach ($argument in $Arguments) { $start.ArgumentList.Add($argument) } + if ($Sterile) { + $start.Environment.Clear() + $start.Environment['SystemRoot'] = $windows + $start.Environment['WINDIR'] = $windows + $start.Environment['PATH'] = $system32 + $start.Environment['COMSPEC'] = Join-Path $system32 'cmd.exe' + $start.Environment['TEMP'] = $runtime + $start.Environment['TMP'] = $runtime + $start.Environment['RUST_BACKTRACE'] = '1' + } + $process = [Diagnostics.Process]::new() + $process.StartInfo = $start + $stdout = [IO.File]::Open($stdoutPath, 'CreateNew', 'Write', 'ReadWrite') + $stderr = [IO.File]::Open($stderrPath, 'CreateNew', 'Write', 'ReadWrite') + $cancel = [Threading.CancellationTokenSource]::new() + $receipt = [ordered]@{ + name = $Name; started = $false; waited = $false; forced = $false; streamsJoined = $false + cleanupErrors = [Collections.Generic.List[string]]::new() + } + $report.processes.Add($receipt) + $copies = @() + $failure = $null + $clock = [Diagnostics.Stopwatch]::StartNew() + try { + if (-not $process.Start()) { throw 'The owned child did not start.' } + $receipt.started = $true + $receipt.pid = $process.Id + $process.StandardInput.Close() + $copies = @( + $process.StandardOutput.BaseStream.CopyToAsync($stdout, 81920, $cancel.Token), + $process.StandardError.BaseStream.CopyToAsync($stderr, 81920, $cancel.Token) + ) + while (-not $process.WaitForExit(100)) { + if ($clock.Elapsed.TotalSeconds -ge $TimeoutSeconds) { throw 'The owned child exceeded its time bound.' } + if ($stdout.Length + $stderr.Length -gt 64MB) { throw 'The owned child exceeded its output bound.' } + } + $receipt.waited = $true + $receipt.exitCode = $process.ExitCode + if (-not [Threading.Tasks.Task]::WhenAll([Threading.Tasks.Task[]]$copies).Wait(5000)) { + throw 'The exited child retained output handles.' + } + $receipt.streamsJoined = $true + if ($stdout.Length + $stderr.Length -gt 64MB) { throw 'The owned child exceeded its output bound.' } + if ($process.ExitCode -ne 0) { throw "The $Name child failed; see retained output." } + } catch { + $failure = $_ + $receipt.failure = $_.Exception.Message + } finally { + try { + try { + if ($receipt.started -and -not $process.HasExited) { + $receipt.forced = $true + $process.Kill($true) + } + } catch { + $receipt.cleanupErrors.Add('Terminate direct child/tree: ' + $_.Exception.Message) + if ($null -eq $failure) { $failure = $_ } + } + try { + if ($receipt.started) { + if (-not $process.WaitForExit(30000)) { throw 'The direct child did not terminate.' } + $receipt.waited = $true + $receipt.exitCode = $process.ExitCode + } + } catch { + $receipt.cleanupErrors.Add('Join direct child: ' + $_.Exception.Message) + if ($null -eq $failure) { $failure = $_ } + } + try { + if (-not $receipt.streamsJoined) { + $cancel.Cancel() + if ($receipt.started) { + try { $process.StandardOutput.Close() } finally { $process.StandardError.Close() } + } + foreach ($copy in $copies) { + try { $copy.GetAwaiter().GetResult() } catch { } + } + $receipt.streamsJoined = $true + } + } catch { + $receipt.cleanupErrors.Add('Join output readers: ' + $_.Exception.Message) + if ($null -eq $failure) { $failure = $_ } + } + } finally { + $receipt.seconds = $clock.Elapsed.TotalSeconds + $stdout.Dispose() + $stderr.Dispose() + $cancel.Dispose() + $process.Dispose() + } + } + if ($null -ne $failure) { throw $failure } + return [IO.File]::ReadAllText($stdoutPath) +} + +try { + $cargo = (Get-Command cargo -CommandType Application).Source + $rustc = (Get-Command rustc -CommandType Application).Source + $report.rust = Invoke-OwnedProcess $rustc @('--version', '--verbose') 'rust-version' 30 + $toolchain = Get-Content -Raw (Join-Path $workspace 'rust-toolchain.toml') + $channel = [regex]::Match($toolchain, '(?m)^channel = "([0-9]+\.[0-9]+\.[0-9]+)"\r?$') + if (-not $channel.Success -or + $report.rust -notmatch ('(?m)^release: ' + [regex]::Escape($channel.Groups[1].Value) + '\r?$') -or + $report.rust -notmatch '(?m)^host: x86_64-pc-windows-msvc\r?$') { + throw 'This gate requires the repository MSVC toolchain.' + } + $build = Invoke-OwnedProcess $cargo @('test', '--locked', '--release', '-p', 'spacetimedb-cli', '--lib', '--no-run', '--message-format=json') 'compile' 600 + $artifacts = @($build -split '\r?\n' | Where-Object { $_ } | ForEach-Object { + $message = $_ | ConvertFrom-Json + if ($message.reason -eq 'compiler-artifact' -and $message.profile.test -eq $true -and + $message.target.name -eq 'spacetimedb_cli' -and $message.target.kind -contains 'lib' -and $message.executable) { + $message.executable + } + } | Select-Object -Unique) + if ($artifacts.Count -ne 1) { throw 'Expected exactly one actual CLI libtest artifact.' } + $binary = (Resolve-Path $artifacts[0]).Path + $report.executable = $binary + $report.executableSha256 = (Get-FileHash -Algorithm SHA256 $binary).Hash.ToLowerInvariant() + $prefix = 'subcommands::container::execute::' + $required = @( + 'native_windows_pipe_backpressure_cancellation_joins_both_workers', + 'native_windows_overlapped_pipe_cancel_observes_exact_completion', + 'native_windows_overlapped_pipe_suppresses_inherited_completion_port_packets', + 'native_windows_close_before_ready_restores_then_releases_callback', + 'native_windows_conpty_normal_error_cancel_restore_console', + 'native_windows_console_cleanup_reports_drain_failure' + ) + $listing = Invoke-OwnedProcess $binary @($prefix, '--list') 'listing' 30 -Sterile + $selected = @() + foreach ($suffix in $required) { + $matches = [regex]::Matches($listing, '(?m)^(' + [regex]::Escape($prefix) + '[^\r\n]*::' + [regex]::Escape($suffix) + '): test\r?$') + if ($matches.Count -ne 1) { throw "Missing or ambiguous required native test: $suffix" } + $selected += $matches[0].Groups[1].Value + } + $report.requiredNativeTests = $selected + $output = Invoke-OwnedProcess $binary @($prefix, '--test-threads=1', '--show-output', '--format=pretty') 'tests' 240 -Sterile + foreach ($name in $selected) { + if (-not [regex]::IsMatch($output, '(?m)^test ' + [regex]::Escape($name) + ' \.\.\. ok\r?$')) { + throw "Required native test did not pass: $name" + } + } + if (-not [regex]::IsMatch($output, '(?m)^test result: ok\. [1-9][0-9]* passed; 0 failed;')) { + throw 'The exec suite did not report a nonempty passing result.' + } + $report.state = 'passed' +} catch { + $report.state = 'failed' + $report.failure = $_.Exception.Message + throw +} finally { + $report | ConvertTo-Json -Depth 8 | Set-Content -Encoding utf8 (Join-Path $results 'report.json') +} From 1ac388c9f96b643ec5534e9280cb2639350b9c2c Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 09:03:11 -0400 Subject: [PATCH 07/16] Fix ConPTY test child standard handle setup --- .../exec/terminal_windows/console_tests.rs | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs b/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs index b6346bd0d40..d9c78edcd2d 100644 --- a/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs +++ b/crates/cli/src/subcommands/container/exec/terminal_windows/console_tests.rs @@ -128,6 +128,11 @@ impl Child { environment.push(0); let mut startup: STARTUPINFOEXW = unsafe { std::mem::zeroed() }; startup.StartupInfo.cb = size_of::() as u32; + // Without this flag, Windows can duplicate the parent's redirected + // handles even with bInheritHandles=false. Explicit null standard + // handles let the pseudoconsole supply its own console handles. + // https://github.com/microsoft/terminal/discussions/15814 + startup.StartupInfo.dwFlags = STARTF_USESTDHANDLES; startup.lpAttributeList = attributes.0; let mut process: PROCESS_INFORMATION = unsafe { std::mem::zeroed() }; unsafe { @@ -166,12 +171,17 @@ impl Child { if let Some(process) = &self.process && !self.waited { - errors.push(anyhow::anyhow!("owned console child required forced cleanup")); unsafe { - if let Err(error) = check(TerminateProcess(process.as_raw_handle(), 1)) { - errors.push(error.context("cannot terminate owned console child")); + // An early fixture failure may already have exited before the + // ready marker. Observe that exact child before forcing it. + self.waited = WaitForSingleObject(process.as_raw_handle(), 0) == WAIT_OBJECT_0; + if !self.waited { + errors.push(anyhow::anyhow!("owned console child required forced cleanup")); + if let Err(error) = check(TerminateProcess(process.as_raw_handle(), 1)) { + errors.push(error.context("cannot terminate owned console child")); + } + self.waited = WaitForSingleObject(process.as_raw_handle(), INFINITE) == WAIT_OBJECT_0; } - self.waited = WaitForSingleObject(process.as_raw_handle(), INFINITE) == WAIT_OBJECT_0; } if !self.waited { errors.push(anyhow::anyhow!("owned console child wait failed")); @@ -259,10 +269,15 @@ fn windows_console_child() { }; let original: Vec<_> = handles .iter() - .map(|handle| { + .enumerate() + .map(|(index, handle)| { let mut value = 0; unsafe { - check(GetConsoleMode(*handle, &mut value)).unwrap(); + assert!( + GetConsoleMode(*handle, &mut value) != 0, + "owned console standard handle {index} is not a console: Win32 error {}", + GetLastError() + ); } value }) From d51106a96e8aa7b077a9c529db69c8c539f53cd4 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 19:33:15 -0400 Subject: [PATCH 08/16] Add bounded container log pages and CLI reading --- crates/cli/src/subcommands/container.rs | 3 + crates/cli/src/subcommands/container/logs.rs | 262 ++++++++++++++++++ .../src/subcommands/container/logs/tests.rs | 261 +++++++++++++++++ .../src/subcommands/container/operations.rs | 4 +- crates/lib/src/container.rs | 2 + crates/lib/src/container/logs.rs | 195 +++++++++++++ crates/lib/src/container/logs/tests.rs | 72 +++++ 7 files changed, 797 insertions(+), 2 deletions(-) create mode 100644 crates/cli/src/subcommands/container/logs.rs create mode 100644 crates/cli/src/subcommands/container/logs/tests.rs create mode 100644 crates/lib/src/container/logs.rs create mode 100644 crates/lib/src/container/logs/tests.rs diff --git a/crates/cli/src/subcommands/container.rs b/crates/cli/src/subcommands/container.rs index 92dfff937cc..8e5d2a3b5d9 100644 --- a/crates/cli/src/subcommands/container.rs +++ b/crates/cli/src/subcommands/container.rs @@ -2,6 +2,7 @@ //! server credentials; network commands use the explicitly selected server. #[path = "container/exec.rs"] mod execute; +mod logs; mod operations; mod url; @@ -20,6 +21,7 @@ pub fn cli() -> Command { .subcommand_required(true) .subcommand(url::cli()) .subcommand(execute::cli()) + .subcommand(logs::cli()) .subcommand( Command::new("build") .about("Prepare verified OCI artifacts locally without publishing") @@ -125,6 +127,7 @@ pub async fn exec(mut config: crate::Config, args: &ArgMatches) -> Result<()> { ("build", args) => exec_build(args).await, ("exec", args) => execute::exec(&mut config, args).await, ("url", args) => url::exec(&config, args).await, + ("logs", args) => logs::exec(&mut config, args).await, (name @ ("status" | "start" | "stop" | "restart"), args) => operations::exec(&mut config, name, args).await, _ => anyhow::bail!("unsupported container command"), } diff --git a/crates/cli/src/subcommands/container/logs.rs b/crates/cli/src/subcommands/container/logs.rs new file mode 100644 index 00000000000..5160eed9f8c --- /dev/null +++ b/crates/cli/src/subcommands/container/logs.rs @@ -0,0 +1,262 @@ +//! Authorized, bounded log pages pinned to one capture across requests. + +use super::operations::ContainerClient; +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use reqwest::{header::AUTHORIZATION, StatusCode}; +use spacetimedb_lib::{ + container::{ + logs::*, + operations::{ContainerApiError, ContainerErrorCode}, + }, + Hash, Identity, Uuid, +}; +use std::{io::Write, time::Duration}; + +pub(super) fn cli() -> Command { + Command::new("logs") + .about("Read retained stdout and stderr from one container attempt") + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg(crate::common_args::server()) + .arg(crate::common_args::yes()) + .arg(Arg::new("generation").long("generation").value_parser(clap::value_parser!(u64).range(1..)) + .help("Read this attempt; omitted selects the current generation once")) + .arg(Arg::new("cursor").long("cursor").requires("generation") + .help("Resume after an opaque cursor returned by --json")) + .arg(Arg::new("follow").long("follow").short('f').action(ArgAction::SetTrue) + .help("Wait for further output from the selected attempt until it ends")) + .arg(Arg::new("json").long("json").action(ArgAction::SetTrue) + .help("Print one JSON page per line, including timestamps, streams, and resume cursors")) + .after_help("A restart does not change the selected attempt. Without --json, stdout and stderr retain their original bytes and streams. Retention gaps and interrupted capture are reported on stderr.") +} + +pub(super) async fn exec(config: &mut crate::Config, args: &ArgMatches) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let server = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; + let database = args.get_one::("database").context("database is required")?; + let mut query = ContainerLogQuery { + generation: args.get_one::("generation").copied(), + cursor: args.get_one::("cursor").cloned(), + follow: args.get_flag("follow"), + }; + query.validate().map_err(anyhow::Error::msg)?; + let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; + let client = ContainerClient::new(server, auth.to_header().context("container logs require a login")?)?; + // Names are resolved only once. All continuation requests use the Identity + // and exact generation/capture returned by the first authorized page. + let mut target = database.clone(); + let mut reader = Selection::new(database.parse().ok(), &query); + let json = args.get_flag("json"); + let mut reported_loss = None; + loop { + let page = tokio::select! { + result = fetch(&client, &target, &query) => result?, + signal = tokio::signal::ctrl_c() => { signal?; return Ok(()); } + }; + reader.accept(&page)?; + { + let mut out = std::io::stdout().lock(); + let mut err = std::io::stderr().lock(); + write_page(&page, json, &mut out, &mut err, &mut reported_loss)?; + } + if let Some(end) = page.end { + if end != LogEnd::Eof || page.loss.is_some() { + bail!("container log capture ended with incomplete output; inspect --json for the recorded reason"); + } + return Ok(()); + } + if !query.follow && !page.has_more { + return Ok(()); + } + target = page.database_identity.to_hex().to_string(); + query.generation = Some(page.generation); + query.cursor = Some(page.next_cursor); + // A healthy follow request long-polls. Bound retries even if a server + // repeatedly returns an immediate heartbeat without new output. + if page.records.is_empty() { + tokio::select! { + _ = tokio::time::sleep(Duration::from_millis(200)) => {}, + signal = tokio::signal::ctrl_c() => { signal?; return Ok(()); } + } + } + } +} + +async fn fetch(client: &ContainerClient, database: &str, query: &ContainerLogQuery) -> Result { + query.validate().map_err(anyhow::Error::msg)?; + let mut response = client + .http + .get(client.url(database, "logs")?) + .header(AUTHORIZATION, client.authorization.clone()) + .query(query) + .send() + .await + .map_err(|_| anyhow::anyhow!("container logs could not be reached on the selected server"))?; + let status = response.status(); + ensure!( + response + .content_length() + .is_none_or(|length| length <= MAX_LOG_PAGE_BYTES as u64), + "container log response exceeds its size limit" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container log response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_LOG_PAGE_BYTES.saturating_sub(bytes.len()), + "container log response exceeds its size limit" + ); + bytes.extend_from_slice(&chunk); + } + if status != StatusCode::OK { + let error = serde_json::from_slice::(&bytes) + .ok() + .map(|value| value.error); + bail!( + "{} (HTTP {})", + match error { + Some(ContainerErrorCode::AccessDenied) => "database role does not permit reading container logs", + Some(ContainerErrorCode::NotFound) => "the selected container log history was not found or has expired", + Some(ContainerErrorCode::InvalidRequest) => "invalid container log selection or cursor", + Some(ContainerErrorCode::Conflict) => "the selected container log capture is no longer available", + _ => "container log service is unavailable on the selected server", + }, + status.as_u16() + ); + } + let page: ContainerLogPage = + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid container log response"))?; + page.validate().map_err(anyhow::Error::msg)?; + Ok(page) +} + +#[derive(PartialEq, Eq)] +struct Capture { + identity: Identity, + generation: u64, + revision: Hash, + publication: Uuid, + epoch: u64, + id: Uuid, +} +struct Selection { + identity: Option, + generation: Option, + capture: Option, + cursor: Option, + sequence: Option, + loss: Option, +} +impl Selection { + fn new(identity: Option, query: &ContainerLogQuery) -> Self { + Self { + identity, + generation: query.generation, + capture: None, + cursor: query.cursor.clone(), + sequence: None, + loss: None, + } + } + fn accept(&mut self, page: &ContainerLogPage) -> Result<()> { + page.validate().map_err(anyhow::Error::msg)?; + ensure!( + self.identity.is_none_or(|identity| identity == page.database_identity) + && self.generation.is_none_or(|generation| generation == page.generation), + "container logs returned another database or generation" + ); + ensure!( + self.loss.is_none_or(|loss| page.loss == Some(loss)), + "container logs changed the recorded capture loss" + ); + let capture = Capture { + identity: page.database_identity, + generation: page.generation, + revision: page.deployment_revision, + publication: page.publication_operation, + epoch: page.publication_epoch, + id: page.capture_id, + }; + ensure!( + self.capture.as_ref().is_none_or(|expected| expected == &capture), + "container logs changed the selected capture" + ); + let mut sequence = self.sequence; + for record in &page.records { + if let Some(previous) = sequence { + ensure!( + record.sequence > previous, + "container logs repeated or reordered records" + ); + ensure!( + page.retention_gap || previous.checked_add(1) == Some(record.sequence), + "container logs omitted records without a retention notice" + ); + } + sequence = Some(record.sequence); + } + ensure!( + page.records.is_empty() || self.cursor.as_ref() != Some(&page.next_cursor), + "container logs did not advance their cursor" + ); + self.identity = Some(page.database_identity); + self.generation = Some(page.generation); + self.capture = Some(capture); + self.cursor = Some(page.next_cursor.clone()); + self.sequence = sequence; + self.loss = page.loss; + Ok(()) + } +} + +fn write_page( + page: &ContainerLogPage, + json: bool, + out: &mut impl Write, + err: &mut impl Write, + reported_loss: &mut Option, +) -> Result<()> { + if json { + serde_json::to_writer(&mut *out, page)?; + out.write_all(b"\n")?; + } else { + if page.retention_gap { + writeln!(err, "[container logs: earlier records were removed by retention]")?; + } + for record in &page.records { + match &record.event { + LogEvent::Data { + stream: LogStream::Stdout, + bytes, + .. + } => out.write_all(bytes)?, + LogEvent::Data { + stream: LogStream::Stderr, + bytes, + .. + } => err.write_all(bytes)?, + LogEvent::Gap { + reason, missed_bytes, .. + } => writeln!( + err, + "[container logs: capture gap {reason:?}, missing bytes {missed_bytes:?}]" + )?, + } + } + if let Some(loss) = page.loss + && Some(loss) != *reported_loss + { + writeln!(err, "[container logs: incomplete capture, {loss:?}]")?; + } + } + *reported_loss = page.loss; + out.flush()?; + err.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/container/logs/tests.rs b/crates/cli/src/subcommands/container/logs/tests.rs new file mode 100644 index 00000000000..73d2570bdc6 --- /dev/null +++ b/crates/cli/src/subcommands/container/logs/tests.rs @@ -0,0 +1,261 @@ +use super::*; +use axum::{ + body::{Body, Bytes}, + extract::{Path, Query}, + http::{header, HeaderMap}, + response::{IntoResponse, Response}, + routing::get, + Json, Router, +}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn page() -> ContainerLogPage { + ContainerLogPage { + database_identity: Identity::ONE, + generation: 7, + deployment_revision: Hash::from_byte_array([3; 32]), + publication_operation: Uuid::from_u128(1), + publication_epoch: 2, + capture_id: Uuid::from_u128(3), + records: vec![ + LogRecord { + sequence: 1, + event: LogEvent::Data { + timestamp_micros: 1, + stream: LogStream::Stdout, + bytes: vec![0, 255, b'a'], + }, + }, + LogRecord { + sequence: 2, + event: LogEvent::Data { + timestamp_micros: 2, + stream: LogStream::Stderr, + bytes: vec![254, b'\n'], + }, + }, + ], + next_cursor: "cursor_2".into(), + oldest_retained_sequence: 1, + retention_gap: false, + has_more: false, + end: None, + loss: None, + } +} + +#[test] +fn output_preserves_bytes_streams_and_structured_metadata() -> Result<()> { + let page = page(); + let (mut out, mut err) = (vec![], vec![]); + write_page(&page, false, &mut out, &mut err, &mut None)?; + assert_eq!(out, [0, 255, b'a']); + assert_eq!(err, [254, b'\n']); + out.clear(); + err.clear(); + write_page(&page, true, &mut out, &mut err, &mut None)?; + assert!(err.is_empty()); + let decoded: ContainerLogPage = serde_json::from_slice(&out)?; + assert!(decoded == page); + assert_eq!(out.last(), Some(&b'\n')); + Ok(()) +} + +#[test] +fn continuation_pins_capture_and_rejects_replays_or_silent_gaps() -> Result<()> { + let mut selection = Selection::new(Some(Identity::ONE), &ContainerLogQuery::default()); + selection.accept(&page())?; + let mut next = page(); + next.records.clear(); + selection.accept(&next)?; // Empty heartbeat can retain the same cursor. + for changed in 0..7 { + let mut different = next.clone(); + match changed { + 0 => different.database_identity = Identity::ZERO, + 1 => different.generation += 1, + 2 => different.capture_id = Uuid::from_u128(4), + 3 => different.deployment_revision = Hash::from_byte_array([4; 32]), + 4 => different.publication_epoch += 1, + 5 => different.publication_operation = Uuid::from_u128(5), + _ => different.records = page().records, + } + assert!(selection.accept(&different).is_err()); + } + next.records.push(LogRecord { + sequence: 4, + event: LogEvent::Data { + timestamp_micros: 3, + stream: LogStream::Stdout, + bytes: b"last".to_vec(), + }, + }); + next.next_cursor = "cursor_4".into(); + assert!(selection.accept(&next).is_err()); + next.retention_gap = true; + next.oldest_retained_sequence = 4; + next.end = Some(LogEnd::Eof); + next.loss = Some(LogLoss::DrainTimeout); + selection.accept(&next)?; + next.records.clear(); + next.loss = None; + assert!(selection.accept(&next).is_err()); + Ok(()) +} + +#[test] +fn incomplete_capture_and_retention_are_visible_without_repeating_loss() -> Result<()> { + let mut page = page(); + page.records.clear(); + page.retention_gap = true; + page.loss = Some(LogLoss::DrainTimeout); + page.end = Some(LogEnd::Eof); + let (mut out, mut err, mut reported) = (vec![], vec![], None); + write_page(&page, false, &mut out, &mut err, &mut reported)?; + let text = String::from_utf8(err.clone())?; + assert!(text.contains("removed by retention")); + assert!(text.contains("incomplete capture, DrainTimeout")); + assert!(out.is_empty()); + page.retention_gap = false; + err.clear(); + write_page(&page, false, &mut out, &mut err, &mut reported)?; + assert!(err.is_empty()); + Ok(()) +} + +struct Server { + origin: String, + stop: Option>, + task: tokio::task::JoinHandle>, +} +impl Server { + async fn start(router: Router) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let origin = format!("http://{}", listener.local_addr()?); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + Ok(Self { + origin, + stop: Some(stop), + task, + }) + } + async fn shutdown(mut self) -> Result<()> { + self.stop.take().unwrap().send(()).ok(); + tokio::time::timeout(Duration::from_secs(5), &mut self.task).await???; + Ok(()) + } +} +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[tokio::test] +async fn requests_preserve_selection_and_bound_untrusted_responses() -> Result<()> { + let count = Arc::new(AtomicUsize::new(0)); + let requests = count.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/logs", + get( + move |Path(database): Path, Query(query): Query, headers: HeaderMap| { + let requests = requests.clone(); + async move { + requests.fetch_add(1, Ordering::SeqCst); + assert_eq!(headers.get(header::AUTHORIZATION).unwrap(), "Bearer disposable-test"); + assert!(!headers.contains_key(header::COOKIE)); + assert_eq!(query.generation, Some(7)); + assert_eq!(query.cursor.as_deref(), Some("cursor_0")); + assert!(query.follow); + match database.as_str() { + "oversize" => Response::new(Body::from_stream(futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from(vec![b' '; MAX_LOG_PAGE_BYTES])), + Ok(Bytes::from_static(b"x")), + ]))), + "redirect" => (StatusCode::FOUND, [(header::LOCATION, "/unexpected")]).into_response(), + "denied" => ( + StatusCode::FORBIDDEN, + Json(ContainerApiError { + error: ContainerErrorCode::AccessDenied, + }), + ) + .into_response(), + "invalid" => Json(serde_json::json!({"private diagnostic": "do not print"})).into_response(), + "project/child?query#fragment" => Json(page()).into_response(), + _ => StatusCode::NOT_FOUND.into_response(), + } + } + }, + ), + )) + .await?; + let result = async { + let client = ContainerClient::new(reqwest::Url::parse(&server.origin)?, "Bearer disposable-test".parse()?)?; + let query = ContainerLogQuery { + generation: Some(7), + cursor: Some("cursor_0".into()), + follow: true, + }; + let actual = fetch(&client, "project/child?query#fragment", &query).await?; + ensure!(actual == page(), "received different page"); + for (database, expected) in [ + ("oversize", "size limit"), + ("redirect", "HTTP 302"), + ("denied", "does not permit"), + ("invalid", "invalid container log response"), + ] { + let error = match fetch(&client, database, &query).await { + Ok(_) => bail!("expected rejection"), + Err(error) => error.to_string(), + }; + ensure!( + error.contains(expected) && !error.contains("private diagnostic"), + "wrong fixed error" + ); + } + ensure!(count.load(Ordering::SeqCst) == 5, "unexpected redirected request"); + Ok(()) + } + .await; + let shutdown = server.shutdown().await; + result.and(shutdown) +} + +#[test] +fn command_requires_generation_for_cursor_and_parses_explicit_server() { + assert!(cli() + .try_get_matches_from(["logs", "demo", "--cursor", "resume"]) + .is_err()); + assert!(cli() + .try_get_matches_from(["logs", "demo", "--generation", "0"]) + .is_err()); + let args = cli() + .try_get_matches_from([ + "logs", + "demo", + "--server", + "http://127.0.0.1:3000", + "--generation", + "7", + "--cursor", + "resume", + "--follow", + "--json", + ]) + .unwrap(); + assert_eq!(args.get_one::("generation"), Some(&7)); + assert_eq!( + args.get_one::("server").map(String::as_str), + Some("http://127.0.0.1:3000") + ); + assert!(args.get_flag("follow") && args.get_flag("json")); +} diff --git a/crates/cli/src/subcommands/container/operations.rs b/crates/cli/src/subcommands/container/operations.rs index 7f12a4ade6a..6f001cde0c9 100644 --- a/crates/cli/src/subcommands/container/operations.rs +++ b/crates/cli/src/subcommands/container/operations.rs @@ -121,9 +121,9 @@ pub(super) async fn exec(config: &mut crate::Config, name: &str, args: &ArgMatch } pub(super) struct ContainerClient { - http: Client, + pub(super) http: Client, server: Url, - authorization: HeaderValue, + pub(super) authorization: HeaderValue, } impl ContainerClient { pub(super) fn new(server: Url, mut authorization: HeaderValue) -> Result { diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs index 564f5a2cc7a..ba9b7949ab6 100644 --- a/crates/lib/src/container.rs +++ b/crates/lib/src/container.rs @@ -12,6 +12,8 @@ pub mod endpoints; #[cfg(feature = "serde")] pub mod exec; #[cfg(feature = "serde")] +pub mod logs; +#[cfg(feature = "serde")] pub mod operations; /// Version of the normalized deployment encoding, independent of module ABI. diff --git a/crates/lib/src/container/logs.rs b/crates/lib/src/container/logs.rs new file mode 100644 index 00000000000..4e9bf7e973f --- /dev/null +++ b/crates/lib/src/container/logs.rs @@ -0,0 +1,195 @@ +//! Bounded process-log pages. One selection follows one exact attempt and +//! capture; it never silently switches to a replacement container. + +use super::operations::decimal_u64; +use crate::{deployment::uuid_json, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +pub const MAX_LOG_PAGE_BYTES: usize = 512 * 1024; +pub const MAX_LOG_PAGE_RECORDS: usize = 64; +pub const MAX_LOG_RECORD_BYTES: usize = 8192; +pub const MAX_LOG_CURSOR_BYTES: usize = 512; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerLogQuery { + #[serde(default, with = "optional_decimal_u64")] + pub generation: Option, + pub cursor: Option, + #[serde(default)] + pub follow: bool, +} + +impl ContainerLogQuery { + pub fn validate(&self) -> Result<(), &'static str> { + if self.generation == Some(0) + || self + .cursor + .as_ref() + .is_some_and(|cursor| !valid_cursor(cursor) || self.generation.is_none()) + { + return Err("invalid container log selection"); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogStream { + Stdout, + Stderr, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogGapReason { + SourceRotation, + SourceBackpressure, + CaptureStartedLate, +} + +/// No Debug implementation: process output must not enter platform diagnostics. +/// bytes preserves arbitrary output, including binary data and partial lines. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LogEvent { + Data { + #[serde(with = "decimal_i64")] + timestamp_micros: i64, + stream: LogStream, + bytes: Vec, + }, + Gap { + #[serde(with = "decimal_i64")] + timestamp_micros: i64, + reason: LogGapReason, + #[serde(with = "optional_decimal_u64")] + missed_bytes: Option, + }, +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LogRecord { + #[serde(with = "decimal_u64")] + pub sequence: u64, + pub event: LogEvent, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogEnd { + Eof, + Cancelled, + SourceFailed, + OwnerLost, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogLoss { + SourceUnavailable, + SourceOpenFailed, + SourceBindingMismatch, + SourceRejected, + AttachmentTimeout, + DrainFailed, + DrainTimeout, +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerLogPage { + pub database_identity: Identity, + #[serde(with = "decimal_u64")] + pub generation: u64, + pub deployment_revision: Hash, + #[serde(with = "uuid_json")] + pub publication_operation: Uuid, + #[serde(with = "decimal_u64")] + pub publication_epoch: u64, + #[serde(with = "uuid_json")] + pub capture_id: Uuid, + pub records: Vec, + pub next_cursor: String, + #[serde(with = "decimal_u64")] + pub oldest_retained_sequence: u64, + pub retention_gap: bool, + /// More records were available when this page was read. + pub has_more: bool, + /// Present only on the page reaching the final record. A loss observation + /// remains incomplete even if the Docker source also reached natural EOF. + pub end: Option, + pub loss: Option, +} + +impl ContainerLogPage { + pub fn validate(&self) -> Result<(), &'static str> { + if self.generation == 0 + || self.publication_epoch == 0 + || self.capture_id.as_u128() == 0 + || self.publication_operation.as_u128() == 0 + || self.oldest_retained_sequence == 0 + || !valid_cursor(&self.next_cursor) + || self.records.len() > MAX_LOG_PAGE_RECORDS + || (self.has_more && (self.records.is_empty() || self.end.is_some())) + { + return Err("invalid container log page"); + } + let mut previous = 0; + for record in &self.records { + if record.sequence < self.oldest_retained_sequence + || record.sequence <= previous + || matches!(&record.event, LogEvent::Data { bytes, .. } if bytes.len() > MAX_LOG_RECORD_BYTES) + { + return Err("invalid container log record"); + } + previous = record.sequence; + } + Ok(()) + } +} + +fn valid_cursor(cursor: &str) -> bool { + (1..=MAX_LOG_CURSOR_BYTES).contains(&cursor.len()) + && cursor + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) +} + +mod decimal_i64 { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(number: &i64, serializer: S) -> Result { + serializer.collect_str(number) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + let value: i64 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal i64")); + } + Ok(value) + } +} + +mod optional_decimal_u64 { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + pub fn serialize(number: &Option, serializer: S) -> Result { + number.map(|value| value.to_string()).serialize(serializer) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + Option::::deserialize(deserializer)? + .map(|text| { + let value: u64 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal u64")); + } + Ok(value) + }) + .transpose() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/container/logs/tests.rs b/crates/lib/src/container/logs/tests.rs new file mode 100644 index 00000000000..f31c0c50894 --- /dev/null +++ b/crates/lib/src/container/logs/tests.rs @@ -0,0 +1,72 @@ +use super::*; + +fn page() -> ContainerLogPage { + ContainerLogPage { + database_identity: Identity::ONE, + generation: u64::MAX, + deployment_revision: Hash::from_byte_array([1; 32]), + publication_operation: Uuid::from_u128(1), + publication_epoch: u64::MAX, + capture_id: Uuid::from_u128(2), + records: vec![LogRecord { + sequence: u64::MAX, + event: LogEvent::Data { + timestamp_micros: i64::MIN, + stream: LogStream::Stderr, + bytes: vec![0, 255, b'\n'], + }, + }], + next_cursor: "encoded_cursor".into(), + oldest_retained_sequence: u64::MAX, + retention_gap: true, + has_more: false, + end: Some(LogEnd::Eof), + loss: Some(LogLoss::DrainTimeout), + } +} + +#[test] +fn pages_preserve_binary_output_and_exact_browser_integers() { + let expected = page(); + expected.validate().unwrap(); + let value = serde_json::to_value(&expected).unwrap(); + assert_eq!(value["generation"], u64::MAX.to_string()); + assert_eq!(value["records"][0]["event"]["timestamp_micros"], i64::MIN.to_string()); + let decoded: ContainerLogPage = serde_json::from_value(value).unwrap(); + assert!(decoded == expected); + assert_eq!(decoded.end, Some(LogEnd::Eof)); + assert_eq!(decoded.loss, Some(LogLoss::DrainTimeout)); +} + +#[test] +fn malformed_or_lossy_integer_and_capture_selections_are_rejected() { + for generation in [ + serde_json::json!(u64::MAX), + serde_json::json!("01"), + serde_json::json!("-1"), + ] { + let mut value = serde_json::to_value(page()).unwrap(); + value["generation"] = generation; + assert!(serde_json::from_value::(value).is_err()); + } + for cursor in ["", "has space", "../other"] { + let query = ContainerLogQuery { + generation: Some(1), + cursor: Some(cursor.into()), + follow: true, + }; + assert!(query.validate().is_err()); + } + assert!(ContainerLogQuery { + generation: None, + cursor: Some("encoded".into()), + follow: true + } + .validate() + .is_err()); + let mut oversized = page(); + if let LogEvent::Data { bytes, .. } = &mut oversized.records[0].event { + bytes.resize(MAX_LOG_RECORD_BYTES + 1, 0); + } + assert!(oversized.validate().is_err()); +} From 2f30e48d58025e4692ee0273eb7e653f9c5d492b Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 19:39:23 -0400 Subject: [PATCH 09/16] Regenerate CLI reference for container logs --- .../00100-cli-reference.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md index bc2d13c3b7f..9dd63aa41ba 100644 --- a/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md +++ b/docs/docs/00300-resources/00200-reference/00100-cli-reference/00100-cli-reference.md @@ -32,6 +32,7 @@ This document contains the help content for the `spacetime` command-line program * [`spacetime container`↴](#spacetime-container) * [`spacetime container url`↴](#spacetime-container-url) * [`spacetime container exec`↴](#spacetime-container-exec) +* [`spacetime container logs`↴](#spacetime-container-logs) * [`spacetime container build`↴](#spacetime-container-build) * [`spacetime container status`↴](#spacetime-container-status) * [`spacetime container start`↴](#spacetime-container-start) @@ -596,6 +597,7 @@ Build and manage a database's container * `url` — Print a container's published HTTPS URL * `exec` — Run a literal command in the current running container +* `logs` — Read retained stdout and stderr from one container attempt * `build` — Prepare verified OCI artifacts locally without publishing * `status` — Inspect container control state without opening its database * `start` — Request container execution @@ -647,6 +649,29 @@ Requires database Admin permission. No shell, container start, or reconnect is i +## `spacetime container logs` + +Read retained stdout and stderr from one container attempt + +**Usage:** `spacetime container logs [OPTIONS] ` + +A restart does not change the selected attempt. Without --json, stdout and stderr retain their original bytes and streams. Retention gaps and interrupted capture are reported on stderr. + +###### **Arguments:** + +* `` — Database name or Identity + +###### **Options:** + +* `-s`, `--server ` — The nickname, host name or URL of the server +* `-y`, `--yes` — Run non-interactively wherever possible. This will answer "yes" to almost all prompts, but will sometimes answer "no" to preserve non-interactivity (e.g. when prompting whether to log in with spacetimedb.com). +* `--generation ` — Read this attempt; omitted selects the current generation once +* `--cursor ` — Resume after an opaque cursor returned by --json +* `-f`, `--follow` — Wait for further output from the selected attempt until it ends +* `--json` — Print one JSON page per line, including timestamps, streams, and resume cursors + + + ## `spacetime container build` Prepare verified OCI artifacts locally without publishing From 52cd9426e0deaee7f6311b1665c40d6a88bebf6a Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Wed, 9 Sep 2026 20:03:02 -0400 Subject: [PATCH 10/16] Expose container image limits and reported resource usage --- .../src/subcommands/container/exec/tests.rs | 3 + .../src/subcommands/container/operations.rs | 29 ++++ crates/lib/src/container/operations.rs | 9 ++ .../lib/src/container/operations/resources.rs | 126 ++++++++++++++++++ 4 files changed, 167 insertions(+) create mode 100644 crates/lib/src/container/operations/resources.rs diff --git a/crates/cli/src/subcommands/container/exec/tests.rs b/crates/cli/src/subcommands/container/exec/tests.rs index bb8b2f7f2ac..356f92ac528 100644 --- a/crates/cli/src/subcommands/container/exec/tests.rs +++ b/crates/cli/src/subcommands/container/exec/tests.rs @@ -186,6 +186,7 @@ fn status_requires_the_exact_running_generation_without_http_readiness() { let mut status = ContainerStatus { database_identity: Identity::ZERO, published: true, + configuration: None, endpoints: EndpointStatus::Pending, operational: Some(OperationalState { desired_revision: spacetimedb_lib::Hash::from_byte_array([0; 32]), @@ -202,6 +203,7 @@ fn status_requires_the_exact_running_generation_without_http_readiness() { applied_env_generation: None, exit_code: None, oom_killed: false, + usage: None, condition: Condition::None, }), }), @@ -367,6 +369,7 @@ async fn status_uses_authenticated_explicit_loopback_and_pins_the_resolved_ident let status = ContainerStatus { database_identity: identity, published: true, + configuration: None, operational: None, endpoints: EndpointStatus::Pending, }; diff --git a/crates/cli/src/subcommands/container/operations.rs b/crates/cli/src/subcommands/container/operations.rs index 6f001cde0c9..4381500f93c 100644 --- a/crates/cli/src/subcommands/container/operations.rs +++ b/crates/cli/src/subcommands/container/operations.rs @@ -266,11 +266,40 @@ fn print_status(status: &ContainerStatus) { if !status.published { println!("Container: none published"); } + if let Some(configuration) = &status.configuration { + println!("Published image: {}", configuration.image_digest); + let limits = &configuration.resources; + println!( + "Published limits: {} millicores, {} memory bytes, {} scratch bytes, {} tasks", + limits.cpu_millicores, limits.memory_bytes, limits.scratch_bytes, limits.pids_max + ); + } if let Some(state) = &status.operational { println!("Desired: {:?} (generation {})", state.desired_state, state.generation); println!("Condition: {:?}", state.condition); if let Some(instance) = &state.current_instance { println!("Observed: {:?}", instance.state); + if let Some(usage) = &instance.usage { + println!( + "Reported usage: sample {}{}", + usage.sample_sequence, + if usage.final_report { " (final totals)" } else { "" } + ); + let totals = &usage.cumulative; + println!( + "CPU: {} ns; memory: {} byte-seconds; scratch: {} byte-seconds; sent: {} bytes; received: {} bytes", + totals.cpu_nanoseconds, + totals.memory_byte_seconds, + totals.scratch_byte_seconds, + totals.transmitted_bytes, + totals.received_bytes + ); + if usage.measurement_interrupted { + println!("Measurement interrupted by host restart; totals include known usage only."); + } + } else { + println!("Usage: not reported for this generation"); + } if let Some(environment) = &instance.applied_env_generation { println!("Environment generation: {environment}"); } diff --git a/crates/lib/src/container/operations.rs b/crates/lib/src/container/operations.rs index 1f477b229f7..5c1d6b7118f 100644 --- a/crates/lib/src/container/operations.rs +++ b/crates/lib/src/container/operations.rs @@ -2,6 +2,9 @@ use crate::{container::endpoints::ContainerEndpoint, deployment::uuid_json, Hash, Identity, Uuid}; use serde::{Deserialize, Serialize}; +mod resources; +pub use resources::{ContainerConfiguration, ReportedUsage, ResourceLimits, UsageTotals}; + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ContainerAction { @@ -91,6 +94,9 @@ pub struct CurrentInstance { pub exit_code: Option, pub oom_killed: bool, pub condition: Condition, + /// Last accepted cumulative measurement for this exact generation. Missing + /// or not-yet-reported usage is None, never manufactured zero consumption. + pub usage: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -123,6 +129,9 @@ pub enum EndpointStatus { pub struct ContainerStatus { pub database_identity: Identity, pub published: bool, + /// Latest published image and limits, which may differ from a draining + /// instance. No argv, environment, credentials, or private node addresses. + pub configuration: Option, pub operational: Option, /// Address discovery is independent of readiness and remains available /// while stopped. Pending never means an empty declaration. diff --git a/crates/lib/src/container/operations/resources.rs b/crates/lib/src/container/operations/resources.rs new file mode 100644 index 00000000000..8d45bdd0a48 --- /dev/null +++ b/crates/lib/src/container/operations/resources.rs @@ -0,0 +1,126 @@ +//! Browser-safe resource metadata and accepted cumulative usage, not live gauges. +use super::decimal_u64; +use crate::container::OciDigest; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerConfiguration { + pub image_digest: OciDigest, + pub resources: ResourceLimits, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResourceLimits { + #[serde(with = "decimal_u64")] + pub cpu_millicores: u64, + #[serde(with = "decimal_u64")] + pub memory_bytes: u64, + #[serde(with = "decimal_u64")] + pub scratch_bytes: u64, + #[serde(with = "decimal_u64")] + pub pids_max: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReportedUsage { + #[serde(with = "decimal_u64")] + pub sample_sequence: u64, + pub cumulative: UsageTotals, + /// Accounting has accepted its final totals. This flag does not establish + /// the container's observed state or physical storage reclamation. + pub final_report: bool, + /// A host reboot interrupted measurement in this or an earlier segment. + /// Cumulative totals include known usage only; the gap is not zero usage. + pub measurement_interrupted: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UsageTotals { + #[serde(with = "decimal_u128")] + pub cpu_nanoseconds: u128, + #[serde(with = "decimal_u128")] + pub memory_byte_seconds: u128, + #[serde(with = "decimal_u128")] + pub scratch_byte_seconds: u128, + #[serde(with = "decimal_u128")] + pub transmitted_bytes: u128, + #[serde(with = "decimal_u128")] + pub received_bytes: u128, +} + +mod decimal_u128 { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(number: &u128, serializer: S) -> Result { + serializer.collect_str(number) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + let value: u128 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal u128")); + } + Ok(value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cumulative_usage_preserves_large_counters_and_measurement_gaps() { + let usage = ReportedUsage { + sample_sequence: u64::MAX, + cumulative: UsageTotals { + cpu_nanoseconds: u128::MAX, + memory_byte_seconds: u128::MAX - 1, + scratch_byte_seconds: 0, + transmitted_bytes: 9_007_199_254_740_993, + received_bytes: 7, + }, + final_report: true, + measurement_interrupted: true, + }; + let encoded = serde_json::to_value(&usage).unwrap(); + assert_eq!(encoded["cumulative"]["cpu_nanoseconds"], u128::MAX.to_string()); + assert_eq!(encoded["cumulative"]["transmitted_bytes"], "9007199254740993"); + assert_eq!(serde_json::from_value::(encoded.clone()).unwrap(), usage); + for value in [ + serde_json::json!(1), + serde_json::json!("01"), + serde_json::json!("-1"), + serde_json::json!("340282366920938463463374607431768211456"), + ] { + let mut invalid = encoded.clone(); + invalid["cumulative"]["cpu_nanoseconds"] = value; + assert!(serde_json::from_value::(invalid).is_err()); + } + let mut invalid = encoded; + invalid["credential"] = serde_json::json!("must not appear"); + assert!(serde_json::from_value::(invalid).is_err()); + } + + #[test] + fn configuration_uses_the_oci_digest_and_exact_limits() { + let configuration = ContainerConfiguration { + image_digest: format!("sha256:{}", "ab".repeat(32)).parse().unwrap(), + resources: ResourceLimits { + cpu_millicores: 500, + memory_bytes: u64::MAX, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + }; + let encoded = serde_json::to_value(&configuration).unwrap(); + assert_eq!(encoded["image_digest"], format!("sha256:{}", "ab".repeat(32))); + assert_eq!(encoded["resources"]["memory_bytes"], u64::MAX.to_string()); + assert_eq!( + serde_json::from_value::(encoded).unwrap(), + configuration + ); + } +} From 1185be08a86ad086f87146acdf9a0e4d8b10cffa Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 03:41:06 -0400 Subject: [PATCH 11/16] Add container credential discovery and managed Rust SDK sessions --- Cargo.lock | 3 + crates/bindings-typescript/README.md | 88 +++ crates/bindings-typescript/package.json | 15 +- .../src/sdk/node/container.ts | 407 ++++++++++ .../bindings-typescript/src/sdk/node/index.ts | 157 ++++ .../tests/node_credentials.test.ts | 376 ++++++++++ .../tests/node_websocket.test.ts | 324 ++++++++ crates/bindings-typescript/tsup.config.ts | 14 + pnpm-lock.yaml | 6 + sdks/rust/Cargo.toml | 3 + sdks/rust/src/credentials.rs | 13 +- sdks/rust/src/credentials/container.rs | 474 ++++++++++++ sdks/rust/src/credentials/container/tests.rs | 297 ++++++++ sdks/rust/src/db_connection.rs | 106 ++- sdks/rust/src/db_connection/builder_tests.rs | 150 ++++ .../src/db_connection/container_session.rs | 621 +++++++++++++++ .../db_connection/container_session/tests.rs | 704 ++++++++++++++++++ sdks/rust/src/db_connection/native_tasks.rs | 8 + sdks/rust/src/lib.rs | 5 + 19 files changed, 3758 insertions(+), 13 deletions(-) create mode 100644 crates/bindings-typescript/src/sdk/node/container.ts create mode 100644 crates/bindings-typescript/src/sdk/node/index.ts create mode 100644 crates/bindings-typescript/tests/node_credentials.test.ts create mode 100644 crates/bindings-typescript/tests/node_websocket.test.ts create mode 100644 sdks/rust/src/credentials/container.rs create mode 100644 sdks/rust/src/credentials/container/tests.rs create mode 100644 sdks/rust/src/db_connection/container_session.rs create mode 100644 sdks/rust/src/db_connection/container_session/tests.rs diff --git a/Cargo.lock b/Cargo.lock index b1a6cab1682..e06e7f60bee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8675,6 +8675,9 @@ dependencies = [ "native-tls", "once_cell", "prometheus", + "reqwest", + "serde", + "serde_json", "shlex", "spacetimedb-client-api-messages", "spacetimedb-data-structures", diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index ddf546c296e..3ab18839fa6 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -255,3 +255,91 @@ by the host in a submodule fail at runtime; ordinary helpers retain their caller scope. Values are private, durable database configuration for secrets and other settings. Database owners and authorized collaborators can read them; module code can expose them through its own outputs. + +### Node.js Authorization transport + +Node.js clients can send a token directly in the WebSocket `Authorization` +header by selecting the explicit Node transport. Install its optional peer: + +```sh +npm install ws +``` + +```ts +import { openNodeWebSocket } from 'spacetimedb/sdk/node'; +import { DbConnection } from './module_bindings'; + +function connect( + serverUri: string, + databaseIdentity: string, + freshToken: string +) { + return DbConnection.builder() + .withUri(serverUri) + .withDatabaseName(databaseIdentity) + .withToken(freshToken) + .withWSFn(openNodeWebSocket) + .onConnect(connection => { + connection.subscriptionBuilder().subscribe('SELECT * FROM jobs'); + }) + .build(); +} +``` + +Use a trusted `wss://` server in deployed clients. This transport sends the +provided token only in the upgrade request header, preserves the SDK connection +ID and subscription options, and refuses redirects. It does not exchange the +token through the generic identity endpoint or place credentials in the URL. +The ordinary browser transport is unchanged. + +This factory transports caller-supplied credentials; it does not discover or +renew container credentials. Obtain a fresh target-specific credential for each +new connection and recreate subscriptions when reconnecting. Unconfirmed +reducer and procedure calls may already have committed and must not be +replayed automatically. Do not log or persist credentials. + +`connection.disconnect()` initiates closure. If retaining the adapter from a +custom `withWSFn` wrapper, `await adapter.shutdown()` waits for actual transport +closure. A stalled upgrade or a peer that ignores the close handshake is +terminated after five seconds. + +### Credentials inside a container + +The Node entry point also reads the platform's discovery variables and obtains +short-lived credentials from the local broker: + +```ts +import { Container, openNodeWebSocket } from 'spacetimedb/sdk/node'; +import { DbConnection } from './module_bindings'; + +const container = Container.fromEnvironment(); +const credential = await container.tokenFor(); +const connection = DbConnection.builder() + .withUri(container.serverUri) + .withDatabaseName(credential.target.toHexString()) + .withToken(credential.value) + .withWSFn(openNodeWebSocket) + .onConnect(connection => { + connection.subscriptionBuilder().subscribe('SELECT * FROM jobs'); + }) + .build(); +``` + +`tokenFor()` targets the container's own database. Pass another database's +`Identity` to request a credential for it, then connect to that database's trusted +server. The broker determines whether access is allowed. For an HTTP request, +send `credential.value` in the `Authorization: Bearer ...` header. + +`Container.fromEnvironment()` reads only `SPACETIMEDB_DATABASE_IDENTITY`, +`SPACETIMEDB_SERVER_URI`, and `SPACETIMEDB_CREDENTIAL_BROKER`. Missing or invalid +values fail explicitly. `new Container({ databaseIdentity, serverUri, +credentialBroker })` accepts explicit discovery configuration. The local broker +uses an absolute Unix socket or numeric loopback HTTP address; requests neither +follow redirects nor use saved credentials or proxy settings. + +Each request opens a fresh connection and completes within five seconds, with +an optional `AbortSignal` as the second argument to `tokenFor`. Tokens last at +most 30 seconds. `expiresAt` and `remainingLifetimeMs` expose their remaining +validity; logging or serializing the token object redacts its value. Rebuild the +database connection with a fresh token and restore subscriptions on expiry or +disconnection. This helper does not manage connection renewal or replay calls. diff --git a/crates/bindings-typescript/package.json b/crates/bindings-typescript/package.json index db598440673..0a5bfe2f2f6 100644 --- a/crates/bindings-typescript/package.json +++ b/crates/bindings-typescript/package.json @@ -101,6 +101,12 @@ "import": "./dist/solid/index.mjs", "require": "./dist/solid/index.cjs", "default": "./dist/solid/index.mjs" + }, + "./sdk/node": { + "types": "./dist/sdk/node/index.d.ts", + "import": "./dist/sdk/node/index.mjs", + "require": "./dist/sdk/node/index.cjs", + "default": "./dist/sdk/node/index.mjs" } }, "size-limit": [ @@ -189,7 +195,8 @@ "pure-rand": "^7.0.1", "safe-stable-stringify": "^2.5.0", "statuses": "^2.0.2", - "url-polyfill": "^1.1.14" + "url-polyfill": "^1.1.14", + "@types/ws": "^8.18.1" }, "peerDependencies": { "@angular/core": ">=17.0.0", @@ -198,7 +205,8 @@ "solid-js": "^1.6.0", "svelte": "^4.0.0 || ^5.0.0", "undici": "^6.19.2", - "vue": "^3.3.0" + "vue": "^3.3.0", + "ws": "^8.18.3" }, "peerDependenciesMeta": { "@tanstack/react-query": { @@ -221,6 +229,9 @@ }, "@angular/core": { "optional": true + }, + "ws": { + "optional": true } }, "devDependencies": { diff --git a/crates/bindings-typescript/src/sdk/node/container.ts b/crates/bindings-typescript/src/sdk/node/container.ts new file mode 100644 index 00000000000..77d059f08b0 --- /dev/null +++ b/crates/bindings-typescript/src/sdk/node/container.ts @@ -0,0 +1,407 @@ +import { request, type ClientRequest, type IncomingMessage } from 'node:http'; +import { isIP, type Socket } from 'node:net'; +import { performance } from 'node:perf_hooks'; +import { inspect } from 'node:util'; +import { Identity } from '../../lib/identity'; + +const REQUEST_MS = 5_000; +const MAX_BODY = 8192 + 256; +const MAX_HEADERS = 4096; +const LOCAL_AUTHORITY = '127.0.0.1:18081'; +const CREDENTIAL_PATH = '/v1/credentials'; + +export type ContainerCredentialErrorCode = + | 'missing_environment' + | 'invalid_discovery' + | 'invalid_target' + | 'denied' + | 'unavailable' + | 'transport' + | 'timeout' + | 'aborted' + | 'invalid_response'; + +/** Errors contain no discovery values, response bodies or credentials. */ +export class ContainerCredentialError extends Error { + constructor(readonly code: ContainerCredentialErrorCode) { + super(`Container credential request failed: ${code}`); + this.name = 'ContainerCredentialError'; + } +} + +type Endpoint = + | { socketPath: string } + | { hostname: string; port: number; host: string }; + +/** Discovery values injected by the container platform; these are not secrets. */ +export interface ContainerDiscovery { + databaseIdentity: string; + serverUri: string; + credentialBroker: string; +} + +/** + * Explicit discovery and short-lived credentials for a Node.js process. + * No stored CLI credentials, default server, anonymous Identity, redirects or + * proxy configuration are used. Construction performs no I/O. + */ +export class Container { + #identity: string; + #server: string; + #endpoint: Endpoint; + + constructor(discovery: ContainerDiscovery) { + this.#identity = identity(discovery.databaseIdentity, 'invalid_discovery'); + const server = parseUrl(discovery.serverUri); + if (!['http:', 'https:', 'ws:', 'wss:'].includes(server.protocol)) { + throw new ContainerCredentialError('invalid_discovery'); + } + if (!server.hostname) { + throw new ContainerCredentialError('invalid_discovery'); + } + this.#server = server.href; + this.#endpoint = endpoint(discovery.credentialBroker); + } + + /** Read only the three platform discovery variables. */ + static fromEnvironment( + environment: Readonly> = process.env + ): Container { + const read = (name: string): string => { + const value = environment[name]; + if (value === undefined) { + throw new ContainerCredentialError('missing_environment'); + } + return value; + }; + return new Container({ + databaseIdentity: read('SPACETIMEDB_DATABASE_IDENTITY'), + serverUri: read('SPACETIMEDB_SERVER_URI'), + credentialBroker: read('SPACETIMEDB_CREDENTIAL_BROKER'), + }); + } + + get databaseIdentity(): Identity { + return new Identity(this.#identity); + } + + get serverUri(): string { + return this.#server; + } + + /** + * Request a fresh token for an exact target Identity, defaulting to this + * container's database. The broker authenticates the process independently + * of these discovery values. Send the token only to the target's trusted + * server. This does not reconnect a database connection or replay calls. + */ + async tokenFor( + target: Identity = this.databaseIdentity, + signal?: AbortSignal + ): Promise { + let targetHex: string; + try { + targetHex = identity(target.toHexString(), 'invalid_target'); + } catch { + throw new ContainerCredentialError('invalid_target'); + } + const response = await readToken(this.#endpoint, targetHex, signal); + return ContainerToken.fromResponse(response, targetHex); + } + + [inspect.custom](): string { + return 'Container { discovery: }'; + } +} + +/** An in-memory bearer token. Inspection and JSON serialization are redacted. */ +export class ContainerToken { + #token: string; + #target: string; + #expiry: number; + #deadline: number; + + private constructor( + token: string, + target: string, + expiry: number, + remaining: number + ) { + this.#token = token; + this.#target = target; + this.#expiry = expiry; + this.#deadline = performance.now() + remaining; + } + + /** @internal */ + static fromResponse(bytes: Buffer, target: string): ContainerToken { + try { + const value: unknown = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(bytes) + ); + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error(); + const fields = value as Record; + const token = fields.token; + const seconds = fields.expires_unix_seconds; + if ( + Object.keys(fields).length !== 2 || + typeof token !== 'string' || + !/^[\x21-\x7e]{1,8192}$/.test(token) || + typeof seconds !== 'number' || + !Number.isSafeInteger(seconds) || + !Number.isSafeInteger(seconds * 1000) + ) + throw new Error(); + const expiry = seconds * 1000; + const remaining = expiry - Date.now(); + if (remaining <= 0 || remaining > 30_000) throw new Error(); + return new ContainerToken(token, target, expiry, remaining); + } catch { + throw new ContainerCredentialError('invalid_response'); + } + } + + /** Do not log, persist, or place this value in a URL or environment variable. */ + get value(): string { + return this.#token; + } + + get target(): Identity { + return new Identity(this.#target); + } + + get expiresAt(): Date { + return new Date(this.#expiry); + } + + /** A monotonic cap prevents a backward wall-clock jump from extending it. */ + get remainingLifetimeMs(): number { + return Math.max( + 0, + Math.min(this.#deadline - performance.now(), this.#expiry - Date.now()) + ); + } + + toJSON(): object { + return { + target: this.#target, + expiresAt: this.expiresAt, + token: '', + }; + } + + [inspect.custom](): object { + return this.toJSON(); + } +} + +function identity(value: string, code: ContainerCredentialErrorCode): string { + if (typeof value !== 'string' || !/^[0-9a-fA-F]{64}$/.test(value)) { + throw new ContainerCredentialError(code); + } + return value.toLowerCase(); +} + +function parseUrl(value: string): URL { + try { + if ( + !value || + Buffer.byteLength(value) > 4096 || + [...value].some(c => c.charCodeAt(0) <= 32 || c.charCodeAt(0) === 127) + ) + throw new Error(); + const url = new URL(value); + if ( + url.username || + url.password || + value.includes('?') || + value.includes('#') + ) + throw new Error(); + return url; + } catch { + throw new ContainerCredentialError('invalid_discovery'); + } +} + +function endpoint(value: string): Endpoint { + const url = parseUrl(value); + if (url.protocol === 'unix:') { + const path = value.slice('unix://'.length); + if ( + !value.startsWith('unix:///') || + url.host || + !path.startsWith('/') || + Buffer.byteLength(path) > 103 || + path.includes('%') || + path.includes('\\') || + path.split('/').some(part => part === '.' || part === '..') + ) + throw new ContainerCredentialError('invalid_discovery'); + return { socketPath: path }; + } + // Validate the original authority before URL normalization can turn a DNS + // spelling or an abbreviated/hexadecimal IPv4 address into loopback. + const match = + /^http:\/\/(\[[0-9a-fA-F:]+\]|[0-9.]+)(?::([0-9]+))?\/v1\/credentials$/.exec( + value + ); + if (!match) throw new ContainerCredentialError('invalid_discovery'); + const hostname = match[1].replace(/^\[|\]$/g, ''); + const version = isIP(hostname); + if ( + (version !== 4 || hostname.split('.')[0] !== '127') && + (version !== 6 || new URL(`http://[${hostname}]/`).hostname !== '[::1]') + ) + throw new ContainerCredentialError('invalid_discovery'); + const port = match[2] === undefined ? 80 : Number(match[2]); + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new ContainerCredentialError('invalid_discovery'); + } + return { hostname, port, host: url.host }; +} + +function responseLength(response: IncomingMessage): number { + if (response.statusCode === 401 || response.statusCode === 403) { + throw new ContainerCredentialError('denied'); + } + if (response.statusCode === 503) + throw new ContainerCredentialError('unavailable'); + if (response.statusCode !== 200) + throw new ContainerCredentialError('invalid_response'); + const count = (name: string): number => + response.rawHeaders.filter( + (v, i) => i % 2 === 0 && v.toLowerCase() === name + ).length; + const headers = response.headers; + if ( + count('content-type') !== 1 || + headers['content-type'] !== 'application/json' || + count('cache-control') !== 1 || + headers['cache-control'] !== 'no-store' || + count('content-length') !== 1 || + count('transfer-encoding') || + count('content-encoding') || + typeof headers['content-length'] !== 'string' || + !/^[0-9]+$/.test(headers['content-length']) + ) + throw new ContainerCredentialError('invalid_response'); + const length = Number(headers['content-length']); + if (!Number.isInteger(length) || length <= 0 || length > MAX_BODY) { + throw new ContainerCredentialError('invalid_response'); + } + return length; +} + +/** Own the request and its socket through terminal close, on every outcome. */ +async function readToken( + endpoint: Endpoint, + target: string, + signal?: AbortSignal +): Promise { + if (signal?.aborted) throw new ContainerCredentialError('aborted'); + return new Promise((resolve, reject) => { + const body = JSON.stringify({ target_database: target }); + let req: ClientRequest | undefined; + let socket: Socket | undefined; + let requestClosed = false; + let socketClosed = false; + let outcome: + | { error?: ContainerCredentialError; bytes?: Buffer } + | undefined; + let timer: ReturnType | undefined; + const complete = (): void => { + if (!outcome || !requestClosed || (socket && !socketClosed)) return; + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + if (outcome.error) reject(outcome.error); + else resolve(outcome.bytes!); + }; + const finish = (error?: ContainerCredentialError, bytes?: Buffer): void => { + if (!outcome) outcome = { error, bytes }; + req?.destroy(); + socket?.destroy(); + complete(); + }; + const fail = (code: ContainerCredentialErrorCode): void => + finish(new ContainerCredentialError(code)); + const abort = (): void => fail('aborted'); + try { + req = request({ + ...endpoint, + // A fresh private agent owns one connection; ambient proxy and HTTP + // pool settings cannot redirect this credential request. + agent: false, + method: 'POST', + path: CREDENTIAL_PATH, + maxHeaderSize: MAX_HEADERS, + headers: { + Host: 'socketPath' in endpoint ? LOCAL_AUTHORITY : endpoint.host, + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body), + Connection: 'close', + }, + }); + req.once('socket', owned => { + socket = owned; + owned.once('close', () => { + socketClosed = true; + complete(); + }); + if (outcome) owned.destroy(); + }); + req.once('error', () => fail('transport')); + req.once('close', () => { + requestClosed = true; + if (!outcome) fail('transport'); + complete(); + }); + req.once('response', response => { + response.on('error', () => fail('transport')); + let length: number; + try { + length = responseLength(response); + } catch (error) { + finish( + error instanceof ContainerCredentialError + ? error + : new ContainerCredentialError('invalid_response') + ); + return; + } + const chunks: Buffer[] = []; + let received = 0; + response.on('data', (chunk: Buffer) => { + received += chunk.length; + if (received > length) { + fail('invalid_response'); + return; + } + chunks.push(chunk); + }); + response.once('aborted', () => fail('transport')); + response.once('end', () => { + if (received !== length) fail('invalid_response'); + else finish(undefined, Buffer.concat(chunks, received)); + }); + }); + // Upgrades never become an untracked or credential-bearing connection. + req.once('upgrade', (_response, upgraded) => { + upgraded.destroy(); + fail('invalid_response'); + }); + timer = setTimeout(() => fail('timeout'), REQUEST_MS); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + else req.end(body); + } catch { + if (req) fail('transport'); + else { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + reject(new ContainerCredentialError('transport')); + } + } + }); +} diff --git a/crates/bindings-typescript/src/sdk/node/index.ts b/crates/bindings-typescript/src/sdk/node/index.ts new file mode 100644 index 00000000000..f9cf9df2a05 --- /dev/null +++ b/crates/bindings-typescript/src/sdk/node/index.ts @@ -0,0 +1,157 @@ +import { WebSocket } from 'ws'; +import { WebsocketDecompressAdapter } from '../websocket_decompress_adapter'; +import type { WebSocketArgs } from '../ws'; + +const IO_TIMEOUT_MS = 5_000; + +/** + * A Node.js socket with an awaited transport shutdown. + * Closure does not establish whether an unconfirmed reducer or procedure + * committed. Callers must never automatically replay those requests. + */ +export class NodeWebSocketAdapter extends WebsocketDecompressAdapter { + #socket: WebSocket; + #timer?: ReturnType; + #completion: Promise; + #closing = false; + + constructor(socket: WebSocket) { + super(socket as unknown as globalThis.WebSocket); + this.#socket = socket; + this.#completion = + socket.readyState === WebSocket.CLOSED + ? Promise.resolve() + : new Promise(resolve => { + socket.once('close', () => { + clearTimeout(this.#timer); + resolve(); + }); + }); + // An early failed handshake must be handled even before the SDK installs + // its error callback. The callback below receives only a fixed diagnostic. + socket.on('error', () => {}); + } + + static override async openWebSocket( + args: WebSocketArgs + ): Promise { + return openNodeWebSocket(args); + } + + override set onerror(handler: (event: globalThis.ErrorEvent) => void) { + this.#socket.onerror = () => + handler( + Object.assign(new Event('error'), { + message: 'Database WebSocket transport failed', + filename: '', + lineno: 0, + colno: 0, + error: undefined, + }) + ); + } + + override close(): void { + if (this.#closing || this.#socket.readyState === WebSocket.CLOSED) return; + this.#closing = true; + if (this.#socket.readyState === WebSocket.CONNECTING) { + this.#socket.terminate(); + return; + } + // terminate() owns the upgraded socket, unlike destroying an HTTP pool + // that has already transferred socket ownership to WebSocket. + this.#timer = setTimeout(() => this.#socket.terminate(), IO_TIMEOUT_MS); + this.#socket.close(); + } + + /** Close the WebSocket and await its actual terminal close event. */ + async shutdown(): Promise { + this.close(); + await this.#completion; + } +} + +/** + * Node.js WebSocket factory for `DbConnection.builder().withWSFn(...)`. + * Sends the supplied token in Authorization without exchanging it for a + * generic WebSocket token or placing it in the URL. Install the optional + * `ws` peer dependency before importing `spacetimedb/sdk/node`. + * + * This transports a token supplied by the caller. It does not fetch or renew + * container credentials. An expired connection must obtain fresh credentials + * and create a new database connection and subscriptions. + */ +export async function openNodeWebSocket( + args: WebSocketArgs +): Promise { + const { url, nameOrAddress, authToken } = args; + const query = [...url.searchParams]; + const connectionId = url.searchParams.get('connection_id'); + if ( + !['ws:', 'wss:'].includes(url.protocol) || + url.username || + url.password || + (query.length !== 0 && + (query.length !== 1 || + query[0][0] !== 'connection_id' || + !/^[0-9a-f]{32}$/.test(connectionId ?? ''))) || + url.hash || + !nameOrAddress || + nameOrAddress === '.' || + nameOrAddress === '..' || + nameOrAddress.length > 256 || + [...nameOrAddress].some(character => { + const code = character.charCodeAt(0); + return code < 32 || code === 127; + }) || + (authToken !== undefined && + (!authToken || + authToken.length > 8192 || + !/^[\x21-\x7e]+$/.test(authToken))) + ) { + throw new Error('Invalid Node.js database WebSocket configuration'); + } + const target = new URL( + `v1/database/${encodeURIComponent(nameOrAddress)}/subscribe`, + url + ); + if (connectionId !== null) { + target.searchParams.set('connection_id', connectionId); + } + target.searchParams.set( + 'compression', + { gzip: 'Gzip', brotli: 'Brotli', none: 'None' }[args.compression] + ); + if (args.lightMode) target.searchParams.set('light', 'true'); + if (args.confirmedReads !== undefined) { + target.searchParams.set('confirmed', String(args.confirmedReads)); + } + try { + const socket = new WebSocket(target, args.wsProtocol, { + // A private one-use HTTP agent avoids inherited global proxy settings. + // The WebSocket itself retains its upgraded socket for terminate(). + agent: false, + followRedirects: false, + // Never inherit NODE_TLS_REJECT_UNAUTHORIZED=0 for bearer credentials. + rejectUnauthorized: true, + handshakeTimeout: IO_TIMEOUT_MS, + perMessageDeflate: false, + headers: + authToken === undefined ? {} : { Authorization: `Bearer ${authToken}` }, + }); + socket.binaryType = 'arraybuffer'; + return new NodeWebSocketAdapter(socket); + } catch { + throw new Error('Database WebSocket transport could not start'); + } +} + +export { + Container, + ContainerToken, + ContainerCredentialError, +} from './container'; +export type { + ContainerDiscovery, + ContainerCredentialErrorCode, +} from './container'; diff --git a/crates/bindings-typescript/tests/node_credentials.test.ts b/crates/bindings-typescript/tests/node_credentials.test.ts new file mode 100644 index 00000000000..8701e7e348e --- /dev/null +++ b/crates/bindings-typescript/tests/node_credentials.test.ts @@ -0,0 +1,376 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { + createServer, + type Server, + type ServerResponse, + type RequestListener, +} from 'node:http'; +import { once } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { + createServer as createNetServer, + type Server as NetServer, + type AddressInfo, + type Socket, +} from 'node:net'; +import { join } from 'node:path'; +import { inspect } from 'node:util'; +import { Container, ContainerCredentialError } from '../src/sdk/node/container'; +import { Identity } from '../src/lib/identity'; + +const OWN = '1'.repeat(64); +const OTHER = '2'.repeat(64); +const SECRET = 'opaque-fixture-credential'; +const servers: NetServer[] = []; +const sockets = new Set(); +const directories: string[] = []; + +async function fixture( + handler: RequestListener, + unix = false +): Promise<{ server: Server; broker: string }> { + const server = createServer(handler); + servers.push(server); + server.on('connection', socket => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + }); + if (unix) { + const directory = await mkdtemp('/tmp/node-creds-'); + directories.push(directory); + const path = join(directory, 'sock'); + server.listen(path); + await once(server, 'listening'); + return { server, broker: `unix://${path}` }; + } + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return { + server, + broker: `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/credentials`, + }; +} + +async function rawFixture( + reply: string, + truncate = false +): Promise<{ server: NetServer; broker: string }> { + const server = createNetServer(socket => { + sockets.add(socket); + socket.on('error', () => {}); + socket.on('close', () => sockets.delete(socket)); + // A raw upgraded peer can retain its writable half after client EOF. + socket.on('end', () => socket.end()); + socket.once('data', () => { + socket.write(reply); + if (truncate) socket.end(); + }); + }); + servers.push(server); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return { + server, + broker: `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/credentials`, + }; +} + +async function rejectsAndClosesPeer( + reply: string, + code: string, + truncate = false +): Promise { + const { server, broker } = await rawFixture(reply, truncate); + const connected = once(server, 'connection'); + const result = expect(container(broker).tokenFor()).rejects.toMatchObject({ + code, + }); + const [peer] = (await connected) as [Socket]; + const closed = once(peer, 'close'); + await result; + // Observe closure before afterEach can destroy a retained socket. + await closed; + expect(peer.destroyed).toBe(true); +} + +afterEach(async () => { + for (const socket of sockets) socket.destroy(); + await Promise.all( + servers.splice(0).map( + server => + new Promise((resolve, reject) => { + server.close(error => (error ? reject(error) : resolve())); + }) + ) + ); + await Promise.all( + directories + .splice(0) + .map(path => rm(path, { recursive: true, force: true })) + ); +}); + +function container(broker: string): Container { + return new Container({ + databaseIdentity: OWN, + serverUri: 'https://127.0.0.1:1/', + credentialBroker: broker, + }); +} + +function success( + response: ServerResponse, + value: unknown = { + token: SECRET, + expires_unix_seconds: Math.floor(Date.now() / 1000) + 10, + } +): void { + const body = JSON.stringify(value); + response.writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'Content-Length': Buffer.byteLength(body), + }); + response.end(body); +} + +describe('Node container credential helper', () => { + it('uses actual loopback HTTP framing, fresh requests and exact target identities', async () => { + const requests: { target: string; headers: object; url?: string }[] = []; + const { broker } = await fixture((request, response) => { + let body = ''; + request.setEncoding('utf8'); + request.on('data', chunk => { + body += chunk; + }); + request.on('end', () => { + requests.push({ + target: body, + headers: request.headers, + url: request.url, + }); + success(response); + }); + }); + const discovered = container(broker); + expect(discovered.databaseIdentity.toHexString()).toBe(OWN); + const mutableCopy = discovered.databaseIdentity; + mutableCopy.__identity__ = 0n; + const first = await discovered.tokenFor(); + const second = await discovered.tokenFor(new Identity(OTHER)); + expect(requests.map(r => r.target)).toEqual([ + JSON.stringify({ target_database: OWN }), + JSON.stringify({ target_database: OTHER }), + ]); + for (const request of requests) { + expect(request.url).toBe('/v1/credentials'); + expect(request.headers).toMatchObject({ + 'content-type': 'application/json', + connection: 'close', + }); + expect(request.headers).not.toHaveProperty('authorization'); + } + expect(first.target.toHexString()).toBe(OWN); + expect(second.target.toHexString()).toBe(OTHER); + expect(first.value).toBe(SECRET); + expect(first.remainingLifetimeMs).toBeGreaterThan(0); + expect(first.remainingLifetimeMs).toBeLessThanOrEqual(10_000); + expect(inspect(first)).not.toContain(SECRET); + expect(JSON.stringify(first)).not.toContain(SECRET); + }); + + it('uses only the owned Unix socket and required logical HTTP authority', async () => { + const hosts: (string | undefined)[] = []; + const { broker } = await fixture((request, response) => { + hosts.push(request.headers.host); + request.resume(); + request.on('end', () => success(response)); + }, true); + const token = await container(broker).tokenFor(); + expect(token.value).toBe(SECRET); + expect(hosts).toEqual(['127.0.0.1:18081']); + }); + + it('rejects invalid discovery without connecting or choosing defaults', () => { + const invalid = [ + 'http://localhost:18081/v1/credentials', + 'http://127.1:18081/v1/credentials', + 'http://0x7f000001:18081/v1/credentials', + 'http://192.0.2.1:18081/v1/credentials', + 'http://user:secret@127.0.0.1:18081/v1/credentials', + 'http://127.0.0.1:18081/v1/credentials?', + 'http://127.0.0.1:18081/v1/credentials#', + 'http://127.0.0.1:0/v1/credentials', + 'https://127.0.0.1:18081/v1/credentials', + 'unix://elsewhere/run/socket', + 'unix:///run/../socket', + 'unix:///run/%73ocket', + `unix:///${'a'.repeat(104)}`, + ]; + for (const broker of invalid) { + expect(() => container(broker)).toThrow(ContainerCredentialError); + } + expect( + () => + new Container({ + databaseIdentity: OWN, + serverUri: `https://127.0.0.1/${'é'.repeat(2100)}`, + credentialBroker: 'unix:///run/credentials.sock', + }) + ).toThrow('invalid_discovery'); + expect(() => Container.fromEnvironment({})).toThrow('missing_environment'); + const discovered = Container.fromEnvironment({ + SPACETIMEDB_DATABASE_IDENTITY: OWN, + SPACETIMEDB_SERVER_URI: 'https://127.0.0.1:1/', + SPACETIMEDB_CREDENTIAL_BROKER: 'unix:///run/spacetimedb/credentials.sock', + }); + expect(discovered.databaseIdentity.toHexString()).toBe(OWN); + expect(inspect(discovered)).not.toContain('/run/spacetimedb'); + }); + + it('does not follow redirects or retry denial, and preserves a later valid request', async () => { + let count = 0; + const { broker } = await fixture((request, response) => { + request.resume(); + const status = [302, 403, 503, 200][count++]; + if (status === 200) success(response); + else + response + .writeHead(status, { + Location: 'http://192.0.2.1/secret', + 'Content-Length': 0, + }) + .end(); + }); + const discovered = container(broker); + for (const code of ['invalid_response', 'denied', 'unavailable']) { + await expect(discovered.tokenFor()).rejects.toMatchObject({ code }); + } + expect((await discovered.tokenFor()).value).toBe(SECRET); + expect(count).toBe(4); + }); + + it('rejects missing no-store, oversized bodies, wrong fields and invalid expiry without exposing tokens', async () => { + const values: unknown[] = [ + { + token: SECRET, + expires_unix_seconds: Math.floor(Date.now() / 1000) - 1, + }, + { + token: SECRET, + expires_unix_seconds: Math.floor(Date.now() / 1000) + 60, + }, + { token: SECRET, expires_unix_seconds: '123' }, + { token: SECRET, expires_unix_seconds: 1, extra: true }, + { + token: '\nsecret', + expires_unix_seconds: Math.floor(Date.now() / 1000) + 10, + }, + ]; + let count = 0; + const { broker } = await fixture((request, response) => { + request.resume(); + if (count === 0) { + response + .writeHead(200, { + 'Content-Type': 'application/json', + 'Content-Length': 2, + }) + .end('{}'); + } else if (count === 1) { + response + .writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'Content-Length': 100000, + }) + .end(); + } else success(response, values[count - 2]); + count++; + }); + for (let i = 0; i < values.length + 2; i++) { + const error = await container(broker) + .tokenFor() + .catch(error => error as unknown); + expect(error).toMatchObject({ code: 'invalid_response' }); + expect(inspect(error)).not.toContain(SECRET); + } + }); + + it('aborts a held response and observes peer closure before fixture cleanup', async () => { + const { server, broker } = await fixture(request => { + request.resume(); + }); + const connected = once(server, 'connection'); + const abort = new AbortController(); + const result = container(broker).tokenFor(undefined, abort.signal); + const [peer] = (await connected) as [Socket]; + const closed = once(peer, 'close'); + abort.abort(); + await expect(result).rejects.toMatchObject({ code: 'aborted' }); + await closed; + expect(peer.destroyed).toBe(true); + await expect( + container(broker).tokenFor(undefined, abort.signal) + ).rejects.toMatchObject({ code: 'aborted' }); + }); + + it('bounds a stalled response by five seconds and destroys its actual socket', async () => { + const { server, broker } = await fixture(request => { + request.resume(); + }); + const connected = once(server, 'connection'); + const result = container(broker).tokenFor(); + const [peer] = (await connected) as [Socket]; + const closed = once(peer, 'close'); + await expect(result).rejects.toMatchObject({ code: 'timeout' }); + await closed; + expect(peer.destroyed).toBe(true); + }, 10_000); + + it('rejects an invalid exact target before opening a broker connection', async () => { + let accepted = 0; + const { server, broker } = await fixture(request => request.resume()); + server.on('connection', () => accepted++); + const target = new Identity(OTHER); + target.toHexString = () => 'not-an-identity'; + await expect(container(broker).tokenFor(target)).rejects.toMatchObject({ + code: 'invalid_target', + }); + expect(accepted).toBe(0); + }); + + it('closes a request cancelled before its socket is assigned', async () => { + const { broker } = await fixture(request => request.resume()); + const abort = new AbortController(); + const result = container(broker).tokenFor(undefined, abort.signal); + abort.abort(); + await expect(result).rejects.toMatchObject({ code: 'aborted' }); + }); + + it('rejects duplicate response headers and closes the actual peer', async () => { + const body = JSON.stringify({ + token: SECRET, + expires_unix_seconds: Math.floor(Date.now() / 1000) + 20, + }); + await rejectsAndClosesPeer( + `HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Type: application/json\r\nCache-Control: no-store\r\nContent-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`, + 'invalid_response' + ); + }); + + it('rejects a truncated response and closes the actual peer', async () => { + await rejectsAndClosesPeer( + 'HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nCache-Control: no-store\r\nContent-Length: 100\r\n\r\n{}', + 'transport', + true + ); + }); + + it('rejects an upgrade while retaining ownership through actual socket close', async () => { + await rejectsAndClosesPeer( + 'HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: arbitrary\r\n\r\n', + 'invalid_response' + ); + }); +}); diff --git a/crates/bindings-typescript/tests/node_websocket.test.ts b/crates/bindings-typescript/tests/node_websocket.test.ts new file mode 100644 index 00000000000..d585bd056a6 --- /dev/null +++ b/crates/bindings-typescript/tests/node_websocket.test.ts @@ -0,0 +1,324 @@ +import { createHash } from 'node:crypto'; +import { createServer, type IncomingMessage } from 'node:http'; +import type { Socket } from 'node:net'; +import { afterEach, describe, expect, it } from 'vitest'; +import { openNodeWebSocket, NodeWebSocketAdapter } from '../src/sdk/node'; +import { WebSocket } from 'ws'; +import { once } from 'node:events'; +import type { WebSocketArgs } from '../src/sdk/ws'; +import { DbConnection } from '../test-app/src/module_bindings'; +import { BinaryWriter, ConnectionId, Identity } from '../src'; +import { ServerMessage } from '../src/sdk/client_api/types'; + +const sockets: Set = new Set(); +const adapters: NodeWebSocketAdapter[] = []; +const servers: ReturnType[] = []; +const token = 'fixture.hosted.credential'; + +afterEach(async () => { + const closed = await Promise.allSettled( + adapters.splice(0).map(a => a.shutdown()) + ); + for (const socket of sockets) socket.destroy(); + await Promise.all( + servers.splice(0).map( + server => + new Promise((resolve, reject) => { + server.close(error => (error ? reject(error) : resolve())); + }) + ) + ); + sockets.clear(); + for (const result of closed) expect(result.status).toBe('fulfilled'); +}); + +async function endpoint( + upgrade: (req: IncomingMessage, socket: Socket) => void +) { + const server = createServer((_req, res) => { + // A token-exchange request must never be sent by this transport. + res.writeHead(500).end(); + }); + server.on('connection', socket => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + server.on('upgrade', (req, socket) => upgrade(req, socket as Socket)); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + servers.push(server); + const address = server.address(); + if (!address || typeof address === 'string') + throw new Error('Expected owned TCP listener'); + return new URL(`ws://127.0.0.1:${address.port}/`); +} + +function args(url: URL): WebSocketArgs { + return { + url, + nameOrAddress: 'test/database', + authToken: token, + wsProtocol: ['fixture-protocol'], + compression: 'none', + lightMode: true, + confirmedReads: false, + }; +} + +function accept( + req: IncomingMessage, + socket: Socket, + payload = Buffer.from([0, 111, 107]), + answerClose = true +) { + const accept = createHash('sha1') + .update( + req.headers['sec-websocket-key'] + '258EAFA5-E914-47DA-95CA-C5AB0DC85B11' + ) + .digest('base64'); + socket.write( + 'HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n' + + `Sec-WebSocket-Accept: ${accept}\r\nSec-WebSocket-Protocol: ${req.headers['sec-websocket-protocol']?.split(',')[0].trim()}\r\n\r\n` + ); + // A single uncompressed SpacetimeDB payload inside a binary WS frame. + if (payload.length >= 126) + throw new Error('Fixture message exceeded short-frame bound'); + socket.write(Buffer.concat([Buffer.from([0x82, payload.length]), payload])); + socket.on('data', () => { + if (answerClose) socket.end(Buffer.from([0x88, 0])); + }); +} + +describe('explicit Node.js header transport', () => { + it('connects through the generated builder with its actual connection ID', async () => { + let request: IncomingMessage | undefined; + const identity = new Identity(123n); + const url = await endpoint((req, socket) => { + request = req; + const connectionId = new URL( + req.url!, + 'http://127.0.0.1' + ).searchParams.get('connection_id'); + if (!connectionId) throw new Error('Builder connection ID missing'); + const writer = new BinaryWriter(1024); + ServerMessage.serialize( + writer, + ServerMessage.InitialConnection({ + identity, + token, + connectionId: new ConnectionId(BigInt(`0x${connectionId}`)), + }) + ); + accept( + req, + socket, + Buffer.concat([Buffer.from([0]), writer.getBuffer()]) + ); + }); + let resolve!: () => void; + let reject!: (error: Error) => void; + const ready = new Promise((yes, no) => { + resolve = yes; + reject = no; + }); + const connection = DbConnection.builder() + .withUri(url) + .withDatabaseName('fixture') + .withToken(token) + .withWSFn(async input => { + const adapter = await openNodeWebSocket(input); + adapters.push(adapter); + return adapter; + }) + .onConnect((_connection, actualIdentity) => { + expect(actualIdentity.toHexString()).toBe(identity.toHexString()); + resolve(); + }) + .onConnectError((_context, error) => reject(error)) + .build(); + try { + await ready; + expect(request?.headers.authorization).toBe(`Bearer ${token}`); + expect( + new URL(request!.url!, url).searchParams.get('connection_id') + ).toMatch(/^[0-9a-f]{32}$/); + } finally { + connection.disconnect(); + } + }); + it('uses Authorization on the exact route and preserves subscription options', async () => { + let request: IncomingMessage | undefined; + const url = await endpoint((req, socket) => { + request = req; + accept(req, socket); + }); + // The exported Node class's static factory must also use headers directly. + const adapter = await NodeWebSocketAdapter.openWebSocket(args(url)); + adapters.push(adapter); + const received = await new Promise((resolve, reject) => { + adapter.onmessage = event => resolve(event.data); + adapter.onerror = () => reject(new Error('Fixture connection failed')); + }); + expect(new TextDecoder().decode(received)).toBe('ok'); + expect(request?.headers.authorization).toBe(`Bearer ${token}`); + expect(request?.url).toBe( + '/v1/database/test%2Fdatabase/subscribe?compression=None&light=true&confirmed=false' + ); + expect(request?.url).not.toContain(token); + expect(adapter.protocol).toBe('fixture-protocol'); + await Promise.all([adapter.shutdown(), adapter.shutdown()]); + expect(adapter.readyState).toBe(3); + }); + + it('does not follow an upgrade redirect with credentials', async () => { + let forwarded = false; + const destination = await endpoint((req, socket) => { + forwarded = true; + accept(req, socket); + }); + const url = await endpoint((_req, socket) => { + socket.end( + `HTTP/1.1 302 Found\r\nLocation: ${destination}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n` + ); + }); + const adapter = await openNodeWebSocket(args(url)); + adapters.push(adapter); + const error = await new Promise(resolve => { + adapter.onerror = resolve; + }); + expect(error.message).toBe('Database WebSocket transport failed'); + expect(error.error).toBeUndefined(); + await adapter.shutdown(); + expect(forwarded).toBe(false); + }); + + it('rejects ambiguous URLs and malformed credentials before connecting', async () => { + let connected = false; + const url = await endpoint((req, socket) => { + connected = true; + accept(req, socket); + }); + const invalid = [ + { ...args(url), authToken: 'secret\r\ninjected: header' }, + { ...args(url), authToken: 'x'.repeat(8193) }, + { ...args(url), authToken: '' }, + { ...args(url), nameOrAddress: '.' }, + { ...args(url), nameOrAddress: '..' }, + { ...args(url), nameOrAddress: 'database\n' }, + { ...args(url), nameOrAddress: 'database\u007f' }, + { ...args(url), url: new URL('?connection_id=invalid', url) }, + { + ...args(url), + url: new URL( + '?connection_id=00000000000000000000000000000001&connection_id=00000000000000000000000000000002', + url + ), + }, + { ...args(url), url: new URL('?token=secret', url) }, + { ...args(url), url: new URL('#secret', url) }, + { ...args(url), url: new URL(`ws://secret:secret@${url.host}/`) }, + { ...args(url), url: new URL('file:///not-a-server') }, + ]; + for (const input of invalid) { + await expect(openNodeWebSocket(input)).rejects.toThrow( + 'Invalid Node.js database WebSocket configuration' + ); + } + expect(connected).toBe(false); + }); + + it('immediately completes shutdown when wrapping an already closed socket', async () => { + const url = await endpoint((req, socket) => accept(req, socket)); + const raw = new WebSocket(url, ['fixture-protocol']); + raw.on('error', () => {}); + try { + await once(raw, 'open'); + const closed = once(raw, 'close'); + raw.close(); + await closed; + const adapter = new NodeWebSocketAdapter(raw); + adapters.push(adapter); + await adapter.shutdown(); + expect(adapter.readyState).toBe(3); + } finally { + raw.terminate(); + } + }); + + it('cleans up a connection cancelled during its upgrade', async () => { + let admitted!: () => void; + const accepted = new Promise(resolve => { + admitted = resolve; + }); + const url = await endpoint(() => admitted()); + const adapter = await openNodeWebSocket(args(url)); + adapters.push(adapter); + await accepted; + let closeEvents = 0; + adapter.onclose = () => { + closeEvents++; + }; + await adapter.shutdown(); + expect(adapter.readyState).toBe(3); + expect(closeEvents).toBe(1); + }); + + it('terminates the actual upgraded socket when the peer ignores close', async () => { + let peerClosed!: () => void; + const closed = new Promise(resolve => { + peerClosed = resolve; + }); + const peerEvents: string[] = []; + const url = await endpoint((req, socket) => { + // Node's upgraded HTTP socket keeps its writable half open after EOF. + // Do not confuse that with a live client transport. Observe the client's + // actual FIN first, then finish the fixture's own writable half. + socket.once('end', () => { + peerEvents.push('end'); + socket.end(); + }); + socket.once('close', () => { + peerEvents.push('close'); + peerClosed(); + }); + accept(req, socket, Buffer.from([0, 111, 107]), false); + }); + const adapter = await openNodeWebSocket(args(url)); + adapters.push(adapter); + await new Promise(resolve => { + adapter.onmessage = () => resolve(); + }); + let closeEvents = 0; + adapter.onclose = () => { + closeEvents++; + }; + await adapter.shutdown(); + await closed; + expect(peerEvents).toEqual(['end', 'close']); + expect(adapter.readyState).toBe(3); + expect(closeEvents).toBe(1); + }, 10_000); + + it('bounds a stalled upgrade without waiting for a caller to close it', async () => { + const url = await endpoint(() => {}); + const adapter = await openNodeWebSocket(args(url)); + adapters.push(adapter); + await new Promise(resolve => { + adapter.onerror = () => resolve(); + }); + await adapter.shutdown(); + expect(adapter.readyState).toBe(3); + }, 10_000); + + it('redacts a constructor failure before opening a socket', async () => { + const url = await endpoint(() => {}); + await expect( + openNodeWebSocket({ + ...args(url), + wsProtocol: ['private invalid protocol'], + }) + ).rejects.toThrow('Database WebSocket transport could not start'); + }); +}); diff --git a/crates/bindings-typescript/tsup.config.ts b/crates/bindings-typescript/tsup.config.ts index 80c4af53d31..30f2953004e 100644 --- a/crates/bindings-typescript/tsup.config.ts +++ b/crates/bindings-typescript/tsup.config.ts @@ -220,6 +220,20 @@ export default defineConfig([ esbuildOptions: commonEsbuildTweaks(), }, + // Explicit Node.js transport, excluded from browser entry points. + { + entry: { index: 'src/sdk/node/index.ts' }, + format: ['esm', 'cjs'], + target: 'es2022', + outDir: 'dist/sdk/node', + dts: false, + sourcemap: true, + clean: true, + platform: 'node', + external: ['ws'], + outExtension, + }, + // SDK browser ESM: dist/sdk/index.browser.mjs { entry: { 'index.browser': 'src/sdk/index.ts' }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5ed3e919ac8..c5493c5c405 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -62,6 +62,9 @@ importers: crates/bindings-typescript: dependencies: + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 base64-js: specifier: ^1.5.1 version: 1.5.1 @@ -98,6 +101,9 @@ importers: vue: specifier: ^3.3.0 version: 3.5.26(typescript@5.9.3) + ws: + specifier: ^8.18.3 + version: 8.18.3 devDependencies: '@angular/compiler': specifier: ^21.2.12 diff --git a/sdks/rust/Cargo.toml b/sdks/rust/Cargo.toml index 7f1f24f6ff7..24c52975e41 100644 --- a/sdks/rust/Cargo.toml +++ b/sdks/rust/Cargo.toml @@ -57,6 +57,9 @@ web-sys = { version = "0.3.77", features = ["HtmlDocument"], optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] home.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true tokio.workspace = true tokio-tungstenite.workspace = true # native-tls 0.2.17 fails to compile with Rust 1.93.0 due non-exhaustive diff --git a/sdks/rust/src/credentials.rs b/sdks/rust/src/credentials.rs index 1b4c5116af1..33828c975de 100644 --- a/sdks/rust/src/credentials.rs +++ b/sdks/rust/src/credentials.rs @@ -1,4 +1,8 @@ -//! Utilities for saving and re-using credentials. +//! Credentials for ordinary clients and hosted container processes. +//! +//! Hosted containers should use `Container` and +//! `DbConnectionBuilder::with_container_credentials` instead of saving +//! their short-lived tokens to a file. Those APIs are available in native builds. //! //! Users are encouraged to import this module by name and refer to its contents by qualified path, like: //! ```ignore @@ -8,6 +12,13 @@ //! } //! ``` +#[cfg(not(feature = "browser"))] +mod container; +#[cfg(all(test, not(feature = "browser")))] +pub(crate) use container::tests as container_tests; +#[cfg(not(feature = "browser"))] +pub use container::{Container, ContainerCredentialError, ContainerToken}; + #[cfg(not(feature = "browser"))] mod native_mod { use home::home_dir; diff --git a/sdks/rust/src/credentials/container.rs b/sdks/rust/src/credentials/container.rs new file mode 100644 index 00000000000..84784fba92b --- /dev/null +++ b/sdks/rust/src/credentials/container.rs @@ -0,0 +1,474 @@ +//! Explicit platform discovery and bounded, target-specific credential requests. + +use crate::Identity; +use http::Uri; +use reqwest::{header, redirect::Policy, Client, Response, Url}; +use serde::Deserialize; +use std::{ + fmt, + net::IpAddr, + path::{Component, PathBuf}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::time::Instant; + +const DATABASE_IDENTITY: &str = "SPACETIMEDB_DATABASE_IDENTITY"; +const SERVER_URI: &str = "SPACETIMEDB_SERVER_URI"; +const CREDENTIAL_BROKER: &str = "SPACETIMEDB_CREDENTIAL_BROKER"; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_LIFETIME: Duration = Duration::from_secs(30); +const MAX_TOKEN: usize = 8192; +const MAX_BODY: usize = MAX_TOKEN + 256; +const MAX_HEADERS: usize = 4096; +const MAX_URI: usize = 4096; +// The Unix transport uses the same HTTP authority and request path as the +// guest proxy. This URL is never resolved or connected over TCP in Unix mode. +const LOCAL_HTTP_URI: &str = "http://127.0.0.1:18081/v1/credentials"; + +/// A credential error whose diagnostics never include discovery values, +/// response bodies, request headers, or tokens. +#[derive(Clone, Copy, Debug, thiserror::Error, PartialEq, Eq)] +#[non_exhaustive] +pub enum ContainerCredentialError { + #[error("Missing or non-Unicode container discovery variable {0}")] + MissingEnvironment(&'static str), + #[error("Invalid container discovery configuration")] + InvalidDiscovery, + #[error("Unsupported container credential transport")] + UnsupportedTransport, + #[error("Container credentials cannot be combined with a static token")] + ConflictingCredentials, + #[error("Container credentials cannot be used with debug files that record tokens")] + DebugLogging, + #[error("Invalid container credential target")] + InvalidTarget, + #[error("Unable to resolve the database Identity on the selected server")] + NameResolution, + #[error("Container credential request was denied")] + Denied, + #[error("Container credential broker is unavailable")] + Unavailable, + #[error("Container credential request failed")] + Transport, + #[error("Container credential request exceeded its deadline")] + Timeout, + #[error("Invalid container credential response")] + InvalidResponse, + #[error("Container credential expired before the connection completed")] + Expired, +} + +type Result = std::result::Result; + +#[derive(Clone)] +enum Endpoint { + Http(Url), + Unix(PathBuf), +} + +/// Discovery configuration for a process running in a SpacetimeDB container. +/// +/// This value contains no credentials. Cloning it never copies a token. The +/// broker derives the sender from its authenticated runtime association, not +/// from the Identity in this configuration or from request fields. +/// +/// ```ignore +/// use spacetimedb_sdk::credentials; +/// let container = credentials::Container::from_env()?; +/// let connection = DbConnection::builder() +/// .with_container_credentials(container) +/// .build_async().await?; +/// ``` +/// +/// The builder defaults to this container's database and server. Use its +/// `with_uri` and `with_database_name` methods to select another database. +/// Explicit `self` resolves locally to [`Self::database_identity`]. +/// +/// Each builder connection requests a fresh token. To renew credentials and +/// reconnect with fresh subscriptions, use [`crate::ContainerSession`]. An +/// ordinary builder connection ends when its credential expires or is revoked. +/// Do not save the token returned by `on_connect` for later use. +#[derive(Clone)] +pub struct Container { + identity: Identity, + server_uri: Uri, + endpoint: Endpoint, +} + +impl fmt::Debug for Container { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Container").finish_non_exhaustive() + } +} + +impl Container { + /// Read only the three platform-injected discovery variables. Missing or + /// invalid configuration is an error; this never reads stored CLI/SDK + /// credentials, allocates an anonymous Identity, or chooses a default server. + /// No network or socket operation occurs until a token is requested. + pub fn from_env() -> Result { + let read = |key| std::env::var(key).map_err(|_| ContainerCredentialError::MissingEnvironment(key)); + Self::new(&read(DATABASE_IDENTITY)?, &read(SERVER_URI)?, &read(CREDENTIAL_BROKER)?) + } + + /// Use explicit discovery values instead of reading the environment. + /// + /// The broker must be an absolute `unix:///path` socket URI or an HTTP URI + /// with a numeric loopback address and the path `/v1/credentials`. Unix + /// sockets are supported on Unix targets. Broker redirects, proxies, DNS + /// lookup, embedded credentials, and fallback transports are forbidden. + pub fn new(database_identity: &str, server_uri: &str, broker_uri: &str) -> Result { + let identity = parse_identity(database_identity).map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + let server_uri = server_url(server_uri)? + .as_str() + .parse() + .map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + Ok(Self { + identity, + server_uri, + endpoint: endpoint(broker_uri)?, + }) + } + + pub fn database_identity(&self) -> Identity { + self.identity + } + + pub fn server_uri(&self) -> &Uri { + &self.server_uri + } + + /// Obtain a fresh credential for HTTP or another client of `target`. + /// + /// Send it only to that database's trusted server. It is a bearer credential + /// with a lifetime of at most 30 seconds, possibly less. Keep it in memory; + /// never put it in environment variables, logs, images, or durable files. + /// This method neither retries a denial nor falls back to an owner token. + pub async fn token_for(&self, target: Identity) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, self.request_token(target)) + .await + .map_err(|_| ContainerCredentialError::Timeout)? + } + + async fn request_token(&self, target: Identity) -> Result { + let builder = client_builder(); + let (builder, uri) = match &self.endpoint { + Endpoint::Http(uri) => (builder, uri.clone()), + Endpoint::Unix(path) => { + #[cfg(unix)] + { + (builder.unix_socket(path.clone()), Url::parse(LOCAL_HTTP_URI).unwrap()) + } + #[cfg(not(unix))] + { + let _ = path; + return Err(ContainerCredentialError::UnsupportedTransport); + } + } + }; + let client = builder.build().map_err(|_| ContainerCredentialError::Transport)?; + // Identity::to_hex is fixed lowercase ASCII, so the body cannot contain + // request framing, another sender, or arbitrary JSON fields. + let body = format!("{{\"target_database\":\"{}\"}}", target.to_hex()); + let response = client + .post(uri) + .header(header::CONTENT_TYPE, "application/json") + .header(header::CONNECTION, "close") + .body(body) + .send() + .await + .map_err(transport_error)?; + match response.status().as_u16() { + 200 => {} + 401 | 403 => return Err(ContainerCredentialError::Denied), + 503 => return Err(ContainerCredentialError::Unavailable), + _ => return Err(ContainerCredentialError::InvalidResponse), + } + let headers = response.headers(); + if header_size(headers) > MAX_HEADERS + || headers.get_all(header::CONTENT_TYPE).iter().count() != 1 + || headers.get(header::CONTENT_TYPE).map(|value| value.as_bytes()) != Some(b"application/json") + || headers.get_all(header::CACHE_CONTROL).iter().count() != 1 + || headers.get(header::CACHE_CONTROL).map(|value| value.as_bytes()) != Some(b"no-store") + || headers.contains_key(header::TRANSFER_ENCODING) + || headers.contains_key(header::CONTENT_ENCODING) + || headers.get_all(header::CONTENT_LENGTH).iter().count() != 1 + { + return Err(ContainerCredentialError::InvalidResponse); + } + let length = headers[header::CONTENT_LENGTH] + .to_str() + .ok() + .filter(|value| !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())) + .and_then(|value| value.parse::().ok()) + .filter(|length| *length > 0 && *length <= MAX_BODY) + .ok_or(ContainerCredentialError::InvalidResponse)?; + let bytes = bounded_body(response, length).await?; + if bytes.len() != length { + return Err(ContainerCredentialError::InvalidResponse); + } + let response: TokenResponse = + serde_json::from_slice(&bytes).map_err(|_| ContainerCredentialError::InvalidResponse)?; + let expiry = UNIX_EPOCH + .checked_add(Duration::from_secs(response.expires_unix_seconds)) + .ok_or(ContainerCredentialError::InvalidResponse)?; + let remaining = expiry + .duration_since(SystemTime::now()) + .ok() + .filter(|remaining| !remaining.is_zero() && *remaining <= MAX_LIFETIME) + .ok_or(ContainerCredentialError::InvalidResponse)?; + if response.token.is_empty() + || response.token.len() > MAX_TOKEN + || !response.token.bytes().all(|byte| byte.is_ascii_graphic()) + { + return Err(ContainerCredentialError::InvalidResponse); + } + // Retain a monotonic cap even if the system clock later moves backward. + let deadline = Instant::now() + remaining; + Ok(ContainerToken { + token: response.token, + target, + expiry, + deadline, + }) + } + + pub(crate) async fn prepare_connection(&self, uri: Option<&Uri>, target: Option<&str>) -> Result { + let (uri, target) = self.resolve_connection_target(uri, target).await?; + let credential = self.token_for(target).await?; + Ok(Prepared { + uri, + target: target.to_hex().to_string(), + credential, + }) + } + + pub(crate) async fn resolve_connection_target( + &self, + uri: Option<&Uri>, + target: Option<&str>, + ) -> Result<(Uri, Identity)> { + let uri = uri.unwrap_or(&self.server_uri); + let mut server = server_url(&uri.to_string())?; + let target = match target { + None | Some("self") => self.identity, + Some(target) => match parse_identity(target) { + Ok(identity) => identity, + Err(_) => { + // Preserve the SDK's server path prefix. Append the name as + // one encoded segment, never as request path syntax. + if target.is_empty() || target.len() > 256 || target.chars().any(char::is_control) { + return Err(ContainerCredentialError::InvalidTarget); + } + server + .path_segments_mut() + .map_err(|_| ContainerCredentialError::InvalidDiscovery)? + .pop_if_empty() + .extend(["v1", "database", target, "identity"]); + resolve_name(server).await? + } + }, + }; + Ok((uri.clone(), target)) + } +} + +pub(crate) struct Prepared { + pub uri: Uri, + pub target: String, + pub credential: ContainerToken, +} + +/// An in-memory, target-specific bearer credential. Debug output is redacted. +/// Expiry is enforced by the receiving host even if its bytes are copied. +pub struct ContainerToken { + token: String, + target: Identity, + expiry: SystemTime, + deadline: Instant, +} + +impl ContainerToken { + /// The token for an `Authorization: Bearer ...` header. Do not log or save it. + pub fn as_str(&self) -> &str { + &self.token + } + + pub fn target(&self) -> Identity { + self.target + } + + pub fn expires_at(&self) -> SystemTime { + self.expiry + } + + pub fn remaining_lifetime(&self) -> Duration { + self.deadline.saturating_duration_since(Instant::now()) + } + + pub(crate) fn deadline(&self) -> Instant { + self.deadline + } +} + +impl fmt::Debug for ContainerToken { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ContainerToken") + .field("target", &self.target) + .field("expires_at", &self.expiry) + .field("token", &"") + .finish() + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TokenResponse { + token: String, + expires_unix_seconds: u64, +} + +fn client_builder() -> reqwest::ClientBuilder { + Client::builder() + .no_proxy() + .redirect(Policy::none()) + .referer(false) + .http1_only() + .pool_max_idle_per_host(0) + .connect_timeout(REQUEST_TIMEOUT) + .timeout(REQUEST_TIMEOUT) + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() +} + +fn transport_error(error: reqwest::Error) -> ContainerCredentialError { + if error.is_timeout() { + ContainerCredentialError::Timeout + } else { + ContainerCredentialError::Transport + } +} + +async fn bounded_body(mut response: Response, limit: usize) -> Result> { + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(transport_error)? { + if chunk.len() > limit.saturating_sub(bytes.len()) { + return Err(ContainerCredentialError::InvalidResponse); + } + bytes.extend_from_slice(&chunk); + } + Ok(bytes) +} + +fn header_size(headers: &header::HeaderMap) -> usize { + headers + .iter() + .map(|(key, value)| key.as_str().len() + value.as_bytes().len() + 4) + .sum() +} + +async fn resolve_name(uri: Url) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, async { + let client = client_builder() + .build() + .map_err(|_| ContainerCredentialError::NameResolution)?; + let response = client + .get(uri) + .send() + .await + .map_err(|_| ContainerCredentialError::NameResolution)?; + if response.status() != reqwest::StatusCode::OK || header_size(response.headers()) > MAX_HEADERS { + return Err(ContainerCredentialError::NameResolution); + } + let body = bounded_body(response, 64) + .await + .map_err(|_| ContainerCredentialError::NameResolution)?; + parse_identity(std::str::from_utf8(&body).map_err(|_| ContainerCredentialError::NameResolution)?) + .map_err(|_| ContainerCredentialError::NameResolution) + }) + .await + .map_err(|_| ContainerCredentialError::Timeout)? +} + +fn parse_identity(value: &str) -> Result { + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(ContainerCredentialError::InvalidTarget); + } + Identity::from_hex(value).map_err(|_| ContainerCredentialError::InvalidTarget) +} + +fn parse_url(value: &str) -> Result { + if value.is_empty() || value.len() > MAX_URI || value.bytes().any(|byte| byte.is_ascii_control() || byte == b' ') { + return Err(ContainerCredentialError::InvalidDiscovery); + } + let uri = Url::parse(value).map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + if !uri.username().is_empty() || uri.password().is_some() || uri.query().is_some() || uri.fragment().is_some() { + return Err(ContainerCredentialError::InvalidDiscovery); + } + Ok(uri) +} + +fn server_url(value: &str) -> Result { + let mut uri = parse_url(value)?; + match uri.scheme() { + "http" | "https" => {} + "ws" => uri + .set_scheme("http") + .map_err(|_| ContainerCredentialError::InvalidDiscovery)?, + "wss" => uri + .set_scheme("https") + .map_err(|_| ContainerCredentialError::InvalidDiscovery)?, + _ => return Err(ContainerCredentialError::InvalidDiscovery), + } + if uri.host_str().is_none() { + return Err(ContainerCredentialError::InvalidDiscovery); + } + Ok(uri) +} + +fn endpoint(value: &str) -> Result { + let uri = parse_url(value)?; + match uri.scheme() { + "http" => { + // Parse the original authority as an IP, rejecting DNS aliases and + // URL parser shortcuts such as hexadecimal/abbreviated IPv4. + let original: Uri = value.parse().map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + let host = original.host().ok_or(ContainerCredentialError::InvalidDiscovery)?; + let ip: IpAddr = host + .trim_matches(['[', ']']) + .parse() + .map_err(|_| ContainerCredentialError::InvalidDiscovery)?; + if !ip.is_loopback() || uri.path() != "/v1/credentials" { + return Err(ContainerCredentialError::InvalidDiscovery); + } + Ok(Endpoint::Http(uri)) + } + "unix" => { + let path = value + .strip_prefix("unix://") + .ok_or(ContainerCredentialError::InvalidDiscovery)?; + if uri.host_str().is_some() + || uri.port().is_some() + || !path.starts_with('/') + || path.len() > 103 + || path.contains('%') + || path.split('/').any(|part| part == "." || part == "..") + || PathBuf::from(path) + .components() + .any(|part| !matches!(part, Component::RootDir | Component::Normal(_))) + { + return Err(ContainerCredentialError::InvalidDiscovery); + } + #[cfg(unix)] + return Ok(Endpoint::Unix(PathBuf::from(path))); + #[cfg(not(unix))] + return Err(ContainerCredentialError::UnsupportedTransport); + } + _ => Err(ContainerCredentialError::UnsupportedTransport), + } +} + +#[cfg(test)] +pub(crate) mod tests; diff --git a/sdks/rust/src/credentials/container/tests.rs b/sdks/rust/src/credentials/container/tests.rs new file mode 100644 index 00000000000..73a41f47338 --- /dev/null +++ b/sdks/rust/src/credentials/container/tests.rs @@ -0,0 +1,297 @@ +use super::*; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::oneshot, + task::JoinHandle, +}; + +pub(crate) const SOURCE: &str = "0000000000000000000000000000000000000000000000000000000000000001"; +const TARGET: &str = "0000000000000000000000000000000000000000000000000000000000000002"; + +pub(crate) fn token_body(token: &str, offset: u64) -> String { + let expiry = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs() + offset; + serde_json::json!({"token": token, "expires_unix_seconds": expiry}).to_string() +} + +pub(crate) fn response(status: u16, body: &str) -> Vec { + format!( + "HTTP/1.1 {status} Fixture\r\nContent-Type: application/json\r\nCache-Control: no-store\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ).into_bytes() +} + +pub(crate) async fn read_request(socket: &mut (impl AsyncRead + Unpin)) -> String { + let mut bytes = Vec::new(); + while !bytes.ends_with(b"\r\n\r\n") { + assert!(bytes.len() < MAX_HEADERS); + bytes.push(socket.read_u8().await.unwrap()); + } + let header = std::str::from_utf8(&bytes).unwrap(); + let length = header + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + key.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().unwrap()) + }) + .unwrap_or(0); + assert!(length <= 4096); + let start = bytes.len(); + bytes.resize(start + length, 0); + socket.read_exact(&mut bytes[start..]).await.unwrap(); + String::from_utf8(bytes).unwrap() +} + +// Every endpoint is created by this test, explicitly uses numeric loopback, +// and has a retained task that callers join. No saved server/client config. +pub(crate) async fn http_fixture(responses: Vec>) -> (String, JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let uri = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + tokio::time::timeout(Duration::from_secs(3), async { + let mut requests = Vec::new(); + for response in responses { + let (mut socket, peer) = listener.accept().await.unwrap(); + assert!(peer.ip().is_loopback()); + requests.push(read_request(&mut socket).await); + socket.write_all(&response).await.unwrap(); + socket.shutdown().await.unwrap(); + } + requests + }) + .await + .expect("owned credential fixture exceeded deadline") + }); + (uri, task) +} + +#[test] +fn discovery_rejects_nonlocal_ambiguous_and_credential_bearing_endpoints() { + for endpoint in [ + "http://example.invalid/v1/credentials", + "http://192.0.2.1/v1/credentials", + "http://localhost/v1/credentials", + "http://127.1/v1/credentials", + "http://0x7f000001/v1/credentials", + "http://127.0.0.1/wrong", + "http://127.0.0.1/v1/credentials?token=never-log-this", + "http://owner:never-log-this@127.0.0.1/v1/credentials", + "https://127.0.0.1/v1/credentials", + "unix://authority/socket", + "unix:relative.sock", + "unix:///run/../other.sock", + "unix:///run/%2e%2e/other.sock", + "unix:///run/credentials.sock#never-log-this", + "spacetimedb-unavailable:///credentials", + ] { + let error = Container::new(SOURCE, "https://server.invalid", endpoint).unwrap_err(); + assert!(!format!("{error:?} {error}").contains("never-log-this")); + } + assert!(Container::new(SOURCE, "https://owner:secret@server.invalid", LOCAL_HTTP_URI).is_err()); + assert!(Container::new("self", "https://server.invalid", LOCAL_HTTP_URI).is_err()); + let config = Container::new(SOURCE, "wss://server.invalid/prefix", LOCAL_HTTP_URI).unwrap(); + assert_eq!(config.database_identity(), parse_identity(SOURCE).unwrap()); + assert_eq!(config.server_uri().to_string(), "https://server.invalid/prefix"); +} + +#[tokio::test] +async fn each_request_fetches_a_fresh_target_specific_token_without_sender_fields() { + let (endpoint, server) = http_fixture(vec![ + response(200, &token_body("first-secret", 20)), + response(200, &token_body("second-secret", 20)), + ]) + .await; + let config = Container::new(SOURCE, "https://server.invalid", &endpoint).unwrap(); + let target = parse_identity(TARGET).unwrap(); + let first = config.token_for(target).await.unwrap(); + let second = config.token_for(target).await.unwrap(); + assert_eq!(first.as_str(), "first-secret"); + assert_eq!(second.as_str(), "second-secret"); + assert_eq!(first.target(), target); + assert!(first.remaining_lifetime() <= MAX_LIFETIME); + assert!(!format!("{first:?}").contains("first-secret")); + for request in server.await.unwrap() { + assert!(request.starts_with("POST /v1/credentials HTTP/1.1\r\n")); + assert_eq!( + request.split_once("\r\n\r\n").unwrap().1, + format!("{{\"target_database\":\"{TARGET}\"}}") + ); + assert!(!request.to_ascii_lowercase().contains("authorization:")); + } +} + +#[tokio::test] +async fn response_validation_rejects_expiry_fields_tokens_and_framing_without_secret_diagnostics() { + let mut bad = vec![ + response(200, &token_body("", 20)), + response(200, &token_body("never-log-this\n", 20)), + response(200, &token_body(&"x".repeat(MAX_TOKEN + 1), 20)), + response(200, &token_body("never-log-this", 0)), + response(200, &token_body("never-log-this", 60)), + response(200, "{\"token\":\"never-log-this\",\"expires_unix_seconds\":1,\"sender\":\"forged\"}"), + response(200, "{\"token\":\"never-log-this\",\"token\":\"duplicate\",\"expires_unix_seconds\":1}"), + response(200, "{\"token\":null,\"expires_unix_seconds\":null}"), + format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", MAX_BODY + 1).into_bytes(), + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: application/json\r\nCache-Control: no-store\r\n\r\n0\r\n\r\n".to_vec(), + ]; + let valid = String::from_utf8(response(200, &token_body("never-log-this", 20))).unwrap(); + bad.push( + valid + .replacen("\r\n", &format!("\r\nX-Padding: {}\r\n", "x".repeat(MAX_HEADERS)), 1) + .into_bytes(), + ); + bad.push(valid.replace("Cache-Control: no-store\r\n", "").into_bytes()); + bad.push( + valid + .replace("Content-Type: application/json", "Content-Type: text/plain") + .into_bytes(), + ); + for bytes in bad { + let (endpoint, server) = http_fixture(vec![bytes]).await; + let config = Container::new(SOURCE, "https://server.invalid", &endpoint).unwrap(); + let error = config.token_for(parse_identity(TARGET).unwrap()).await.unwrap_err(); + assert!(matches!( + error, + ContainerCredentialError::InvalidResponse | ContainerCredentialError::Transport + )); + assert!(!format!("{error:?} {error}").contains("never-log-this")); + server.await.unwrap(); + } +} + +#[tokio::test] +async fn broker_denial_unavailability_and_redirects_never_fall_back() { + for (status, expected) in [ + (401, ContainerCredentialError::Denied), + (403, ContainerCredentialError::Denied), + (503, ContainerCredentialError::Unavailable), + (302, ContainerCredentialError::InvalidResponse), + ] { + let bytes = String::from_utf8(response(status, "never-log-this")) + .unwrap() + .replacen("\r\n", "\r\nLocation: https://must-not-contact.invalid/\r\n", 1) + .into_bytes(); + let (endpoint, server) = http_fixture(vec![bytes]).await; + let config = Container::new(SOURCE, "https://server.invalid", &endpoint).unwrap(); + assert_eq!( + config.token_for(parse_identity(TARGET).unwrap()).await.unwrap_err(), + expected + ); + assert_eq!(server.await.unwrap().len(), 1); + } +} + +#[tokio::test] +async fn cancelling_a_pending_request_closes_its_socket() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let config = Container::new(SOURCE, "https://server.invalid", &endpoint).unwrap(); + let (ready_tx, ready_rx) = oneshot::channel(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + read_request(&mut socket).await; + ready_tx.send(()).unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), socket.read_u8()) + .await + .unwrap() + .unwrap_err() + .kind(), + std::io::ErrorKind::UnexpectedEof + ); + }); + let mut request = Box::pin(config.token_for(parse_identity(TARGET).unwrap())); + tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + _ = &mut request => panic!("stalled broker unexpectedly answered"), + ready = ready_rx => ready.unwrap(), + } + }) + .await + .unwrap(); + drop(request); + server.await.unwrap(); +} + +#[tokio::test] +async fn self_alias_stays_local_and_name_resolution_pins_the_returned_identity() { + let (broker, broker_task) = http_fixture(vec![ + response(200, &token_body("self-token", 20)), + response(200, &token_body("other-token", 20)), + ]) + .await; + let config = Container::new(SOURCE, "https://must-not-contact.invalid", &broker).unwrap(); + let own = config.prepare_connection(None, Some("self")).await.unwrap(); + assert_eq!(own.target, SOURCE); + let (resolver, resolver_task) = http_fixture(vec![response(200, TARGET)]).await; + let server: Uri = resolver.trim_end_matches("v1/credentials").parse().unwrap(); + let other = config.prepare_connection(Some(&server), Some("tasks")).await.unwrap(); + assert_eq!(other.target, TARGET); + assert_eq!(other.credential.target(), parse_identity(TARGET).unwrap()); + let requests = broker_task.await.unwrap(); + assert!(requests[0].ends_with(&format!("{{\"target_database\":\"{SOURCE}\"}}"))); + assert!(requests[1].ends_with(&format!("{{\"target_database\":\"{TARGET}\"}}"))); + assert!(resolver_task.await.unwrap()[0].starts_with("GET /v1/database/tasks/identity HTTP/1.1\r\n")); +} + +#[cfg(unix)] +#[tokio::test] +async fn unix_socket_uses_the_same_bounded_http_protocol_without_tcp_fallback() { + use std::{ + os::unix::fs::PermissionsExt, + sync::atomic::{AtomicUsize, Ordering}, + }; + use tokio::net::UnixListener; + static NEXT: AtomicUsize = AtomicUsize::new(0); + // Use an explicitly short base because ambient TMPDIR can exceed sockaddr_un + // limits on macOS. Atomic creation and private permissions retain ownership. + let directory = PathBuf::from("/tmp").join(format!( + "sdk-cred-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&directory).unwrap(); + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o700)).unwrap(); + struct Directory(PathBuf); + impl Drop for Directory { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + let directory = Directory(directory); + let socket = directory.0.join("broker.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let config = Container::new( + SOURCE, + "https://server.invalid", + &format!("unix://{}", socket.display()), + ) + .unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let request = read_request(&mut socket).await; + socket + .write_all(&response(200, &token_body("unix-secret", 20))) + .await + .unwrap(); + socket.shutdown().await.unwrap(); + request + }); + assert_eq!( + config + .token_for(parse_identity(TARGET).unwrap()) + .await + .unwrap() + .as_str(), + "unix-secret" + ); + let request = tokio::time::timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); + assert!(request.to_ascii_lowercase().contains("host: 127.0.0.1:18081\r\n")); + // The listener is gone but its pathname is retained: failure must not fall + // back to the HTTP authority used to format requests on this transport. + assert!(config.token_for(parse_identity(TARGET).unwrap()).await.is_err()); +} diff --git a/sdks/rust/src/db_connection.rs b/sdks/rust/src/db_connection.rs index 7f9f131dbaa..c93e8e7ec73 100644 --- a/sdks/rust/src/db_connection.rs +++ b/sdks/rust/src/db_connection.rs @@ -60,6 +60,8 @@ pub(crate) type SharedCell = Arc>; #[cfg(all(test, not(feature = "browser")))] mod builder_tests; +#[cfg(not(feature = "browser"))] +pub(crate) mod container_session; #[cfg(not(feature = "browser"))] mod native_tasks; #[cfg(not(feature = "browser"))] @@ -985,6 +987,9 @@ pub struct DbConnectionBuilder { token: Option, + #[cfg(not(feature = "browser"))] + container_credentials: Option, + on_connect: Option>, on_connect_error: Option>, on_disconnect: Option>, @@ -1044,6 +1049,8 @@ impl DbConnectionBuilder { uri: None, database_name: None, token: None, + #[cfg(not(feature = "browser"))] + container_credentials: None, on_connect: None, on_connect_error: None, on_disconnect: None, @@ -1122,10 +1129,44 @@ but you must call one of them, or else the connection will never progress. } /// Share native construction between the synchronous and asynchronous API. - /// The handshake is the only suspension point. Until it succeeds, this - /// future owns the socket and callbacks without spawning background tasks. + /// Credential acquisition and the handshake precede background connection + /// tasks. Cancelling either operation releases the pending connection. + #[cfg(not(feature = "browser"))] + async fn build_native_impl(mut self, handle: runtime::Handle) -> crate::Result> { + let credential = if let Some(container) = &self.container_credentials { + use crate::credentials::ContainerCredentialError; + // Reject before any socket or debug file is opened. Debug records + // include InitialConnection tokens; ordinary clients retain their + // existing behavior, but hosted credentials cannot be persisted. + if self.token.is_some() { + return Err(container_connect_error( + ContainerCredentialError::ConflictingCredentials, + )); + } + if self.additional_logging_path.is_some() { + return Err(container_connect_error(ContainerCredentialError::DebugLogging)); + } + let prepared = container + .prepare_connection(self.uri.as_ref(), self.database_name.as_deref()) + .await + .map_err(container_connect_error)?; + self.uri = Some(prepared.uri); + // Connect using the exact resolved Identity, so a later name + // reassignment cannot send this credential to another database. + self.database_name = Some(prepared.target); + Some(prepared.credential) + } else { + None + }; + self.build_native_with_credential(handle, credential).await + } + #[cfg(not(feature = "browser"))] - async fn build_native_impl(self, handle: runtime::Handle) -> crate::Result> { + async fn build_native_with_credential( + self, + handle: runtime::Handle, + credential: Option, + ) -> crate::Result> { let extra_logging = self .additional_logging_path .map(|path| { @@ -1137,17 +1178,33 @@ but you must call one of them, or else the connection will never progress. .map(|file| Arc::new(StdMutex::new(file))); let connection_id_override = get_connection_id_override(); - let ws_connection = WsConnection::connect( + let connect = WsConnection::connect( self.uri.unwrap(), self.database_name.as_ref().unwrap(), - self.token.as_deref(), + credential + .as_ref() + .map(|token| token.as_str()) + .or(self.token.as_deref()), connection_id_override, self.params, - ) - .await - .map_err(|source| crate::Error::FailedToConnect { - source: InternalError::new("Failed to initiate WebSocket connection").with_cause(source), - })?; + ); + let ws_connection = if let Some(credential) = &credential { + use crate::credentials::ContainerCredentialError; + let connection = tokio::time::timeout_at(credential.deadline(), connect) + .await + .map_err(|_| container_connect_error(ContainerCredentialError::Expired))? + // HTTP upgrade errors may contain response bodies/headers that + // echo secrets. Never retain those diagnostics for this mode. + .map_err(|_| container_connect_error(ContainerCredentialError::Transport))?; + if credential.remaining_lifetime().is_zero() { + return Err(container_connect_error(ContainerCredentialError::Expired)); + } + connection + } else { + connect.await.map_err(|source| crate::Error::FailedToConnect { + source: InternalError::new("Failed to initiate WebSocket connection").with_cause(source), + })? + }; let (websocket_loop_handle, raw_msg_recv, raw_msg_send) = ws_connection.spawn_message_loop(&handle, extra_logging.clone()); @@ -1250,6 +1307,28 @@ but you must call one of them, or else the connection will never progress. self } + /// Authenticate as the database backing this hosted container. + /// + /// Without explicit `with_uri` or `with_database_name` values, connect to + /// the database and server supplied by `container`. Other database names + /// are resolved on the selected server before requesting a target-specific + /// credential. The literal `self` resolves to the backing database Identity. + /// + /// Both `build` and `build_async` fetch a fresh credential on every call. + /// Failure never falls back to an anonymous Identity or saved owner token. + /// Combining this mode with a nonempty `with_token` or `with_debug_to_file` + /// is an error, regardless of the order of builder calls. + /// + /// The SDK does not automatically renew or reconnect this connection. + /// Handle expiration through `on_disconnect`, build a fresh connection, + /// and restore subscriptions as appropriate for your application. Tokens + /// returned through `on_connect` must not be saved or reused for reconnects. + #[cfg(not(feature = "browser"))] + pub fn with_container_credentials(mut self, container: crate::credentials::Container) -> Self { + self.container_credentials = Some(container); + self + } + /// Sets the compression used when a certain threshold in the message size has been reached. /// /// The current threshold used by the host is 1KiB for the entire server message @@ -1358,6 +1437,13 @@ Instead of registering multiple `on_disconnect` callbacks, register a single cal } } +#[cfg(not(feature = "browser"))] +fn container_connect_error(error: crate::credentials::ContainerCredentialError) -> crate::Error { + crate::Error::FailedToConnect { + source: InternalError::new("Failed to obtain or use container credentials").with_cause(error), + } +} + /// Create a [`DbContextImplInner`] wrapped in `Arc>`. fn build_db_ctx_inner( #[cfg(not(feature = "browser"))] runtime: Option, diff --git a/sdks/rust/src/db_connection/builder_tests.rs b/sdks/rust/src/db_connection/builder_tests.rs index 7158c02b9e5..0217d50c16d 100644 --- a/sdks/rust/src/db_connection/builder_tests.rs +++ b/sdks/rust/src/db_connection/builder_tests.rs @@ -87,3 +87,153 @@ fn sync_builder_returns_connect_errors_without_dropping_its_runtime_inside_block .build(); assert!(matches!(result, Err(crate::Error::FailedToConnect { .. }))); } + +#[tokio::test] +async fn container_builds_fetch_fresh_tokens_and_send_only_to_the_concrete_target() { + use crate::credentials::{container_tests as fixture, Container}; + use tokio::io::AsyncWriteExt; + let (broker, broker_task) = fixture::http_fixture(vec![ + fixture::response(200, &fixture::token_body("first-private-token", 20)), + fixture::response(200, &fixture::token_body("second-private-token", 20)), + ]) + .await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let server_uri = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + for expected in ["first-private-token", "second-private-token"] { + let (mut socket, _) = listener.accept().await.unwrap(); + let request = fixture::read_request(&mut socket).await; + assert!(request.starts_with(&format!("GET /v1/database/{}/subscribe?", fixture::SOURCE))); + assert!(request + .to_ascii_lowercase() + .contains(&format!("authorization: bearer {expected}\r\n"))); + // Deliberately echo secret material in a failed upgrade. Hosted + // connection errors must not retain the response or its headers. + socket.write_all(&fixture::response(403, expected)).await.unwrap(); + socket.shutdown().await.unwrap(); + } + }); + let container = Container::new(fixture::SOURCE, &server_uri, &broker).unwrap(); + for _ in 0..2 { + let error = DbConnectionBuilder::::new() + .with_container_credentials(container.clone()) + .with_database_name("self") + .build_async() + .await + .err() + .expect("fixture rejects the WebSocket upgrade"); + assert!(!format!("{error:?} {error}").contains("private-token")); + } + assert_eq!(broker_task.await.unwrap().len(), 2); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); +} + +#[tokio::test] +async fn container_auth_rejects_static_credentials_and_debug_files_before_io() { + use crate::credentials::{container_tests as fixture, Container}; + let container = Container::new( + fixture::SOURCE, + "https://must-not-contact.invalid", + "http://127.0.0.1:1/v1/credentials", + ) + .unwrap(); + let path = std::env::temp_dir().join(format!("must-not-write-sdk-credentials-{}", std::process::id())); + assert!(!path.exists()); + let result = DbConnectionBuilder::::new() + .with_token(Some("owner-secret")) + .with_container_credentials(container.clone()) + .build_async() + .await; + let error = result.err().unwrap(); + assert!(error.to_string().contains("static token")); + assert!(!format!("{error:?} {error}").contains("owner-secret")); + let result = DbConnectionBuilder::::new() + .with_container_credentials(container) + .with_debug_to_file(&path) + .build_async() + .await; + assert!(result.err().unwrap().to_string().contains("debug files")); + assert!(!path.exists()); +} + +#[tokio::test] +async fn cancelling_container_build_closes_broker_socket_and_releases_callbacks() { + use crate::credentials::{container_tests as fixture, Container}; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let broker = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let container = Container::new(fixture::SOURCE, "https://must-not-contact.invalid", &broker).unwrap(); + let (ready_tx, ready_rx) = oneshot::channel(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + ready_tx.send(()).unwrap(); + let mut byte = [0]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), socket.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + }); + let capture = Arc::new(()); + let weak = Arc::downgrade(&capture); + let mut connect = Box::pin( + DbConnectionBuilder::::new() + .with_container_credentials(container) + .on_connect(move |_, _, _| drop(capture)) + .build_async(), + ); + tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + _ = &mut connect => panic!("stalled broker unexpectedly answered"), + ready = ready_rx => ready.unwrap(), + } + }) + .await + .unwrap(); + drop(connect); + assert!(weak.upgrade().is_none()); + server.await.unwrap(); +} + +#[tokio::test] +async fn container_token_expiry_closes_a_stalled_websocket_handshake() { + use crate::credentials::{container_tests as fixture, Container}; + let (broker, broker_task) = + fixture::http_fixture(vec![fixture::response(200, &fixture::token_body("expiring-token", 1))]).await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let server_uri = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + let mut byte = [0]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), socket.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + }); + let container = Container::new(fixture::SOURCE, &server_uri, &broker).unwrap(); + let error = tokio::time::timeout( + Duration::from_secs(2), + DbConnectionBuilder::::new() + .with_container_credentials(container) + .build_async(), + ) + .await + .unwrap() + .err() + .unwrap(); + assert!(error.to_string().contains("expired")); + broker_task.await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); +} diff --git a/sdks/rust/src/db_connection/container_session.rs b/sdks/rust/src/db_connection/container_session.rs new file mode 100644 index 00000000000..e4500202e43 --- /dev/null +++ b/sdks/rust/src/db_connection/container_session.rs @@ -0,0 +1,621 @@ +//! Explicit ownership of successive container-authenticated connections. + +use super::{ConnectionLifecycle, DbConnectionBuilder, DbContextImpl, SpacetimeModule, WsParams}; +use crate::{ + credentials::{Container, ContainerCredentialError, ContainerToken}, + Identity, +}; +use futures::FutureExt; +use http::Uri; +use std::{ + panic::{resume_unwind, AssertUnwindSafe}, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, + }, + time::{Duration, SystemTime}, +}; +use tokio::{ + sync::Notify, + task::AbortHandle, + time::{sleep_until, Instant}, +}; + +#[cfg(test)] +mod tests; + +const RETRY_MIN: Duration = Duration::from_millis(250); +const RETRY_MAX: Duration = Duration::from_secs(5); +const MIN_EXTENSION: Duration = Duration::from_secs(1); +const REFRESH_MARGIN: Duration = Duration::from_secs(10); + +type Result = std::result::Result; +type Configure = Box) -> DbConnectionBuilder + Send>; + +/// One connection generation. The target is resolved once for the owner's +/// lifetime; reconnecting cannot follow a database name to another Identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ContainerSessionInfo { + pub generation: u64, + pub target: Identity, +} + +/// Why a managed connection is being closed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum ContainerSessionEndReason { + CredentialRenewal, + CredentialExpiry, + Disconnected, + Shutdown, + Failed, +} + +/// Unconfirmed reducer and procedure calls may already have committed. +/// The session does not track individual calls and never replays them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OutstandingCallOutcomes { + Unknown, +} + +/// Redacted reason for a retry. No endpoint, token, or server response is retained. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum ContainerSessionRetryReason { + Credential(ContainerCredentialError), + ExpiryNotExtended, + Connection, +} + +/// Session events contain no credentials. `Connected` means the server sent its +/// initial connection message. Subscriptions become ready separately, through +/// their normal `on_applied` callbacks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum ContainerSessionEvent { + Connecting(ContainerSessionInfo), + Connected(ContainerSessionInfo), + /// The outgoing channel is sealed before this event is delivered. + Closing { + session: ContainerSessionInfo, + reason: ContainerSessionEndReason, + outstanding_calls: OutstandingCallOutcomes, + }, + /// Both native tasks have been joined before this event is delivered. + Closed { + session: ContainerSessionInfo, + reason: ContainerSessionEndReason, + outstanding_calls: OutstandingCallOutcomes, + }, + Retrying(ContainerSessionRetryReason), +} + +/// Redacted, terminal session errors. A denied credential is never retried or +/// replaced with an owner or anonymous credential. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum ContainerSessionError { + #[error("Invalid container session configuration")] + Configuration, + #[error("Container credential error: {0}")] + Credential(ContainerCredentialError), + #[error("Container session server returned a different sender Identity")] + IdentityMismatch, + #[error("Container session connection tasks failed during shutdown")] + Shutdown, + #[error("Container session application callback panicked")] + Panicked, + #[error("Container session generation counter exhausted")] + GenerationExhausted, +} + +/// Own successive connections authenticated through a container's broker. +/// +/// Pass a generated `DbConnection::builder()` with container discovery and an +/// optional server and database selection. The factory receives a fresh builder +/// for every attempt. Use it to create callbacks and recreate subscriptions in +/// `on_connect`. It must not set authentication, a target, or debug-file logging. +/// Put all callbacks in the factory, not in the initial builder. +/// +/// ```ignore +/// let mut session = spacetimedb_sdk::ContainerSession::new( +/// DbConnection::builder() +/// .with_container_credentials(spacetimedb_sdk::credentials::Container::from_env()?), +/// |generation, builder| builder.on_connect(move |conn, _, _| { +/// // Register callbacks and build new subscriptions for this generation. +/// conn.subscription_builder().subscribe("SELECT * FROM jobs"); +/// }), +/// )?.on_event(|event| { /* observe rotation and uncertain outstanding calls */ }); +/// tokio::select! { +/// result = session.run() => result?, +/// _ = shutdown_signal => {}, +/// } +/// session.shutdown_and_join().await?; +/// ``` +/// +/// `run` drives the connection and refreshes credentials before expiry. A token +/// that extends the lease causes a reconnect with an empty cache. There is an +/// interruption while the old connection shuts down and subscriptions rebuild. +/// The old WebSocket and parser are joined before the next WebSocket is opened. +/// Reducers and procedures are never replayed. Treat unconfirmed calls in a +/// closing generation as having unknown outcomes; use application request IDs +/// when an operation must be safely retried. +/// +/// Do not drive generated connection handles independently of this owner. +/// Handles supplied to callbacks belong to that generation and become inactive +/// when it closes. A new generation gets new callbacks, subscriptions and cache. +/// +/// Cancelling `run` pauses this owner, including renewal. Call `run` again to +/// resume, or call `shutdown_and_join` to finish shutdown. Cancelling the latter +/// is resumable. Dropping the owner aborts native tasks even if application code +/// retained a connection, but cannot wait for them; explicit shutdown is required +/// for completed cleanup. An application panic is propagated after task cleanup. +pub struct ContainerSession { + container: Container, + uri: Option, + database: Option, + target: Option<(Uri, Identity)>, + params: WsParams, + configure: Configure, + event: Option>, + current: Option>, + next_token: Option, + generation: u64, + retry_at: Option, + retry_delay: Duration, + stopped: bool, + failure: Option, +} + +struct Current { + context: DbContextImpl, + info: ContainerSessionInfo, + validity: Validity, + refresh_at: Instant, + signal: Arc, + connected_announced: bool, + closing: Option, + driver_finished: bool, + aborts: Vec, +} + +#[derive(Clone, Copy)] +struct Validity { + expiry: SystemTime, + deadline: Instant, +} + +impl Validity { + fn from_token(token: &ContainerToken) -> Self { + Self { + expiry: token.expires_at(), + deadline: token.deadline(), + } + } + + fn refresh_at(self) -> Instant { + let remaining = self.deadline.saturating_duration_since(Instant::now()); + self.deadline - REFRESH_MARGIN.min(remaining / 2) + } + + fn extended_by(self, new: Self) -> bool { + // The broker may issue another token with the same lease expiry. A + // fresh receipt time alone must not trigger endless reconnects. + new.expiry + .duration_since(self.expiry) + .is_ok_and(|delta| delta >= MIN_EXTENSION) + && new.deadline.saturating_duration_since(self.deadline) >= MIN_EXTENSION + } +} + +#[derive(Default)] +struct Signal { + // 0: waiting; 1: connected; 2: unexpected sender Identity. + state: AtomicU8, + notify: Notify, +} + +impl ContainerSession { + /// Create an owner without performing I/O. The initial builder supplies + /// discovery, target and WebSocket options. The factory supplies callbacks. + pub fn new( + mut initial: DbConnectionBuilder, + configure: impl FnMut(ContainerSessionInfo, DbConnectionBuilder) -> DbConnectionBuilder + Send + 'static, + ) -> Result { + if initial.token.is_some() + || initial.additional_logging_path.is_some() + || initial.on_connect.is_some() + || initial.on_disconnect.is_some() + || initial.on_connect_error.is_some() + { + return Err(ContainerSessionError::Configuration); + } + let container = initial + .container_credentials + .take() + .ok_or(ContainerSessionError::Configuration)?; + Ok(Self { + container, + uri: initial.uri, + database: initial.database_name, + target: None, + params: initial.params, + configure: Box::new(configure), + event: None, + current: None, + next_token: None, + generation: 0, + retry_at: None, + retry_delay: RETRY_MIN, + stopped: false, + failure: None, + }) + } + + /// Register a synchronous observer for redacted session events. + pub fn on_event(mut self, event: impl FnMut(ContainerSessionEvent) + Send + 'static) -> Self { + self.event = Some(Box::new(event)); + self + } + + /// Drive connections and credential renewal until shutdown or a terminal + /// error. This future owns no detached manager task and may be cancelled and + /// resumed. Calls made through each connection retain their normal semantics. + pub async fn run(&mut self) -> Result<()> { + let result = AssertUnwindSafe(async { + match self.run_loop().await { + Ok(()) => self.failure.map_or(Ok(()), Err), + Err(error) => { + self.failure.get_or_insert(error); + self.stopped = true; + self.next_token = None; + self.close_current(ContainerSessionEndReason::Failed); + self.finish_current().await?; + Err(self.failure.unwrap()) + } + } + }) + .catch_unwind() + .await; + match result { + Ok(result) => result, + Err(panic) => { + self.stopped = true; + self.next_token = None; + self.failure.get_or_insert(ContainerSessionError::Panicked); + // Retain the observer until cleanup finishes: even a callback + // capture's destructor must not bypass task ownership. + let observer = self.event.take(); + self.close_current(ContainerSessionEndReason::Failed); + let _ = AssertUnwindSafe(self.finish_current()).catch_unwind().await; + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| drop(observer))); + resume_unwind(panic) + } + } + } + + /// Stop renewal, seal outgoing calls and positively join the current native + /// connection. Safe to call again after cancellation or completed shutdown. + pub async fn shutdown_and_join(&mut self) -> Result<()> { + self.stopped = true; + self.next_token = None; + let result = AssertUnwindSafe(async { + self.close_current(ContainerSessionEndReason::Shutdown); + self.finish_current().await + }) + .catch_unwind() + .await; + match result { + Ok(result) => result.and_then(|()| self.failure.map_or(Ok(()), Err)), + Err(panic) => { + self.failure.get_or_insert(ContainerSessionError::Panicked); + let observer = self.event.take(); + let _ = AssertUnwindSafe(self.finish_current()).catch_unwind().await; + let _ = std::panic::catch_unwind(AssertUnwindSafe(|| drop(observer))); + resume_unwind(panic) + } + } + } + + async fn run_loop(&mut self) -> Result<()> { + loop { + if self.current.as_ref().is_some_and(|current| current.closing.is_some()) { + self.finish_current().await?; + } + if self.stopped { + return self.failure.map_or(Ok(()), Err); + } + if self.current.is_none() { + if let Some(at) = self.retry_at { + sleep_until(at).await; + self.retry_at = None; + } + if self.target.is_none() { + // Store the concrete target before awaiting token issuance, + // including when a caller cancels during that request. + self.target = Some( + self.container + .resolve_connection_target(self.uri.as_ref(), self.database.as_deref()) + .await + .map_err(ContainerSessionError::Credential)?, + ); + } + let target = self.target.as_ref().unwrap().1; + let token = match self + .next_token + .take() + .filter(|token| !token.remaining_lifetime().is_zero()) + { + Some(token) => token, + None => match self.container.token_for(target).await { + Ok(token) => token, + Err(error) => { + self.credential_retry(error)?; + continue; + } + }, + }; + if !self.start(token).await? { + continue; + } + } + self.announce_connected()?; + let current = self.current.as_ref().unwrap(); + let context = current.context.clone(); + let signal = current.signal.clone(); + let deadline = current.validity.deadline; + let refresh_at = current.refresh_at; + tokio::select! { + biased; + result = AssertUnwindSafe(context.run_async()).catch_unwind() => self.driver_finished(result)?, + _ = signal.notify.notified() => {}, + _ = sleep_until(deadline) => self.close_current(ContainerSessionEndReason::CredentialExpiry), + _ = sleep_until(refresh_at) => self.refresh().await?, + } + } + } + + async fn start(&mut self, token: ContainerToken) -> Result { + self.generation = self + .generation + .checked_add(1) + .ok_or(ContainerSessionError::GenerationExhausted)?; + let (uri, target) = self.target.as_ref().unwrap().clone(); + let info = ContainerSessionInfo { + generation: self.generation, + target, + }; + let mut seed = DbConnectionBuilder::new(); + seed.params = self.params; + let mut builder = (self.configure)(info, seed); + if builder.uri.is_some() + || builder.database_name.is_some() + || builder.token.is_some() + || builder.container_credentials.is_some() + || builder.additional_logging_path.is_some() + { + return Err(ContainerSessionError::Configuration); + } + let signal = Arc::new(Signal::default()); + let connected_signal = signal.clone(); + let expected_sender = self.container.database_identity(); + let on_connect = builder.on_connect.take(); + builder.on_connect = Some(Box::new(move |connection, identity, token| { + if identity != expected_sender { + connected_signal.state.store(2, Ordering::Release); + } else { + if let Some(callback) = on_connect { + callback(connection, identity, token); + } + connected_signal.state.store(1, Ordering::Release); + } + connected_signal.notify.notify_one(); + })); + builder.uri = Some(uri); + builder.database_name = Some(target.to_hex().to_string()); + let validity = Validity::from_token(&token); + self.emit(ContainerSessionEvent::Connecting(info)); + let handle = tokio::runtime::Handle::current(); + let context = match builder.build_native_with_credential(handle, Some(token)).await { + Ok(context) => context, + Err(_) => { + self.retry(ContainerSessionRetryReason::Connection); + return Ok(false); + } + }; + // A new context has no driver yet. Its task owner cannot be contended. + // No suspension is allowed between construction and retaining Current. + let aborts = context + .inner + .lock() + .unwrap() + .background_tasks + .try_lock() + .unwrap() + .abort_handles(); + self.current = Some(Current { + context, + info, + validity, + refresh_at: validity.refresh_at(), + signal, + connected_announced: false, + closing: None, + driver_finished: false, + aborts, + }); + self.retry_at = None; + Ok(true) + } + + async fn refresh(&mut self) -> Result<()> { + let current = self.current.as_ref().unwrap(); + let context = current.context.clone(); + let signal = current.signal.clone(); + let deadline = current.validity.deadline; + let target = current.info.target; + tokio::select! { + biased; + result = AssertUnwindSafe(context.run_async()).catch_unwind() => self.driver_finished(result)?, + _ = signal.notify.notified() => {}, + _ = sleep_until(deadline) => self.close_current(ContainerSessionEndReason::CredentialExpiry), + result = self.container.token_for(target) => { + match result { + Ok(token) if self.current.as_ref().unwrap().validity.extended_by(Validity::from_token(&token)) => { + self.next_token = Some(token); + self.close_current(ContainerSessionEndReason::CredentialRenewal); + } + Ok(_) => self.retry(ContainerSessionRetryReason::ExpiryNotExtended), + Err(error) => self.credential_retry(error)?, + } + } + } + Ok(()) + } + + fn credential_retry(&mut self, error: ContainerCredentialError) -> Result<()> { + match error { + ContainerCredentialError::Unavailable + | ContainerCredentialError::Transport + | ContainerCredentialError::Timeout => { + self.retry(ContainerSessionRetryReason::Credential(error)); + Ok(()) + } + _ => Err(ContainerSessionError::Credential(error)), + } + } + + fn retry(&mut self, reason: ContainerSessionRetryReason) { + let at = Instant::now() + self.retry_delay; + self.retry_delay = (self.retry_delay * 2).min(RETRY_MAX); + if let Some(current) = self.current.as_mut() { + current.refresh_at = at; + } else { + self.retry_at = Some(at); + } + self.emit(ContainerSessionEvent::Retrying(reason)); + } + + fn announce_connected(&mut self) -> Result<()> { + let current = self.current.as_mut().unwrap(); + match current.signal.state.load(Ordering::Acquire) { + 2 => return Err(ContainerSessionError::IdentityMismatch), + 1 if !current.connected_announced => { + current.connected_announced = true; + self.retry_delay = RETRY_MIN; + let info = current.info; + self.emit(ContainerSessionEvent::Connected(info)); + } + _ => {} + } + Ok(()) + } + + fn driver_finished(&mut self, result: std::thread::Result>) -> Result<()> { + // run_async joins both native tasks before returning or unwinding. + self.current.as_mut().unwrap().driver_finished = true; + match result { + Err(panic) => resume_unwind(panic), + Ok(_) => { + // EOF can win the select immediately after InitialConnection. + // Check the observed sender before discarding this generation. + self.announce_connected()?; + self.close_current(ContainerSessionEndReason::Disconnected); + self.retry_at = Some(Instant::now() + self.retry_delay); + self.retry_delay = (self.retry_delay * 2).min(RETRY_MAX); + Ok(()) + } + } + } + + fn close_current(&mut self, reason: ContainerSessionEndReason) { + let Some(current) = self.current.as_mut() else { + return; + }; + if current.closing.is_some() { + return; + } + current.closing = Some(reason); + current.request_stop(); + let session = current.info; + self.emit(ContainerSessionEvent::Closing { + session, + reason, + outstanding_calls: OutstandingCallOutcomes::Unknown, + }); + } + + async fn finish_current(&mut self) -> Result<()> { + let Some(current) = self.current.as_ref() else { + return Ok(()); + }; + let reason = current.closing.expect("only a closing connection can be joined"); + let info = current.info; + let context = current.context.clone(); + if !current.driver_finished { + let tasks = context + .inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .background_tasks + .clone(); + let joined = tasks.lock().await.stop_and_join().await; + // A cancelled join is resumable through the retained NativeTasks. + self.current.as_mut().unwrap().driver_finished = true; + if joined.is_err() { + self.failure.get_or_insert(ContainerSessionError::Shutdown); + self.stopped = true; + } + } + // Complete normal SDK terminal cleanup only after native tasks joined. + // This discards queued and in-flight calls without replaying them or + // manufacturing success, including when a retained handle exists. + let terminal = std::panic::catch_unwind(AssertUnwindSafe(|| context.end_connection(None))); + self.current = None; + if let Err(panic) = terminal { + resume_unwind(panic); + } + self.emit(ContainerSessionEvent::Closed { + session: info, + reason, + outstanding_calls: OutstandingCallOutcomes::Unknown, + }); + self.failure.map_or(Ok(()), Err) + } + + fn emit(&mut self, event: ContainerSessionEvent) { + if let Some(observer) = self.event.as_mut() { + observer(event); + } + } +} + +impl Current { + fn request_stop(&self) { + // Seal calls synchronously, including handles retained by applications. + // Closing before InitialConnection has the ordinary disconnect semantics. + let mut inner = self.context.inner.lock().unwrap_or_else(|error| error.into_inner()); + if inner.connection_lifecycle == ConnectionLifecycle::Connecting { + inner.connection_lifecycle = ConnectionLifecycle::Ended; + } + drop(inner); + let outgoing = self + .context + .send_chan + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + drop(outgoing); + for task in &self.aborts { + task.abort(); + } + } +} + +impl Drop for ContainerSession { + fn drop(&mut self) { + if let Some(current) = &self.current { + current.request_stop(); + } + } +} diff --git a/sdks/rust/src/db_connection/container_session/tests.rs b/sdks/rust/src/db_connection/container_session/tests.rs new file mode 100644 index 00000000000..a39afc708f3 --- /dev/null +++ b/sdks/rust/src/db_connection/container_session/tests.rs @@ -0,0 +1,704 @@ +use super::super::{ + native_tasks::NativeTasks, + terminal_tests::bindings::{self, RemoteModule}, +}; +use super::*; +use crate::{credentials::container_tests as fixture, DbContext, Table}; +use bindings::{identity_connected, ConnectedTableAccess}; +use futures::{SinkExt, StreamExt}; +use spacetimedb_client_api_messages::websocket::{ + common::{BsatnRowList, RowSizeHint}, + v2 as ws, +}; +use spacetimedb_lib::{bsatn, ConnectionId}; +use std::{ + future::Future, + sync::{atomic::AtomicUsize, Mutex}, + time::UNIX_EPOCH, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{mpsc, oneshot}, + task::JoinHandle, + time::timeout, +}; +use tokio_tungstenite::{ + tungstenite::{handshake::server::Response, Message}, + WebSocketStream, +}; + +#[derive(spacetimedb_lib::ser::Serialize)] +#[sats(crate = spacetimedb_lib)] +struct EmptyArgs {} +impl crate::spacetime_module::InModule for EmptyArgs { + type Module = RemoteModule; +} +use std::result::Result; + +struct Task(Option>); +impl Task { + fn spawn(body: impl Future + Send + 'static) -> Self + where + T: Send + 'static, + { + Self(Some(tokio::spawn(body))) + } + async fn join(mut self) -> T { + let result = timeout(Duration::from_secs(7), self.0.as_mut().unwrap()) + .await + .unwrap() + .unwrap(); + self.0 = None; + result + } +} +impl Drop for Task { + fn drop(&mut self) { + if let Some(task) = &self.0 { + task.abort(); + } + } +} + +#[derive(Default)] +struct PeerStats { + subscriptions: usize, + reducers: usize, + procedures: usize, +} + +async fn send(socket: &mut WebSocketStream, message: ws::ServerMessage) { + let mut bytes = vec![0]; + bytes.extend(bsatn::to_vec(&message).unwrap()); + socket.send(Message::Binary(bytes.into())).await.unwrap(); +} + +async fn peer(sessions: usize, sender: Identity) -> (String, mpsc::UnboundedReceiver, Task>) { + peer_with_close(sessions, sender, false).await +} + +async fn peer_with_close( + sessions: usize, + sender: Identity, + close_initially: bool, +) -> (String, mpsc::UnboundedReceiver, Task>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let uri = format!("http://{}", listener.local_addr().unwrap()); + let (applied, rx) = mpsc::unbounded_channel(); + let task = Task::spawn(async move { + timeout(Duration::from_secs(6), async { + let mut stats = Vec::new(); + for generation in 1..=sessions { + let (socket, remote) = listener.accept().await.unwrap(); + assert!(remote.ip().is_loopback()); + let mut socket = tokio_tungstenite::accept_hdr_async( + socket, + |request: &http::Request<()>, mut response: Response| { + assert!(request.uri().path().contains(fixture::SOURCE)); + assert!(request.headers()["authorization"] + .to_str() + .unwrap() + .starts_with("Bearer disposable-")); + response + .headers_mut() + .insert("sec-websocket-protocol", ws::BIN_PROTOCOL.parse().unwrap()); + Ok(response) + }, + ) + .await + .unwrap(); + send( + &mut socket, + ws::ServerMessage::InitialConnection(ws::InitialConnection { + identity: sender, + connection_id: ConnectionId::from_u128(generation as u128), + token: "disposable-server-token".into(), + }), + ) + .await; + let mut current = PeerStats::default(); + if close_initially { + socket.close(None).await.unwrap(); + stats.push(current); + continue; + } + while let Some(message) = socket.next().await { + let Ok(Message::Binary(bytes)) = message else { + break; + }; + match bsatn::from_slice::(&bytes).unwrap() { + ws::ClientMessage::Subscribe(subscribe) => { + current.subscriptions += 1; + let row = bsatn::to_vec(&bindings::Connected { identity: sender }).unwrap(); + send( + &mut socket, + ws::ServerMessage::SubscribeApplied(ws::SubscribeApplied { + request_id: subscribe.request_id, + query_set_id: subscribe.query_set_id, + rows: ws::QueryRows { + tables: vec![ws::SingleTableRows { + table: "connected".into(), + rows: BsatnRowList::new( + RowSizeHint::RowOffsets(vec![0].into()), + row.into(), + ), + }] + .into(), + }, + }), + ) + .await; + let _ = applied.send(generation); + } + ws::ClientMessage::CallReducer(_) => current.reducers += 1, + ws::ClientMessage::CallProcedure(_) => current.procedures += 1, + _ => panic!("unexpected fixture request"), + } + } + stats.push(current); + } + stats + }) + .await + .expect("owned WebSocket fixture exceeded deadline") + }); + (uri, rx, task) +} + +fn source() -> Identity { + Identity::from_hex(fixture::SOURCE).unwrap() +} +fn config(uri: &str, broker: &str) -> DbConnectionBuilder { + bindings::DbConnection::builder().with_container_credentials(Container::new(fixture::SOURCE, uri, broker).unwrap()) +} + +async fn drive_until(session: &mut ContainerSession, condition: impl Future) { + timeout(Duration::from_secs(5), async { + tokio::select! { + result = session.run() => panic!("session terminated unexpectedly: {result:?}"), + _ = condition => {}, + } + }) + .await + .unwrap(); +} + +#[test] +fn rejects_ambiguous_initial_configuration_without_io() { + let factory = |_, builder| builder; + assert!(matches!( + ContainerSession::new(bindings::DbConnection::builder(), factory), + Err(ContainerSessionError::Configuration) + )); + let new = || config("https://must-not-contact.invalid", "http://127.0.0.1:1/v1/credentials"); + for builder in [ + new().with_token(Some("owner-secret")), + new().with_debug_to_file("must-not-write"), + new().on_connect(|_, _, _| {}), + ] { + let error = ContainerSession::new(builder, factory).err().unwrap(); + assert_eq!(error, ContainerSessionError::Configuration); + assert!(!format!("{error:?} {error}").contains("owner-secret")); + } +} + +#[test] +fn requires_material_extension_of_both_returned_expiry_and_monotonic_deadline() { + let old = Validity { + expiry: UNIX_EPOCH + Duration::from_secs(100), + deadline: Instant::now(), + }; + assert!(!old.extended_by(Validity { + expiry: old.expiry, + deadline: old.deadline + Duration::from_secs(20) + })); + assert!(!old.extended_by(Validity { + expiry: old.expiry + Duration::from_secs(20), + deadline: old.deadline + })); + assert!(!old.extended_by(Validity { + expiry: old.expiry + Duration::from_millis(999), + deadline: old.deadline + Duration::from_secs(20) + })); + assert!(old.extended_by(Validity { + expiry: old.expiry + Duration::from_secs(2), + deadline: old.deadline + Duration::from_secs(2) + })); +} + +#[tokio::test] +async fn renewal_rebuilds_subscriptions_and_cache_without_replaying_calls() { + let (broker, broker_task) = fixture::http_fixture(vec![ + fixture::response(200, &fixture::token_body("disposable-first", 3)), + fixture::response(200, &fixture::token_body("disposable-second", 20)), + ]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _wire_applied, peer_task) = peer(2, source()).await; + let events = Arc::new(Mutex::new(Vec::new())); + let event_capture = events.clone(); + let (ready, mut readiness) = mpsc::unbounded_channel(); + let drops = Arc::new(AtomicUsize::new(0)); + let captures = drops.clone(); + let mut session = ContainerSession::new(config(&uri, &broker), move |info, builder| { + let ready = ready.clone(); + let captures = captures.clone(); + builder.on_connect(move |conn, _, _| { + // The first generation will receive a row before rotation. The + // second must still start with an empty, newly allocated cache. + assert_eq!(conn.db.connected().count(), 0); + conn.subscription_builder() + .on_applied(move |ctx| { + assert_eq!(ctx.db.connected().count(), 1); + ready.send(info.generation).unwrap(); + }) + .subscribe("SELECT * FROM connected"); + if info.generation == 1 { + struct Probe(Arc); + impl Drop for Probe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let probe = Probe(captures); + conn.reducers + .identity_connected_then(move |_, _| { + let _ = &probe; + panic!("unconfirmed reducer outcome must remain unknown"); + }) + .unwrap(); + } + }) + }) + .unwrap() + .on_event(move |event| event_capture.lock().unwrap().push(event)); + drive_until(&mut session, async { + assert_eq!(readiness.recv().await, Some(1)); + }) + .await; + let retained = session.current.as_ref().unwrap().context.clone(); + retained.invoke_procedure_with_callback::<_, ()>("unconfirmed", EmptyArgs {}, |_, _| { + panic!("unconfirmed procedure must not report completion") + }); + let first_aborts = session.current.as_ref().unwrap().aborts.clone(); + let first_cache = retained.cache.clone(); + drive_until(&mut session, async { + assert_eq!(readiness.recv().await, Some(2)); + }) + .await; + assert!(first_aborts.iter().all(AbortHandle::is_finished)); + assert!(!retained.is_active()); + assert!(!Arc::ptr_eq( + &first_cache, + &session.current.as_ref().unwrap().context.cache + )); + assert_eq!(drops.load(Ordering::SeqCst), 1); + session.shutdown_and_join().await.unwrap(); + session.shutdown_and_join().await.unwrap(); + assert!(session.current.is_none()); + let requests = broker_task.join().await; + assert_eq!(requests.len(), 2); + assert!(requests + .iter() + .all(|request| request.ends_with(&format!("{{\"target_database\":\"{}\"}}", fixture::SOURCE)))); + let stats = peer_task.join().await; + assert_eq!(stats[0].subscriptions, 1); + assert_eq!(stats[0].reducers, 1); + assert_eq!(stats[0].procedures, 1); + assert_eq!(stats[1].subscriptions, 1); + assert_eq!(stats[1].reducers, 0); + assert_eq!(stats[1].procedures, 0); + let events = events.lock().unwrap(); + let closed = events.iter().position(|event| matches!(event, ContainerSessionEvent::Closed { session, reason: ContainerSessionEndReason::CredentialRenewal, outstanding_calls: OutstandingCallOutcomes::Unknown } if session.generation == 1)).unwrap(); + let next = events + .iter() + .position(|event| matches!(event, ContainerSessionEvent::Connecting(info) if info.generation == 2)) + .unwrap(); + assert!(closed < next); + assert!(!format!("{events:?}").contains("disposable-")); +} + +#[tokio::test] +async fn unchanged_lease_expiry_does_not_rotate_and_denial_closes_the_active_session() { + let body = fixture::token_body("disposable-same-lease", 3); + let (broker, broker_task) = fixture::http_fixture(vec![ + fixture::response(200, &body), + fixture::response(200, &body), + fixture::response(200, &body), + fixture::response(403, "private-error-body"), + ]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, source()).await; + let events = Arc::new(Mutex::new(Vec::new())); + let capture = events.clone(); + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| builder) + .unwrap() + .on_event(move |event| capture.lock().unwrap().push(event)); + let error = timeout(Duration::from_secs(5), session.run()) + .await + .unwrap() + .unwrap_err(); + assert_eq!( + error, + ContainerSessionError::Credential(ContainerCredentialError::Denied) + ); + assert!(session.current.is_none()); + assert_eq!(broker_task.join().await.len(), 4); + assert_eq!(peer_task.join().await.len(), 1); + assert_eq!( + events + .lock() + .unwrap() + .iter() + .filter(|event| matches!(event, ContainerSessionEvent::Connecting(_))) + .count(), + 1 + ); + assert!(!format!("{error:?} {:?}", events.lock().unwrap()).contains("private-error-body")); + assert_eq!(session.run().await.unwrap_err(), error); +} + +#[tokio::test] +async fn factory_cannot_override_the_pinned_target_or_supply_a_static_token() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-factory", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let mut session = ContainerSession::new(config("https://must-not-contact.invalid", &broker), |_, builder| { + builder.with_database_name("other").with_token(Some("owner-secret")) + }) + .unwrap(); + assert_eq!(session.run().await.unwrap_err(), ContainerSessionError::Configuration); + assert_eq!(broker_task.join().await.len(), 1); + assert!(session.current.is_none()); +} + +#[tokio::test] +async fn cancelling_token_request_preserves_resolved_target_and_closes_request_socket() { + let server = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let uri = format!("http://{}", server.local_addr().unwrap()); + let resolver = Task::spawn(async move { + let (mut socket, _) = server.accept().await.unwrap(); + assert!(fixture::read_request(&mut socket) + .await + .starts_with("GET /v1/database/alias/identity ")); + socket + .write_all(&fixture::response(200, fixture::SOURCE)) + .await + .unwrap(); + }); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let broker = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let (received, receipt) = oneshot::channel(); + let server = Task::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + received.send(()).unwrap(); + let mut byte = [0]; + assert_eq!( + timeout(Duration::from_secs(2), socket.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + let (mut socket, _) = listener.accept().await.unwrap(); + assert!(fixture::read_request(&mut socket).await.contains(fixture::SOURCE)); + socket.write_all(&fixture::response(403, "denied")).await.unwrap(); + }); + let mut session = + ContainerSession::new(config(&uri, &broker).with_database_name("alias"), |_, builder| builder).unwrap(); + drive_until(&mut session, async { + receipt.await.unwrap(); + }) + .await; + assert_eq!(session.target.as_ref().unwrap().1, source()); + resolver.join().await; + // Resolver is gone. Resume still requests exactly the pinned Identity. + assert_eq!( + session.run().await.unwrap_err(), + ContainerSessionError::Credential(ContainerCredentialError::Denied) + ); + server.join().await; + session.shutdown_and_join().await.unwrap_err(); +} + +#[tokio::test] +async fn dropping_owner_aborts_native_tasks_despite_retained_connection() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-drop", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, source()).await; + let (connected, receipt) = oneshot::channel(); + let mut connected = Some(connected); + let mut session = ContainerSession::new(config(&uri, &broker), move |_, builder| { + let connected = connected.take().unwrap(); + builder.on_connect(move |_, _, _| { + connected.send(()).unwrap(); + }) + }) + .unwrap(); + drive_until(&mut session, async { + receipt.await.unwrap(); + }) + .await; + let retained = session.current.as_ref().unwrap().context.clone(); + let tasks = retained.inner.lock().unwrap().background_tasks.clone(); + drop(session); + assert!(!retained.is_active()); + tasks.lock().await.stop_and_join().await.unwrap(); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn cancelling_shutdown_keeps_native_join_ownership_until_resumed() { + // A task destructor deliberately waits on a gate. This proves that the + // shutdown method retains the JoinHandle while its wait is cancelled. + let (entered, entry) = oneshot::channel(); + let (release, gate) = std::sync::mpsc::channel(); + struct BlockingDrop { + entered: Option>, + gate: std::sync::mpsc::Receiver<()>, + } + impl Drop for BlockingDrop { + fn drop(&mut self) { + let _ = self.entered.take().unwrap().send(()); + self.gate.recv_timeout(Duration::from_secs(3)).unwrap(); + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let destructor = BlockingDrop { + entered: Some(entered), + gate, + }; + let websocket = runtime.spawn(async move { + let _destructor = destructor; + futures::future::pending::<()>().await; + }); + let parser = runtime.spawn(futures::future::pending()); + let mut session = ContainerSession::new( + config("http://127.0.0.1:1", "http://127.0.0.1:1/v1/credentials"), + |_, builder| builder, + ) + .unwrap(); + let inner = + super::super::build_db_ctx_inner::(None, NativeTasks::new(websocket, parser), None, None, None); + let aborts = inner + .lock() + .unwrap() + .background_tasks + .try_lock() + .unwrap() + .abort_handles(); + let (outgoing, _outgoing_rx) = futures_channel::mpsc::unbounded(); + let (_incoming, incoming_rx) = futures_channel::mpsc::unbounded(); + let (pending, pending_rx) = futures_channel::mpsc::unbounded(); + let context = super::super::build_db_ctx( + tokio::runtime::Handle::current(), + inner, + outgoing, + Arc::new(tokio::sync::Mutex::new(incoming_rx)), + pending, + Arc::new(tokio::sync::Mutex::new(pending_rx)), + None, + None, + ); + session.current = Some(Current { + context, + info: ContainerSessionInfo { + generation: 1, + target: source(), + }, + validity: Validity { + expiry: SystemTime::now(), + deadline: Instant::now(), + }, + refresh_at: Instant::now(), + signal: Arc::new(Signal::default()), + connected_announced: false, + closing: None, + driver_finished: false, + aborts, + }); + { + let mut shutdown = Box::pin(session.shutdown_and_join()); + tokio::select! { result = &mut shutdown => panic!("shutdown completed before destructor release: {result:?}"), result = entry => result.unwrap(), } + } + assert!(session.current.is_some()); + release.send(()).unwrap(); + session.shutdown_and_join().await.unwrap(); + assert!(session.current.is_none()); + session.shutdown_and_join().await.unwrap(); + runtime.shutdown_background(); +} + +#[tokio::test] +async fn callback_panic_propagates_only_after_socket_and_native_tasks_close() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-panic", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, source()).await; + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| { + builder.on_connect(|_, _, _| panic!("injected application panic")) + }) + .unwrap(); + assert!(AssertUnwindSafe(session.run()).catch_unwind().await.is_err()); + assert!(session.current.is_none()); + assert!(session.stopped); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn observer_panic_during_rotation_still_joins_the_previous_connection() { + let (broker, broker_task) = fixture::http_fixture(vec![ + fixture::response(200, &fixture::token_body("disposable-old", 3)), + fixture::response(200, &fixture::token_body("disposable-new", 20)), + ]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, source()).await; + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| builder) + .unwrap() + .on_event(|event| { + if matches!(event, ContainerSessionEvent::Closing { .. }) { + panic!("injected observer panic"); + } + }); + assert!(AssertUnwindSafe(session.run()).catch_unwind().await.is_err()); + assert!(session.current.is_none()); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn unexpected_sender_is_not_exposed_to_application_callback() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-identity", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer(1, Identity::ZERO).await; + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| { + builder.on_connect(|_, _, _| panic!("wrong sender must never reach application")) + }) + .unwrap(); + assert_eq!( + session.run().await.unwrap_err(), + ContainerSessionError::IdentityMismatch + ); + assert!(session.current.is_none()); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn stalled_refresh_is_cancelled_at_expiry_and_never_extends_the_old_connection() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let broker = format!("http://{}/v1/credentials", listener.local_addr().unwrap()); + let broker_task = Task::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + socket + .write_all(&fixture::response(200, &fixture::token_body("disposable-expiring", 2))) + .await + .unwrap(); + drop(socket); + let (mut socket, _) = listener.accept().await.unwrap(); + fixture::read_request(&mut socket).await; + let mut byte = [0]; + assert_eq!( + timeout(Duration::from_secs(3), socket.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + }); + let (uri, _, peer_task) = peer(1, source()).await; + let (closed, receipt) = oneshot::channel(); + let mut closed = Some(closed); + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| builder) + .unwrap() + .on_event(move |event| { + if matches!( + event, + ContainerSessionEvent::Closing { + reason: ContainerSessionEndReason::CredentialExpiry, + .. + } + ) { + closed.take().unwrap().send(()).unwrap(); + } + }); + drive_until(&mut session, async { + receipt.await.unwrap(); + }) + .await; + if let Some(current) = &session.current { + assert!(!current.context.is_active()); + } + session.shutdown_and_join().await.unwrap(); + assert!(session.current.is_none()); + broker_task.join().await; + peer_task.join().await; +} + +#[tokio::test] +async fn factory_panic_leaves_no_connection_and_is_a_terminal_owner_failure() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-factory-panic", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let mut session = ContainerSession::new(config("https://must-not-contact.invalid", &broker), |_, _| { + panic!("injected factory panic") + }) + .unwrap(); + assert!(AssertUnwindSafe(session.run()).catch_unwind().await.is_err()); + assert_eq!(session.run().await.unwrap_err(), ContainerSessionError::Panicked); + assert!(session.current.is_none()); + broker_task.join().await; +} + +#[tokio::test] +async fn unexpected_sender_still_fails_when_initial_connection_is_followed_by_immediate_close() { + let (broker, broker_task) = fixture::http_fixture(vec![fixture::response( + 200, + &fixture::token_body("disposable-fast-close", 20), + )]) + .await; + let broker_task = Task(Some(broker_task)); + let (uri, _, peer_task) = peer_with_close(1, Identity::ZERO, true).await; + let mut session = ContainerSession::new(config(&uri, &broker), |_, builder| { + builder.on_connect(|_, _, _| panic!("wrong sender must never reach application")) + }) + .unwrap(); + assert_eq!( + timeout(Duration::from_secs(3), session.run()) + .await + .unwrap() + .unwrap_err(), + ContainerSessionError::IdentityMismatch + ); + assert!(session.current.is_none()); + broker_task.join().await; + peer_task.join().await; +} diff --git a/sdks/rust/src/db_connection/native_tasks.rs b/sdks/rust/src/db_connection/native_tasks.rs index 7671a5dbfda..bf27f3e4a2e 100644 --- a/sdks/rust/src/db_connection/native_tasks.rs +++ b/sdks/rust/src/db_connection/native_tasks.rs @@ -22,6 +22,14 @@ impl NativeTasks { } } + pub fn abort_handles(&self) -> Vec { + [&self.websocket, &self.parser] + .into_iter() + .flatten() + .map(JoinHandle::abort_handle) + .collect() + } + pub fn record_failure(&mut self, error: crate::Error) { // A cancelled terminal wait must not discard its original failure. self.failure.get_or_insert(error); diff --git a/sdks/rust/src/lib.rs b/sdks/rust/src/lib.rs index e08938b04f3..6d1decc56ae 100644 --- a/sdks/rust/src/lib.rs +++ b/sdks/rust/src/lib.rs @@ -28,6 +28,11 @@ pub mod error; pub mod event; pub mod table; +#[cfg(not(feature = "browser"))] +pub use db_connection::container_session::{ + ContainerSession, ContainerSessionEndReason, ContainerSessionError, ContainerSessionEvent, ContainerSessionInfo, + ContainerSessionRetryReason, OutstandingCallOutcomes, +}; pub use db_connection::DbConnectionBuilder; pub use db_context::DbContext; pub use error::{Error, Result}; From 0fe5d1c19d49a00f707d55abda181463ddc2bd13 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 04:15:28 -0400 Subject: [PATCH 12/16] Add managed Node container credential sessions --- crates/bindings-typescript/README.md | 56 ++ .../src/sdk/db_connection_impl.ts | 189 ++++- .../src/sdk/event_emitter.ts | 6 + .../src/sdk/managed_session_lifecycle.ts | 23 + .../src/sdk/node/container_session.ts | 621 +++++++++++++++++ .../bindings-typescript/src/sdk/node/index.ts | 9 + .../src/sdk/table_cache.ts | 6 + .../tests/node_container_session.test.ts | 653 ++++++++++++++++++ 8 files changed, 1549 insertions(+), 14 deletions(-) create mode 100644 crates/bindings-typescript/src/sdk/managed_session_lifecycle.ts create mode 100644 crates/bindings-typescript/src/sdk/node/container_session.ts create mode 100644 crates/bindings-typescript/tests/node_container_session.test.ts diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index 3ab18839fa6..68132fd9573 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -343,3 +343,59 @@ most 30 seconds. `expiresAt` and `remainingLifetimeMs` expose their remaining validity; logging or serializing the token object redacts its value. Rebuild the database connection with a fresh token and restore subscriptions on expiry or disconnection. This helper does not manage connection renewal or replay calls. + +### Managed container connections in Node.js + +`ContainerSession` owns credential renewal and successive connections. Pass the +unmodified generated `DbConnection` class so it can create a fresh builder, +cache, and subscriptions for every generation: + +```ts +import { Container, ContainerSession } from 'spacetimedb/sdk/node'; +import { DbConnection } from './module_bindings'; + +const session = new ContainerSession(DbConnection, { + container: Container.fromEnvironment(), + onConnect(connection, { generation }) { + connection.subscriptionBuilder().subscribe('SELECT * FROM jobs'); + console.log('Connected generation', generation); + }, +}); +const pause = new AbortController(); +process.once('SIGTERM', () => pause.abort()); +try { + await session.run(pause.signal); +} finally { + await session.shutdown(); +} +``` + +The owner pins the database Identity and server before requesting credentials. +Optional `target: Identity` and `serverUri` select another database and its +trusted server. It verifies the connecting sender against the container's +Identity before calling `onConnect`. Setup and event callbacks must be +synchronous. Application asynchronous work must have its own cancellation and +error handling. + +Renewal requests a fresh token before expiry. Both the returned expiry and its +monotonic lifetime must extend materially before the owner rotates connections; +the same lease expiry does not cause repeated rotations. A rejected credential +is terminal. Transient broker failures retry while the current credential +remains valid, and expiry seals the old generation even if a refresh stalls. +The old WebSocket is closed and joined before a new one is opened. This causes a +brief interruption and requires restoring subscriptions in every `onConnect`. + +Aborting `run(signal)` immediately seals that generation, then joins its broker +request and WebSocket before resolving. Calling `run` again resumes with fresh +credentials. `shutdown()` is permanent and also awaits cleanup. Concurrent +`run` calls are rejected. Keep and await the owner; discarding its promise does +not cancel it. + +No reducer or procedure is replayed. A pending call receives +`ContainerSessionCallError`: `outcome: 'not_sent'` means it was never handed to +the transport, while `outcome: 'unknown'` means no confirmed result was received +after handoff. A confirmed result wins a later shutdown. Generation +`disconnected` events also identify the interruption and warn about unconfirmed +calls. Retained old connections reject new calls as `not_sent`, reject new +subscriptions, and release their callbacks. Their cache is an old snapshot; +the next generation starts with a separate empty cache. diff --git a/crates/bindings-typescript/src/sdk/db_connection_impl.ts b/crates/bindings-typescript/src/sdk/db_connection_impl.ts index 12dfca7afbf..91acc700ed7 100644 --- a/crates/bindings-typescript/src/sdk/db_connection_impl.ts +++ b/crates/bindings-typescript/src/sdk/db_connection_impl.ts @@ -1,3 +1,9 @@ +import { + INTERNAL_MANAGED_SESSION, + INTERNAL_CLEAR_TABLE_CALLBACKS, + ContainerSessionCallError, + type ManagedSessionLifecycle, +} from './managed_session_lifecycle'; import { ConnectionId, ProductBuilder, ProductType } from '../'; import { AlgebraicType, type ComparablePrimitive } from '../'; import BinaryReader from '../lib/binary_reader.ts'; @@ -274,6 +280,79 @@ export class DbConnectionImpl >(); #reducerCallInfo = new Map(); #procedureCallbacks = new Map(); + #managedCalls?: Map< + number, + { reject: (error: Error) => void; sent: boolean } + >; + #managedQueuedCalls = new Map(); + #managedTerminalError?: Error; + + [INTERNAL_MANAGED_SESSION](): ManagedSessionLifecycle { + return { + enable: () => { + if ( + this.#managedCalls || + this.#managedTerminalError || + this.#outboundQueue.length || + this.#reducerCallbacks.size || + this.#procedureCallbacks.size || + this.#subscriptionManager.subscriptions.size + ) + throw new Error('Managed boundary requires an unused connection'); + this.#managedCalls = new Map(); + }, + seal: error => { + if (!this.#managedCalls || this.#managedTerminalError) return; + // Seal first, before any subscription callback can attempt another call. + this.#managedTerminalError = error; + this.isActive = false; + this.isDisconnectRequested = true; + this.token = undefined; + this.#outboundQueue.length = 0; + this.#inboundQueue.length = 0; + this.#inboundQueueOffset = 0; + this.#managedQueuedCalls.clear(); + const pending = [...this.#managedCalls.values()]; + this.#managedCalls.clear(); + this.#reducerCallbacks.clear(); + this.#procedureCallbacks.clear(); + this.#reducerCallInfo.clear(); + this.#emitter.clear(); + for (const table of this.clientCache.tables.values()) + table[INTERNAL_CLEAR_TABLE_CALLBACKS](); + const subscriptions = [ + ...this.#subscriptionManager.subscriptions.values(), + ]; + this.#subscriptionManager.subscriptions.clear(); + for (const call of pending) + call.reject( + new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent') + ); + let callbackFailed = false; + for (const { emitter } of subscriptions) { + try { + emitter.emit('error', this, error); + } catch { + callbackFailed = true; + } finally { + emitter.clear(); + } + } + if (callbackFailed) + throw new Error('Managed subscription callback failed'); + }, + }; + } + + #markManagedCallSent(message: Uint8Array): void { + const requestId = this.#managedQueuedCalls.get(message); + this.#managedQueuedCalls.delete(message); + if (requestId !== undefined) { + const call = this.#managedCalls?.get(requestId); + if (call) call.sent = true; + } + } + #rowDeserializers: Record>; #rowIdMetadata: Record< string, @@ -560,6 +639,7 @@ export class DbConnectionImpl >, querySql: string[] ): number { + if (this.#managedTerminalError) throw this.#managedTerminalError; const querySetId = this.#getNextQueryId(); this.#subscriptionManager.subscriptions.set(querySetId, { handle, @@ -715,6 +795,7 @@ export class DbConnectionImpl #flushOutboundQueueV2(wsResolved: WebSocketAdapter): void { const pending = this.#outboundQueue.splice(0); for (const message of pending) { + this.#markManagedCallSent(message); wsResolved.send(message); } } @@ -731,6 +812,8 @@ export class DbConnectionImpl this.#outboundQueue, MAX_V3_OUTBOUND_FRAME_BYTES ); + for (let index = 0; index < batchSize; index++) + this.#markManagedCallSent(this.#outboundQueue[index]); wsResolved.send( encodeClientMessagesV3( this.#clientFrameEncoder, @@ -786,35 +869,51 @@ export class DbConnectionImpl #clientMessageEncoder = new BinaryWriter(1024); #sendEncodedMessage( encoded: Uint8Array, - describe: () => string + describe: () => string, + managedRequestId?: number ): void { + if (this.#managedTerminalError) throw this.#managedTerminalError; stdbLogger('trace', describe); if (this.ws && this.isActive) { if (this.#negotiatedWsProtocol === V2_WS_PROTOCOL) { if (this.#outboundQueue.length) this.#flushOutboundQueue(this.ws); + const call = + managedRequestId === undefined + ? undefined + : this.#managedCalls?.get(managedRequestId); + if (call) call.sent = true; this.ws.send(encoded); return; } - this.#outboundQueue.push(encoded.slice()); + const queued = encoded.slice(); + this.#outboundQueue.push(queued); + if (this.#managedCalls && managedRequestId !== undefined) + this.#managedQueuedCalls.set(queued, managedRequestId); this.#scheduleOutboundFlush(); } else { // Use slice() to copy, in case the clientMessageEncoder's buffer gets reused // before the connection opens or before a v3 microbatch flush runs. - this.#outboundQueue.push(encoded.slice()); + const queued = encoded.slice(); + this.#outboundQueue.push(queued); + if (this.#managedCalls && managedRequestId !== undefined) + this.#managedQueuedCalls.set(queued, managedRequestId); } } - #sendMessage(message: ClientMessage): void { + #sendMessage(message: ClientMessage, managedRequestId?: number): void { const writer = this.#clientMessageEncoder; writer.clear(); ClientMessage.serialize(writer, message); const encoded = writer.getBuffer(); const isLive = !!(this.ws && this.isActive); - this.#sendEncodedMessage(encoded, () => - isLive - ? `Sending message to server: ${stringify(message)}` - : `Queuing message to server: ${stringify(message)}` + this.#sendEncodedMessage( + encoded, + () => + isLive + ? `Sending message to server: ${stringify(message)}` + : `Queuing message to server: ${stringify(message)}`, + managedRequestId ); } @@ -833,7 +932,8 @@ export class DbConnectionImpl const encoded = writer.getBuffer(); this.#sendEncodedMessage( encoded, - () => `Sending reducer call message to server: requestId=${requestId}` + () => `Sending reducer call message to server: requestId=${requestId}`, + requestId ); } @@ -852,7 +952,8 @@ export class DbConnectionImpl const encoded = writer.getBuffer(); this.#sendEncodedMessage( encoded, - () => `Sending procedure call message to server: requestId=${requestId}` + () => `Sending procedure call message to server: requestId=${requestId}`, + requestId ); } @@ -926,11 +1027,13 @@ export class DbConnectionImpl () => `Calling ${callbacks.length} triggered row callbacks` ); for (const callback of callbacks) { + if (this.#managedTerminalError) return; callback.cb(); } } #processServerMessage(serverMessage: ServerMessage): void { + if (this.#managedTerminalError) return; stdbLogger( 'trace', () => `Processing server message: ${stringify(serverMessage)}` @@ -1054,6 +1157,14 @@ export class DbConnectionImpl } case 'ReducerResult': { const { requestId, result } = serverMessage.value; + // A received result wins a later managed seal from a row callback. + // Promise reactions still run after this synchronous cache update. + if (this.#managedCalls?.has(requestId)) { + this.#managedCalls.delete(requestId); + const confirmed = this.#reducerCallbacks.get(requestId); + this.#reducerCallbacks.delete(requestId); + confirmed?.(result); + } if (result.tag === 'Ok') { const reducerInfo = this.#reducerCallInfo.get(requestId); @@ -1086,6 +1197,7 @@ export class DbConnectionImpl this.#reducerCallInfo.delete(requestId); const cb = this.#reducerCallbacks.get(requestId); this.#reducerCallbacks.delete(requestId); + this.#managedCalls?.delete(requestId); cb?.(result); break; } @@ -1097,6 +1209,7 @@ export class DbConnectionImpl : { tag: 'Err', value: status.value }; const cb = this.#procedureCallbacks.get(requestId); this.#procedureCallbacks.delete(requestId); + this.#managedCalls?.delete(requestId); cb?.(result); break; } @@ -1200,9 +1313,20 @@ export class DbConnectionImpl argsBuffer: Uint8Array, reducerArgs?: object ): Promise { + if (this.#managedTerminalError) + return Promise.reject(new ContainerSessionCallError('not_sent')); const { promise, resolve, reject } = createDeferred(); const requestId = this.#getNextRequestId(); - this.#sendCallReducerMessage(requestId, encodedReducerName, argsBuffer); + this.#managedCalls?.set(requestId, { reject, sent: false }); + try { + this.#sendCallReducerMessage(requestId, encodedReducerName, argsBuffer); + } catch (error) { + const call = this.#managedCalls?.get(requestId); + if (!call) throw error; + this.#managedCalls!.delete(requestId); + reject(new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent')); + return promise; + } if (reducerArgs) { this.#reducerCallInfo.set(requestId, { name: reducerName, @@ -1235,15 +1359,26 @@ export class DbConnectionImpl argsBuffer: Uint8Array, reducerArgs?: object ): Promise { + if (this.#managedTerminalError) + return Promise.reject(new ContainerSessionCallError('not_sent')); const { promise, resolve, reject } = createDeferred(); const requestId = this.#getNextRequestId(); + this.#managedCalls?.set(requestId, { reject, sent: false }); const message = ClientMessage.CallReducer({ reducer: reducerName, args: argsBuffer, requestId, flags: 0, }); - this.#sendMessage(message); + try { + this.#sendMessage(message, requestId); + } catch (error) { + const call = this.#managedCalls?.get(requestId); + if (!call) throw error; + this.#managedCalls!.delete(requestId); + reject(new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent')); + return promise; + } if (reducerArgs) { this.#reducerCallInfo.set(requestId, { name: reducerName, @@ -1316,9 +1451,24 @@ export class DbConnectionImpl encodedProcedureName: Uint8Array, argsBuffer: Uint8Array ): Promise { + if (this.#managedTerminalError) + return Promise.reject(new ContainerSessionCallError('not_sent')); const { promise, resolve, reject } = createDeferred(); const requestId = this.#getNextRequestId(); - this.#sendCallProcedureMessage(requestId, encodedProcedureName, argsBuffer); + this.#managedCalls?.set(requestId, { reject, sent: false }); + try { + this.#sendCallProcedureMessage( + requestId, + encodedProcedureName, + argsBuffer + ); + } catch (error) { + const call = this.#managedCalls?.get(requestId); + if (!call) throw error; + this.#managedCalls!.delete(requestId); + reject(new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent')); + return promise; + } this.#procedureCallbacks.set(requestId, result => { if (result.tag === 'Ok') { resolve(result.value); @@ -1333,8 +1483,11 @@ export class DbConnectionImpl procedureName: string, argsBuffer: Uint8Array ): Promise { + if (this.#managedTerminalError) + return Promise.reject(new ContainerSessionCallError('not_sent')); const { promise, resolve, reject } = createDeferred(); const requestId = this.#getNextRequestId(); + this.#managedCalls?.set(requestId, { reject, sent: false }); const message = ClientMessage.CallProcedure({ procedure: procedureName, args: argsBuffer, @@ -1342,7 +1495,15 @@ export class DbConnectionImpl // reserved for future use - 0 is the only valid value flags: 0, }); - this.#sendMessage(message); + try { + this.#sendMessage(message, requestId); + } catch (error) { + const call = this.#managedCalls?.get(requestId); + if (!call) throw error; + this.#managedCalls!.delete(requestId); + reject(new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent')); + return promise; + } this.#procedureCallbacks.set(requestId, result => { if (result.tag === 'Ok') { resolve(result.value); diff --git a/crates/bindings-typescript/src/sdk/event_emitter.ts b/crates/bindings-typescript/src/sdk/event_emitter.ts index a3447ce4b58..f823e9862d0 100644 --- a/crates/bindings-typescript/src/sdk/event_emitter.ts +++ b/crates/bindings-typescript/src/sdk/event_emitter.ts @@ -19,6 +19,12 @@ export class EventEmitter { callbacks.delete(callback); } + /** @internal Clear callbacks when an explicitly managed generation ends. */ + clear(): void { + for (const callbacks of this.#events.values()) callbacks.clear(); + this.#events.clear(); + } + emit(event: Key, ...args: any[]): void { const callbacks = this.#events.get(event); if (!callbacks) { diff --git a/crates/bindings-typescript/src/sdk/managed_session_lifecycle.ts b/crates/bindings-typescript/src/sdk/managed_session_lifecycle.ts new file mode 100644 index 00000000000..e41781120db --- /dev/null +++ b/crates/bindings-typescript/src/sdk/managed_session_lifecycle.ts @@ -0,0 +1,23 @@ +/** Internal opt-in boundary used only by managed container sessions. */ +export const INTERNAL_MANAGED_SESSION: unique symbol = + Symbol('managed session'); +export const INTERNAL_CLEAR_TABLE_CALLBACKS: unique symbol = Symbol( + 'clear table callbacks' +); + +/** A local terminal outcome, never a claim that the server rolled a call back. */ +export class ContainerSessionCallError extends Error { + constructor(readonly outcome: 'not_sent' | 'unknown') { + super( + outcome === 'not_sent' + ? 'Container session ended before this call was sent' + : 'Container session ended without a confirmed call result; outcome unknown' + ); + this.name = 'ContainerSessionCallError'; + } +} + +export interface ManagedSessionLifecycle { + enable(): void; + seal(error: Error): void; +} diff --git a/crates/bindings-typescript/src/sdk/node/container_session.ts b/crates/bindings-typescript/src/sdk/node/container_session.ts new file mode 100644 index 00000000000..59d09947cb2 --- /dev/null +++ b/crates/bindings-typescript/src/sdk/node/container_session.ts @@ -0,0 +1,621 @@ +import { performance } from 'node:perf_hooks'; +import { Identity } from '../../lib/identity'; +import type { DbConnectionBuilder } from '../db_connection_builder'; +import type { DbConnectionImpl } from '../db_connection_impl'; +import { + INTERNAL_MANAGED_SESSION, + type ManagedSessionLifecycle, +} from '../managed_session_lifecycle'; +import type { WebSocketAdapter } from '../ws'; +import { + Container, + ContainerCredentialError, + type ContainerToken, +} from './container'; +import { openNodeWebSocket, type NodeWebSocketAdapter } from './index'; + +export type ContainerSessionErrorCode = + | 'invalid_configuration' + | 'already_running' + | 'terminated' + | 'denied' + | 'credentials_failed' + | 'sender_mismatch' + | 'callback_failed' + | 'message_processing_failed'; +export class ContainerSessionError extends Error { + constructor(readonly code: ContainerSessionErrorCode) { + super(`Container session failed: ${code}`); + this.name = 'ContainerSessionError'; + } +} +export interface ContainerGeneration { + readonly generation: number; + readonly target: Identity; + readonly expiresAt: Date; +} +export type ContainerSessionEvent = + | { readonly type: 'connected'; readonly generation: number } + | { + readonly type: 'disconnected'; + readonly generation: number; + readonly reason: + | 'renewed' + | 'expired' + | 'connection_closed' + | 'paused' + | 'shutdown' + | 'failed'; + readonly unconfirmedCalls: 'unknown'; + } + | { readonly type: 'retrying_credentials' }; +export interface ContainerSessionOptions> { + readonly container: Container; + readonly target?: Identity; + readonly serverUri?: string; + /** Synchronous setup, called after sender verification for every fresh cache. */ + readonly onConnect: (connection: C, generation: ContainerGeneration) => void; + readonly onEvent?: (event: ContainerSessionEvent) => void; + readonly compression?: 'gzip' | 'brotli' | 'none'; + readonly lightMode?: boolean; + readonly confirmedReads?: boolean; +} +type EndReason = Extract< + ContainerSessionEvent, + { type: 'disconnected' } +>['reason']; + +class Wake { + #listeners = new Set<() => void>(); + notify(): void { + for (const listener of this.#listeners) listener(); + } + async wait(milliseconds: number, signal: AbortSignal): Promise { + if (signal.aborted) return; + await new Promise(resolve => { + const done = (): void => { + clearTimeout(timer); + this.#listeners.delete(done); + signal.removeEventListener('abort', done); + resolve(); + }; + const timer = setTimeout(done, Math.max(0, Math.min(milliseconds, 250))); + this.#listeners.add(done); + signal.addEventListener('abort', done, { once: true }); + if (signal.aborted) done(); + }); + } +} + +/** Drops all late decompression/message callbacks once the generation is sealed. */ +class ManagedSocket implements WebSocketAdapter { + #sealed = false; + #failure?: () => void; + #closed?: () => void; + constructor( + private adapter: NodeWebSocketAdapter, + failure: () => void, + closed: () => void + ) { + this.#failure = failure; + this.#closed = closed; + } + get protocol(): string { + return this.adapter.protocol; + } + get readyState(): number { + return this.adapter.readyState; + } + send(bytes: Uint8Array): void { + if (this.#sealed) throw new ContainerSessionError('terminated'); + this.adapter.send(bytes); + } + seal(): void { + this.#sealed = true; + } + close(): void { + this.seal(); + this.adapter.close(); + } + async shutdown(): Promise { + this.seal(); + await this.adapter.shutdown(); + this.#failure = undefined; + this.#closed = undefined; + this.adapter.onmessage = () => {}; + this.adapter.onopen = () => {}; + this.adapter.onerror = () => {}; + this.adapter.onclose = () => {}; + } + set onopen(handler: () => void) { + this.adapter.onopen = () => { + if (!this.#sealed) { + try { + handler(); + } catch { + this.#failure?.(); + } + } + }; + } + set onmessage(handler: (message: { data: Uint8Array }) => void) { + this.adapter.onmessage = message => { + if (!this.#sealed) { + try { + handler(message); + } catch { + this.#failure?.(); + } + } + }; + } + set onerror(handler: (event: ErrorEvent) => void) { + this.adapter.onerror = event => { + if (!this.#sealed) { + try { + handler(event); + } catch { + this.#failure?.(); + } + } + }; + } + set onclose(handler: (event: CloseEvent) => void) { + this.adapter.onclose = event => { + this.seal(); + this.#closed?.(); + try { + handler(event); + } catch { + this.#failure?.(); + } + }; + } +} +interface Generation { + number: number; + token: ContainerToken; + connection?: C; + lifecycle?: ManagedSessionLifecycle; + socket?: ManagedSocket; + socketPromise?: Promise; + wake: Wake; + connected: boolean; + closed: boolean; + sealed: boolean; + failure?: ContainerSessionError; +} + +/** + * An explicit owner of sequential Node database connections and credential renewal. + * Pass the unmodified generated DbConnection class. Each generation gets its own + * builder, callbacks, subscriptions and cache. run(signal) pauses by closing and + * joining the current request/socket before resolving; a later run resumes with + * fresh credentials. shutdown() is terminal. Always await one of these owners. + * No reducer or procedure is replayed, and closure does not imply rollback. + */ +export class ContainerSession> { + #builder: () => DbConnectionBuilder; + #options: ContainerSessionOptions; + #target: string; + #sender: string; + #server: string; + #running?: Promise; + #abort?: AbortController; + #terminal = false; + #nextGeneration = 0; + #active?: Generation; + #builders = new WeakSet>(); + #wake = new Wake(); + + constructor( + connectionType: { builder(): DbConnectionBuilder }, + options: ContainerSessionOptions + ) { + try { + this.#builder = connectionType.builder.bind(connectionType); + this.#options = { ...options }; + this.#target = (options.target ?? options.container.databaseIdentity) + .toHexString() + .toLowerCase(); + this.#sender = options.container.databaseIdentity + .toHexString() + .toLowerCase(); + if ( + !/^[0-9a-f]{64}$/.test(this.#target) || + !/^[0-9a-f]{64}$/.test(this.#sender) || + typeof options.onConnect !== 'function' + ) + throw new Error(); + const value = options.serverUri ?? options.container.serverUri; + if ( + !value || + Buffer.byteLength(value) > 4096 || + [...value].some( + character => + character.charCodeAt(0) <= 32 || + character.charCodeAt(0) === 127 || + character === '?' || + character === '#' + ) + ) + throw new Error(); + const url = new URL(value); + if ( + !['http:', 'https:', 'ws:', 'wss:'].includes(url.protocol) || + !url.hostname || + url.username || + url.password + ) + throw new Error(); + url.protocol = ['https:', 'wss:'].includes(url.protocol) ? 'wss:' : 'ws:'; + this.#server = url.href; + } catch { + throw new ContainerSessionError('invalid_configuration'); + } + } + + run(signal?: AbortSignal): Promise { + if (this.#terminal) + return Promise.reject(new ContainerSessionError('terminated')); + if (this.#running) + return Promise.reject(new ContainerSessionError('already_running')); + const abort = new AbortController(); + this.#abort = abort; + const pause = (): void => { + if (this.#active) this.#seal(this.#active); + abort.abort(); + }; + signal?.addEventListener('abort', pause, { once: true }); + if (signal?.aborted) pause(); + const running = this.#drive(abort.signal) + .catch(error => { + this.#terminal = true; + throw error instanceof ContainerSessionError + ? error + : new ContainerSessionError('credentials_failed'); + }) + .finally(() => { + signal?.removeEventListener('abort', pause); + this.#abort = undefined; + this.#running = undefined; + }); + this.#running = running; + return running; + } + + async shutdown(): Promise { + this.#terminal = true; + if (this.#active) this.#seal(this.#active); + this.#abort?.abort(); + // run owns and reports its failure. shutdown still positively joins cleanup. + await this.#running?.catch(() => {}); + } + + #invoke(callback: (() => void) | undefined): void { + if (!callback) return; + try { + const result: unknown = callback(); + if ( + result && + typeof (result as PromiseLike).then === 'function' + ) { + // Async application work cannot be cancelled or joined by the SDK. + void Promise.resolve(result).catch(() => {}); + throw new Error(); + } + } catch { + throw new ContainerSessionError('callback_failed'); + } + } + #event(event: ContainerSessionEvent): void { + this.#invoke( + this.#options.onEvent && + (() => this.#options.onEvent!(Object.freeze(event))) + ); + } + #seal(generation: Generation): void { + if (generation.sealed) return; + generation.sealed = true; + generation.socket?.seal(); + try { + generation.lifecycle?.seal(new ContainerSessionError('terminated')); + } catch { + generation.failure ??= new ContainerSessionError('callback_failed'); + } + generation.wake.notify(); + } + async #close(generation: Generation, reason: EndReason): Promise { + this.#seal(generation); + generation.connection?.disconnect(); + const socket = + generation.socket ?? + (await generation.socketPromise?.catch(() => undefined)); + await socket?.shutdown(); + if (this.#active === generation) this.#active = undefined; + this.#event({ + type: 'disconnected', + generation: generation.number, + reason, + unconfirmedCalls: 'unknown', + }); + } + + async #open( + token: ContainerToken, + signal: AbortSignal + ): Promise> { + const generation: Generation = { + number: ++this.#nextGeneration, + token, + wake: new Wake(), + connected: false, + closed: false, + sealed: false, + }; + this.#active = generation; + const fail = (code: ContainerSessionErrorCode): void => { + generation.failure ??= new ContainerSessionError(code); + this.#seal(generation); + }; + try { + const builder = this.#builder(); + if (this.#builders.has(builder)) + throw new ContainerSessionError('invalid_configuration'); + this.#builders.add(builder); + const connection = builder + .withUri(this.#server) + .withDatabaseName(this.#target) + .withToken(token.value) + .withCompression(this.#options.compression ?? 'gzip') + .withLightMode(this.#options.lightMode ?? false) + .withWSFn(args => { + const server = new URL(args.url); + server.searchParams.delete('connection_id'); + if ( + server.href !== this.#server || + args.nameOrAddress !== this.#target || + args.authToken !== token.value + ) + throw new ContainerSessionError('invalid_configuration'); + const promise = openNodeWebSocket(args).then(adapter => { + const socket = new ManagedSocket( + adapter, + () => fail('message_processing_failed'), + () => { + generation.closed = true; + generation.wake.notify(); + } + ); + generation.socket = socket; + if (generation.sealed || signal.aborted) socket.close(); + return socket; + }); + generation.socketPromise = promise; + return promise; + }) + .onConnect((connected, identity) => { + if ( + generation.sealed || + signal.aborted || + token.remainingLifetimeMs <= 0 + ) { + this.#seal(generation); + return; + } + if (identity.toHexString().toLowerCase() !== this.#sender) { + fail('sender_mismatch'); + return; + } + if (generation.connected) { + fail('message_processing_failed'); + return; + } + generation.connected = true; + try { + this.#invoke(() => + this.#options.onConnect( + connected, + Object.freeze({ + generation: generation.number, + target: new Identity(this.#target), + expiresAt: token.expiresAt, + }) + ) + ); + if (!generation.sealed) + this.#event({ type: 'connected', generation: generation.number }); + } catch { + fail('callback_failed'); + } + generation.wake.notify(); + }) + .onConnectError(() => { + generation.closed = true; + generation.wake.notify(); + }) + .onDisconnect(() => { + generation.closed = true; + generation.wake.notify(); + }); + if (this.#options.confirmedReads !== undefined) + connection.withConfirmedReads(this.#options.confirmedReads); + generation.connection = connection.build(); + generation.lifecycle = generation.connection[INTERNAL_MANAGED_SESSION](); + generation.lifecycle.enable(); + return generation; + } catch (error) { + await this.#close(generation, 'failed'); + throw error instanceof ContainerSessionError + ? error + : new ContainerSessionError('invalid_configuration'); + } + } + + async #request( + signal: AbortSignal, + generation?: Generation + ): Promise { + if (signal.aborted) return undefined; + const abort = new AbortController(); + const cancel = (): void => abort.abort(); + signal.addEventListener('abort', cancel, { once: true }); + const wake = generation?.wake ?? this.#wake; + let done = false; + let value: ContainerToken | undefined; + let failure: unknown; + const pending = this.#options.container + .tokenFor(new Identity(this.#target), abort.signal) + .then( + token => { + value = token; + }, + error => { + failure = error; + } + ) + .finally(() => { + done = true; + wake.notify(); + }); + try { + while (!done) { + if ( + signal.aborted || + generation?.closed || + generation?.failure || + (generation && generation.token.remainingLifetimeMs <= 0) + ) { + if (generation) this.#seal(generation); + abort.abort(); + await pending; + return undefined; + } + await wake.wait(250, signal); + } + if (signal.aborted) return undefined; + if (failure) throw failure; + if (!value || value.target.toHexString() !== this.#target) + throw new ContainerSessionError('credentials_failed'); + return value; + } finally { + signal.removeEventListener('abort', cancel); + abort.abort(); + await pending; + } + } + + #transient(error: unknown): boolean { + return ( + error instanceof ContainerCredentialError && + ['transport', 'timeout', 'unavailable'].includes(error.code) + ); + } + #credentialFailure(error: unknown): ContainerSessionError { + return error instanceof ContainerCredentialError && error.code === 'denied' + ? new ContainerSessionError('denied') + : new ContainerSessionError('credentials_failed'); + } + async #delay( + milliseconds: number, + signal: AbortSignal, + generation?: Generation + ): Promise { + const until = performance.now() + milliseconds; + while ( + !signal.aborted && + performance.now() < until && + !generation?.closed && + !generation?.failure && + !(generation && generation.token.remainingLifetimeMs <= 0) + ) { + await (generation?.wake ?? this.#wake).wait( + until - performance.now(), + signal + ); + } + } + async #drive(signal: AbortSignal): Promise { + let next: ContainerToken | undefined; + while (!signal.aborted) { + if (!next || next.remainingLifetimeMs <= 0) { + try { + next = await this.#request(signal); + } catch (error) { + if (!this.#transient(error)) throw this.#credentialFailure(error); + this.#event({ type: 'retrying_credentials' }); + await this.#delay(500, signal); + continue; + } + } + if (!next || signal.aborted) break; + const generation = await this.#open(next, signal); + next = undefined; + let reason: EndReason = 'connection_closed'; + try { + const openingDeadline = performance.now() + 5000; + while ( + !generation.connected && + !generation.closed && + !generation.failure && + !signal.aborted && + generation.token.remainingLifetimeMs > 0 && + performance.now() < openingDeadline + ) + await generation.wake.wait(250, signal); + if (generation.failure) throw generation.failure; + if (generation.connected) { + let wait = Math.max( + 0, + generation.token.remainingLifetimeMs - + Math.min(5000, generation.token.remainingLifetimeMs / 3) + ); + while ( + !signal.aborted && + !generation.closed && + !generation.failure && + generation.token.remainingLifetimeMs > 0 + ) { + await this.#delay(wait, signal, generation); + if ( + signal.aborted || + generation.closed || + generation.failure || + generation.token.remainingLifetimeMs <= 0 + ) + break; + let refreshed: ContainerToken | undefined; + try { + refreshed = await this.#request(signal, generation); + } catch (error) { + if (!this.#transient(error)) throw this.#credentialFailure(error); + this.#event({ type: 'retrying_credentials' }); + } + if ( + refreshed && + refreshed.expiresAt.getTime() >= + generation.token.expiresAt.getTime() + 1000 && + refreshed.remainingLifetimeMs >= + generation.token.remainingLifetimeMs + 1000 + ) { + next = refreshed; + reason = 'renewed'; + break; + } + wait = 500; + } + } + if (generation.failure) throw generation.failure; + if (signal.aborted) reason = this.#terminal ? 'shutdown' : 'paused'; + else if (generation.token.remainingLifetimeMs <= 0) reason = 'expired'; + } catch (error) { + reason = 'failed'; + throw error; + } finally { + await this.#close(generation, reason); + } + if (generation.failure) throw generation.failure; + if (!next && !signal.aborted) await this.#delay(250, signal); + } + } +} diff --git a/crates/bindings-typescript/src/sdk/node/index.ts b/crates/bindings-typescript/src/sdk/node/index.ts index f9cf9df2a05..7a7390b2bbe 100644 --- a/crates/bindings-typescript/src/sdk/node/index.ts +++ b/crates/bindings-typescript/src/sdk/node/index.ts @@ -155,3 +155,12 @@ export type { ContainerDiscovery, ContainerCredentialErrorCode, } from './container'; + +export { ContainerSession, ContainerSessionError } from './container_session'; +export type { + ContainerGeneration, + ContainerSessionOptions, + ContainerSessionEvent, + ContainerSessionErrorCode, +} from './container_session'; +export { ContainerSessionCallError } from '../managed_session_lifecycle'; diff --git a/crates/bindings-typescript/src/sdk/table_cache.ts b/crates/bindings-typescript/src/sdk/table_cache.ts index 543d3fb78ed..6b68f2c710c 100644 --- a/crates/bindings-typescript/src/sdk/table_cache.ts +++ b/crates/bindings-typescript/src/sdk/table_cache.ts @@ -1,3 +1,4 @@ +import { INTERNAL_CLEAR_TABLE_CALLBACKS } from './managed_session_lifecycle'; import { EventEmitter } from './event_emitter.ts'; import { stdbLogger } from './logger.ts'; @@ -219,6 +220,11 @@ export class TableCacheImpl< } } + /** @internal Release callbacks when an explicitly managed generation ends. */ + [INTERNAL_CLEAR_TABLE_CALLBACKS](): void { + this.emitter.clear(); + } + /** * @returns number of rows in the table */ diff --git a/crates/bindings-typescript/tests/node_container_session.test.ts b/crates/bindings-typescript/tests/node_container_session.test.ts new file mode 100644 index 00000000000..4c117db467d --- /dev/null +++ b/crates/bindings-typescript/tests/node_container_session.test.ts @@ -0,0 +1,653 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { createServer, type Server, type ServerResponse } from 'node:http'; +import { once } from 'node:events'; +import type { AddressInfo, Socket } from 'node:net'; +import { WebSocketServer, WebSocket } from 'ws'; +import { + BinaryReader, + BinaryWriter, + ConnectionId, + Identity, + Timestamp, +} from '../src'; +import { + Container, + ContainerSession, + ContainerSessionError, + ContainerSessionCallError, +} from '../src/sdk/node'; +import { ClientMessage, ServerMessage } from '../src/sdk/client_api/types'; +import { INTERNAL_MANAGED_SESSION } from '../src/sdk/managed_session_lifecycle'; +import WebsocketTestAdapter from '../src/sdk/websocket_test_adapter'; +import { V2_WS_PROTOCOL, V3_WS_PROTOCOL } from '../src/sdk/websocket_protocols'; +import { + decodeClientMessagesV3, + encodeServerMessagesV3, +} from '../src/sdk/websocket_v3_frames'; +import { DbConnection } from '../test-app/src/module_bindings'; +import { encodePlayer, makeQueryRows, makeQuerySetUpdate } from './utils'; + +const OWN = '1'.repeat(64); +const TARGET = '2'.repeat(64); +const sessions: ContainerSession[] = []; +const fixtures: Fixture[] = []; +class Deferred { + resolve!: (value: T) => void; + reject!: (reason?: unknown) => void; + promise = new Promise((resolve, reject) => { + this.resolve = resolve; + this.reject = reject; + }); +} +function send(socket: WebSocket, message: ServerMessage): void { + const writer = new BinaryWriter(1024); + ServerMessage.serialize(writer, message); + const payload = + socket.protocol === V3_WS_PROTOCOL + ? encodeServerMessagesV3(new BinaryWriter(1024), [writer.getBuffer()]) + : writer.getBuffer(); + socket.send(Buffer.concat([Buffer.from([0]), payload])); +} +function credential( + response: ServerResponse, + expiry: number, + token = 'fixture-session-token' +): void { + const body = JSON.stringify({ token, expires_unix_seconds: expiry }); + response + .writeHead(200, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'Content-Length': Buffer.byteLength(body), + }) + .end(body); +} +class Fixture { + server: Server; + ws = new WebSocketServer({ + noServer: true, + handleProtocols: () => this.protocol, + }); + protocol: string = V2_WS_PROTOCOL; + sockets = new Set(); + peerClosures: Promise[] = []; + live = 0; + maxLive = 0; + connections = 0; + requests = 0; + calls: { generation: number; message: ClientMessage }[] = []; + routes: string[] = []; + targets: string[] = []; + serverUri = ''; + broker = ''; + source!: Container; + onRequest: (response: ServerResponse, count: number) => void = response => + credential(response, Math.floor(Date.now() / 1000) + 15); + onConnection?: (socket: WebSocket, generation: number) => void; + onMessage?: ( + socket: WebSocket, + message: ClientMessage, + generation: number + ) => void; + sender = OWN; + initial = true; + constructor() { + this.server = createServer((request, response) => { + if (request.url !== '/v1/credentials') { + response.writeHead(500).end(); + return; + } + let body = ''; + request.on('data', data => { + body += data; + }); + request.on('end', () => { + this.targets.push(JSON.parse(body).target_database); + this.onRequest(response, ++this.requests); + }); + }); + this.server.on('connection', socket => { + this.sockets.add(socket); + socket.once('close', () => this.sockets.delete(socket)); + }); + this.server.on('upgrade', (request, socket, head) => { + this.routes.push(request.url!); + this.ws.handleUpgrade(request, socket, head, client => { + const generation = ++this.connections; + this.live++; + this.maxLive = Math.max(this.live, this.maxLive); + this.peerClosures.push( + new Promise(resolve => + client.once('close', () => { + this.live--; + resolve(); + }) + ) + ); + client.on('error', () => {}); + client.on('message', bytes => { + const data = new Uint8Array(bytes as Buffer); + const messages = + this.protocol === V3_WS_PROTOCOL + ? decodeClientMessagesV3(data) + : [data]; + for (const bytes of messages) { + const message = ClientMessage.deserialize(new BinaryReader(bytes)); + this.calls.push({ generation, message }); + this.onMessage?.(client, message, generation); + if (message.tag === 'Subscribe') { + send( + client, + ServerMessage.SubscribeApplied({ + requestId: message.value.requestId, + querySetId: message.value.querySetId, + rows: makeQueryRows( + 'player', + encodePlayer({ + id: generation, + userId: new Identity(OWN), + name: `generation-${generation}`, + location: { x: 0, y: 0 }, + }) + ), + }) + ); + } + } + }); + if (this.initial) { + const id = new URL(request.url!, 'http://127.0.0.1').searchParams.get( + 'connection_id' + )!; + send( + client, + ServerMessage.InitialConnection({ + identity: new Identity(this.sender), + token: '', + connectionId: new ConnectionId(BigInt('0x' + id)), + }) + ); + } + this.onConnection?.(client, generation); + }); + }); + } + async start(): Promise { + this.server.listen(0, '127.0.0.1'); + await once(this.server, 'listening'); + this.serverUri = `http://127.0.0.1:${(this.server.address() as AddressInfo).port}/`; + this.broker = this.serverUri + 'v1/credentials'; + this.source = new Container({ + databaseIdentity: OWN, + serverUri: this.serverUri, + credentialBroker: this.broker, + }); + fixtures.push(this); + return this; + } + async joinPeers(): Promise { + await Promise.all(this.peerClosures); + } + async close(): Promise { + for (const socket of this.sockets) socket.destroy(); + await new Promise(resolve => this.ws.close(() => resolve())); + await new Promise((resolve, reject) => + this.server.close(error => (error ? reject(error) : resolve())) + ); + } +} +afterEach(async () => { + await Promise.all(sessions.splice(0).map(session => session.shutdown())); + await Promise.all(fixtures.splice(0).map(fixture => fixture.close())); +}); +function own( + fixture: Fixture, + onConnect: (connection: DbConnection, info: { generation: number }) => void, + onEvent?: ConstructorParameters< + typeof ContainerSession + >[1]['onEvent'] +): ContainerSession { + const session = new ContainerSession(DbConnection, { + container: fixture.source, + target: new Identity(TARGET), + onConnect, + onEvent, + compression: 'none', + }); + sessions.push(session); + return session; +} + +describe('managed Node container sessions', () => { + it('renews with a fresh cache/subscription and no overlapping sockets or replay', async () => { + const fixture = await new Fixture().start(); + fixture.onRequest = (response, count) => + credential( + response, + Math.floor(Date.now() / 1000) + (count === 1 ? 3 : 15) + ); + const renewed = new Deferred(); + const connections: DbConnection[] = []; + const rows: number[][] = []; + let uncertain: Promise | undefined; + const session = own(fixture, (connection, info) => { + connections.push(connection); + expect(connection.db.player.count()).toBe(0n); + connection + .subscriptionBuilder() + .onApplied(() => { + rows.push([...connection.db.player.iter()].map(row => row.id)); + if (info.generation === 2) renewed.resolve(); + }) + .subscribe('SELECT * FROM player'); + if (info.generation === 1) + uncertain = connection.reducers + .createPlayer({ name: 'once', location: { x: 0, y: 0 } }) + .catch(error => error); + }); + const run = session.run(); + await renewed.promise; + expect(await uncertain).toMatchObject({ outcome: 'unknown' }); + expect(rows).toEqual([[1], [2]]); + expect(fixture.maxLive).toBe(1); + expect( + fixture.calls.filter(call => call.message.tag === 'CallReducer') + ).toHaveLength(1); + expect(fixture.targets.every(target => target === TARGET)).toBe(true); + expect( + fixture.routes.every(route => + route.startsWith(`/v1/database/${TARGET}/subscribe?`) + ) + ).toBe(true); + await expect( + connections[0].reducers.createPlayer({ + name: 'late', + location: { x: 0, y: 0 }, + }) + ).rejects.toMatchObject({ outcome: 'not_sent' }); + await expect( + connections[0].callProcedure('late', new Uint8Array()) + ).rejects.toMatchObject({ outcome: 'not_sent' }); + expect(() => + connections[0].subscriptionBuilder().subscribe('SELECT * FROM player') + ).toThrow(ContainerSessionError); + await session.shutdown(); + await run; + }, 10000); + + it('does not rotate repeatedly on the same lease expiry and denial is terminal', async () => { + const fixture = await new Fixture().start(); + const expiry = Math.floor(Date.now() / 1000) + 4; + fixture.onRequest = (response, count) => + count < 3 + ? credential(response, expiry) + : response.writeHead(403, { 'Content-Length': 0 }).end(); + let connected = 0; + const session = own(fixture, () => { + connected++; + }); + await expect(session.run()).rejects.toMatchObject({ code: 'denied' }); + expect(fixture.requests).toBe(3); + expect(connected).toBe(1); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + await expect(session.run()).rejects.toMatchObject({ code: 'terminated' }); + }, 10000); + + it('expires and closes the old socket while a refresh response is stalled', async () => { + const fixture = await new Fixture().start(); + fixture.onRequest = (response, count) => { + if (count === 1) credential(response, Math.floor(Date.now() / 1000) + 3); + }; + const expired = new Deferred(); + const pause = new AbortController(); + const session = own( + fixture, + () => {}, + event => { + if (event.type === 'disconnected' && event.reason === 'expired') { + expired.resolve(); + pause.abort(); + } + } + ); + const run = session.run(pause.signal); + await expired.promise; + await run; + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + expect(fixture.connections).toBe(1); + }, 10000); + + it('joins cancellation and can resume with another fresh generation', async () => { + const fixture = await new Fixture().start(); + let pause = new AbortController(); + const session = own(fixture, () => pause.abort()); + await session.run(pause.signal); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + pause = new AbortController(); + await session.run(pause.signal); + expect(fixture.connections).toBe(2); + expect(fixture.requests).toBe(2); + expect(fixture.maxLive).toBe(1); + }); + + it('rejects a wrong sender before exposing onConnect, including immediate close', async () => { + for (const immediateClose of [false, true]) { + const fixture = await new Fixture().start(); + fixture.sender = TARGET; + if (immediateClose) fixture.onConnection = socket => socket.close(); + let exposed = 0; + const session = own(fixture, () => { + exposed++; + }); + await expect(session.run()).rejects.toMatchObject({ + code: 'sender_mismatch', + }); + expect(exposed).toBe(0); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + } + }); + + it('keeps a confirmed result when a later seal wins other pending calls', async () => { + const fixture = await new Fixture().start(); + const result = new Deferred(); + fixture.onMessage = (socket, message) => { + if (message.tag === 'CallReducer') + send( + socket, + ServerMessage.ReducerResult({ + requestId: message.value.requestId, + timestamp: new Timestamp(0n), + result: { tag: 'OkEmpty' }, + }) + ); + }; + const pause = new AbortController(); + const session = own(fixture, connection => { + void connection.reducers + .createPlayer({ name: 'confirmed', location: { x: 0, y: 0 } }) + .then( + () => { + result.resolve(); + pause.abort(); + }, + error => result.reject(error) + ); + }); + const run = session.run(pause.signal); + await result.promise; + await run; + expect( + fixture.calls.filter(call => call.message.tag === 'CallReducer') + ).toHaveLength(1); + }); + + it('rejects a queued managed call as not sent before socket readiness', async () => { + const fixture = await new Fixture().start(); + const ready = new Deferred(); + const session = own(fixture, connection => ready.resolve(connection)); + const run = session.run(); + const connection = await ready.promise; + connection.isActive = false; + const queued = connection.reducers + .createPlayer({ name: 'queued', location: { x: 0, y: 0 } }) + .catch(error => error); + await session.shutdown(); + await run; + expect(await queued).toMatchObject({ outcome: 'not_sent' }); + expect( + fixture.calls.filter(call => call.message.tag === 'CallReducer') + ).toHaveLength(0); + }); + + it('seals before subscription callbacks and joins even when a callback throws', async () => { + const fixture = await new Fixture().start(); + const applied = new Deferred(); + let oldCall: Promise | undefined; + let laterSubscriptionErrors = 0; + const session = own(fixture, connection => { + connection + .subscriptionBuilder() + .onApplied(() => applied.resolve()) + .onError(() => { + oldCall = connection.reducers + .createPlayer({ name: 'after-seal', location: { x: 0, y: 0 } }) + .catch(error => error); + throw new Error('application-private-error'); + }) + .subscribe('SELECT * FROM player'); + connection + .subscriptionBuilder() + .onError(() => laterSubscriptionErrors++) + .subscribe('SELECT * FROM player'); + }); + const run = session.run().catch(error => error); + await applied.promise; + await session.shutdown(); + expect(await run).toMatchObject({ code: 'callback_failed' }); + expect(await oldCall).toBeInstanceOf(ContainerSessionCallError); + expect(await oldCall).toMatchObject({ outcome: 'not_sent' }); + expect(laterSubscriptionErrors).toBe(1); + await session.shutdown(); + expect(laterSubscriptionErrors).toBe(1); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + }); + + it('joins the socket before reporting an application callback failure', async () => { + const fixture = await new Fixture().start(); + const session = own(fixture, () => { + throw new Error('private app failure'); + }); + await expect(session.run()).rejects.toMatchObject({ + code: 'callback_failed', + }); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + }); + + it('cancels an in-flight initial credential request and owns its socket until close', async () => { + const fixture = await new Fixture().start(); + const received = new Deferred(); + let brokerClosed: Promise | undefined; + fixture.onRequest = response => { + brokerClosed = once(response.socket!, 'close'); + received.resolve(); + }; + const pause = new AbortController(); + const session = own(fixture, () => { + throw new Error('must not connect'); + }); + const run = session.run(pause.signal); + await received.promise; + pause.abort(); + await run; + await brokerClosed; + expect(fixture.connections).toBe(0); + expect(fixture.requests).toBe(1); + }); + + it('classifies v3 batched calls as unsent when shutdown seals before the flush', async () => { + const fixture = new Fixture(); + fixture.protocol = V3_WS_PROTOCOL; + await fixture.start(); + const pause = new AbortController(); + let reducer: Promise | undefined; + let procedure: Promise | undefined; + const session = own(fixture, connection => { + reducer = connection.reducers + .createPlayer({ name: 'queued-v3', location: { x: 0, y: 0 } }) + .catch(error => error); + procedure = connection + .callProcedure('queued_procedure', new Uint8Array()) + .catch(error => error); + pause.abort(); + }); + await session.run(pause.signal); + await fixture.joinPeers(); + expect(await reducer).toMatchObject({ outcome: 'not_sent' }); + expect(await procedure).toMatchObject({ outcome: 'not_sent' }); + expect(fixture.calls).toHaveLength(0); + }); + + it('classifies v3 reducer/procedure handoffs without results as unknown', async () => { + const fixture = new Fixture(); + fixture.protocol = V3_WS_PROTOCOL; + await fixture.start(); + const received = new Deferred(); + fixture.onMessage = () => { + if (fixture.calls.length === 2) received.resolve(); + }; + let reducer: Promise | undefined; + let procedure: Promise | undefined; + const session = own(fixture, connection => { + reducer = connection.reducers + .createPlayer({ name: 'sent-v3', location: { x: 0, y: 0 } }) + .catch(error => error); + procedure = connection + .callProcedure('sent_procedure', new Uint8Array()) + .catch(error => error); + }); + const run = session.run(); + await received.promise; + await session.shutdown(); + await run; + await fixture.joinPeers(); + expect(await reducer).toMatchObject({ outcome: 'unknown' }); + expect(await procedure).toMatchObject({ outcome: 'unknown' }); + expect(fixture.calls).toHaveLength(2); + }); + + it('reports unsupported async setup after joining, without an unhandled rejection', async () => { + const fixture = await new Fixture().start(); + const session = own(fixture, async () => { + await Promise.resolve(); + throw new Error('private async callback'); + }); + await expect(session.run()).rejects.toMatchObject({ + code: 'callback_failed', + }); + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + }); + + it('keeps a confirmed reducer result when its row callback initiates shutdown', async () => { + const fixture = await new Fixture().start(); + const confirmed = new Deferred(); + let querySetId = 0; + fixture.onMessage = (socket, message) => { + if (message.tag === 'Subscribe') querySetId = message.value.querySetId.id; + if (message.tag === 'CallReducer') + send( + socket, + ServerMessage.ReducerResult({ + requestId: message.value.requestId, + timestamp: new Timestamp(0n), + result: { + tag: 'Ok', + value: { + retValue: new Uint8Array(), + transactionUpdate: { + querySets: [ + makeQuerySetUpdate( + querySetId, + 'player', + encodePlayer({ + id: 99, + userId: new Identity(OWN), + name: 'confirmed', + location: { x: 0, y: 0 }, + }) + ), + ], + }, + }, + }, + }) + ); + }; + const session = own(fixture, connection => { + connection.db.player.onInsert(context => { + if (context.event.tag === 'Reducer') void session.shutdown(); + }); + connection + .subscriptionBuilder() + .onApplied(() => { + void connection.reducers + .createPlayer({ name: 'confirmed', location: { x: 0, y: 0 } }) + .then( + () => confirmed.resolve(), + error => confirmed.reject(error) + ); + }) + .subscribe('SELECT * FROM player'); + }); + const run = session.run(); + await confirmed.promise; + await run; + await fixture.joinPeers(); + expect(fixture.live).toBe(0); + }); + + it('rejects invalid server configuration before broker I/O and owns run state', async () => { + const fixture = await new Fixture().start(); + for (const serverUri of [ + fixture.serverUri + '?token=secret', + fixture.serverUri + '#fragment', + fixture.serverUri + '\u0000', + 'http://user:secret@127.0.0.1/', + ]) { + expect( + () => + new ContainerSession(DbConnection, { + container: fixture.source, + serverUri, + onConnect: () => {}, + }) + ).toThrow(ContainerSessionError); + } + expect(fixture.requests).toBe(0); + const session = own(fixture, () => {}); + const stopped = new AbortController(); + stopped.abort(); + await session.run(stopped.signal); + expect(fixture.requests).toBe(0); + const pause = new AbortController(); + const run = session.run(pause.signal); + await expect(session.run()).rejects.toMatchObject({ + code: 'already_running', + }); + pause.abort(); + await run; + await session.shutdown(); + await expect(session.run()).rejects.toMatchObject({ code: 'terminated' }); + }); + + it('rejects adopting a connection with subscriptions already queued', async () => { + const adapter = new WebsocketTestAdapter(); + const closed = new Deferred(); + const close = adapter.close.bind(adapter); + adapter.close = () => { + close(); + closed.resolve(); + }; + const connection = DbConnection.builder() + .withUri('http://127.0.0.1:1') + .withDatabaseName(TARGET) + .withWSFn(adapter.openWebSocket) + .build(); + try { + connection.subscriptionBuilder().subscribe('SELECT * FROM player'); + expect(() => connection[INTERNAL_MANAGED_SESSION]().enable()).toThrow( + 'Managed boundary requires an unused connection' + ); + } finally { + connection.disconnect(); + await closed.promise; + expect(adapter.closed).toBe(true); + } + }); +}); From 69c76301f4c3dc81bd71628f658c251982cd98e7 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 04:25:23 -0400 Subject: [PATCH 13/16] Protect publication preparation and fix Stage 1 CI failures --- crates/cli/src/container/mod.rs | 15 ++-- crates/cli/src/container/publish/journal.rs | 19 ++++- crates/cli/src/container/publish/tests.rs | 38 +++++++--- .../publish/tests/environment_tests.rs | 18 +++-- crates/cli/src/container/tests.rs | 54 ++++++++++++++ crates/cli/src/subcommands/publish/managed.rs | 71 +++++++++++++++++-- .../publish/managed/environment_tests.rs | 6 +- crates/pg/src/pg_server.rs | 12 ++-- crates/smoketests/modules/Cargo.lock | 1 + 9 files changed, 193 insertions(+), 41 deletions(-) diff --git a/crates/cli/src/container/mod.rs b/crates/cli/src/container/mod.rs index 02af7005921..bfb9accfc2d 100644 --- a/crates/cli/src/container/mod.rs +++ b/crates/cli/src/container/mod.rs @@ -156,11 +156,16 @@ pub async fn prepare_container( ); let cancel = cancel.child_token(); let _cancel = CancelOnDrop(cancel.clone()); - let workspace = Arc::new( - tempfile::Builder::new() - .prefix(".spacetime-image-") - .tempdir_in(workspace_parent)?, - ); + let mut workspace = tempfile::Builder::new(); + workspace.prefix(".spacetime-image-"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + // Registry credentials and build secrets are copied here before a + // publication journal exists, including under an existing public base. + workspace.permissions(fs::Permissions::from_mode(0o700)); + } + let workspace = Arc::new(workspace.tempdir_in(workspace_parent)?); let base = workspace.path(); fs::create_dir(base.join("auth"))?; let auth_file = base.join("auth/config.json"); diff --git a/crates/cli/src/container/publish/journal.rs b/crates/cli/src/container/publish/journal.rs index 93f61ddcceb..a93e8caf8ce 100644 --- a/crates/cli/src/container/publish/journal.rs +++ b/crates/cli/src/container/publish/journal.rs @@ -127,6 +127,22 @@ impl Drop for IncompleteDirectory { } } impl Journal { + /// Prepare ancestors before local image preparation. Keep the returned + /// directory handles alive while preparing or accessing retained inputs. + pub(crate) fn prepare_base(base: &Path) -> Result> { + let mut directories = fs::DirBuilder::new(); + directories.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + // New ancestors must protect retained environment values even + // when the user's umask permits group or other-user writes. + directories.mode(0o700); + } + directories.create(base)?; + protected::pin_parents(base) + } + pub fn create( base: &Path, record: Record, @@ -136,8 +152,7 @@ impl Journal { let request = record.request()?; let has_image = image.is_some(); let directory = base.join(request.manifest.current().envelope.operation_id.to_string()); - fs::create_dir_all(base)?; - let _base_parents = protected::pin_parents(base)?; + let _base_parents = Self::prepare_base(base)?; protected::create_directory(&directory) .context("cannot create protected publication directory; resume an existing operation instead")?; let mut incomplete = IncompleteDirectory(Some(directory.clone())); diff --git a/crates/cli/src/container/publish/tests.rs b/crates/cli/src/container/publish/tests.rs index e5dab901ece..c3c7bf27a84 100644 --- a/crates/cli/src/container/publish/tests.rs +++ b/crates/cli/src/container/publish/tests.rs @@ -22,6 +22,22 @@ use std::{ }; const TOKEN: &str = "Bearer isolated-fixture-credential"; + +pub(crate) fn temporary_directory() -> tempfile::TempDir { + let mut builder = tempfile::Builder::new(); + builder.prefix("publication-test-"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + // The fixture itself must satisfy the production ancestor policy. + // tempfile's default directory mode depends on the process umask. + builder.permissions(std::fs::Permissions::from_mode(0o700)); + builder.tempdir_in("/tmp").unwrap() + } + #[cfg(not(unix))] + builder.tempdir().unwrap() +} + pub(crate) fn empty_module_artifact() -> ModuleArtifact { let bytes = system_empty::empty().bytes.as_ref(); ModuleArtifact { @@ -400,7 +416,7 @@ async fn lost_append_and_committed_response_resume_without_reupload_or_current_p s.lose_append = true; s.lose_submit_after_commit = true; } - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut journal = journal(temporary.path(), fixture.record(false, false)); let exact = journal.record.request_json.clone(); assert!(run_now(&fixture.client(), &mut journal).await.is_err()); @@ -430,7 +446,7 @@ async fn lost_reservation_and_unadmitted_put_replay_exact_request_and_generated_ s.lose_reservation = true; s.lose_submit_before_commit = true; } - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut journal = journal(temporary.path(), fixture.record(true, false)); let exact = journal.record.request_json.clone(); assert!(run_now(&fixture.client(), &mut journal).await.is_err()); @@ -453,7 +469,7 @@ async fn lost_reservation_and_unadmitted_put_replay_exact_request_and_generated_ async fn stale_revision_preserves_operation_and_redacts_response_without_fallback() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().stale = true; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut journal = journal(temporary.path(), fixture.record(false, false)); let exact = journal.record.request_json.clone(); let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); @@ -472,7 +488,7 @@ async fn mismatched_reservation_or_upload_receipt_never_reaches_admission() { s.wrong_reservation = reservation; s.bad_receipt = !reservation; } - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut journal = journal(temporary.path(), fixture.record(reservation, false)); assert!(run_now(&fixture.client(), &mut journal).await.is_err()); assert!(!journal.record.submitted); @@ -484,7 +500,7 @@ async fn mismatched_reservation_or_upload_receipt_never_reaches_admission() { async fn completion_descriptor_must_match_before_admission() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().bad_completion = true; - let directory = tempfile::tempdir().unwrap(); + let directory = crate::container::publish::tests::temporary_directory(); let mut journal = journal(directory.path(), fixture.record(true, false)); let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); assert!(error.to_string().contains("completion descriptor changed")); @@ -505,7 +521,7 @@ async fn completion_descriptor_must_match_before_admission() { async fn lost_completion_response_observes_the_same_complete_session() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().lose_completion = true; - let directory = tempfile::tempdir().unwrap(); + let directory = crate::container::publish::tests::temporary_directory(); let mut journal = journal(directory.path(), fixture.record(true, false)); assert!(matches!( run_now(&fixture.client(), &mut journal).await.unwrap(), @@ -531,7 +547,7 @@ async fn wrong_publication_scope_and_denied_upload_are_not_accepted() { s.deny_upload = denied; s.bad_status = !denied; } - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut journal = journal(temporary.path(), fixture.record(false, false)); assert!(run_now(&fixture.client(), &mut journal).await.is_err()); assert!(journal.record.status.is_none()); @@ -543,7 +559,7 @@ async fn wrong_publication_scope_and_denied_upload_are_not_accepted() { async fn naming_failure_reports_active_identity_without_republishing_or_overwriting_later_names() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().fail_naming = true; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut journal = journal(temporary.path(), fixture.record(true, true)); assert!(matches!( run_now(&fixture.client(), &mut journal).await.unwrap(), @@ -582,7 +598,7 @@ async fn redirect_and_cross_origin_never_forward_credentials_without_exact_appro #[tokio::test] async fn journal_locks_and_reverifies_local_bytes_without_persisting_credentials() { let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut journal = journal(temporary.path(), fixture.record(false, false)); let path = journal.directory().to_owned(); assert!(Journal::open(&path).is_err()); @@ -632,7 +648,7 @@ fn endpoints_and_receipts_are_bounded_and_unambiguous() { #[tokio::test] async fn incomplete_artifact_retention_never_publishes_a_resume_record_or_mutates_server() { let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let base = temporary.path().join("new").join("nested").join("state"); let record = fixture.record(false, false); let id = record.request().unwrap().manifest.current().envelope.operation_id; @@ -648,7 +664,7 @@ async fn incomplete_artifact_retention_never_publishes_a_resume_record_or_mutate async fn revoked_status_access_replays_original_put_without_missing_local_artifacts_or_new_uploads() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().lose_submit_after_commit = true; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut journal = journal(temporary.path(), fixture.record(false, false)); let exact = journal.record.request_json.clone(); assert!(run_now(&fixture.client(), &mut journal).await.is_err()); diff --git a/crates/cli/src/container/publish/tests/environment_tests.rs b/crates/cli/src/container/publish/tests/environment_tests.rs index ba0d335f5a1..10ad97b24a6 100644 --- a/crates/cli/src/container/publish/tests/environment_tests.rs +++ b/crates/cli/src/container/publish/tests/environment_tests.rs @@ -11,7 +11,7 @@ fn set_environment(record: &mut Record, values: BTreeMap) { async fn complete_large_environment_is_private_and_exact_after_lost_reply_and_revocation() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().lose_submit_after_commit = true; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut record = fixture.record(false, false); set_environment( &mut record, @@ -55,7 +55,7 @@ async fn missing_changed_and_malformed_secret_body_never_becomes_empty_or_leaks_ Some(b"{}".as_slice()), Some(br#"{"environment":{"KEY":"secret-body-sentinel", "KEY":17}}"#.as_slice()), ] { - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut record = fixture.record(false, false); set_environment(&mut record, BTreeMap::from([("KEY".into(), "original-secret".into())])); let retained = journal(temporary.path(), record); @@ -77,7 +77,7 @@ async fn missing_changed_and_malformed_secret_body_never_becomes_empty_or_leaks_ #[tokio::test] async fn status_must_match_previous_operation_and_the_first_confirmed_epoch() { let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut retained = journal(temporary.path(), fixture.record(false, false)); run_now(&fixture.client(), &mut retained).await.unwrap(); let request = retained.record.request().unwrap(); @@ -98,8 +98,12 @@ async fn status_must_match_previous_operation_and_the_first_confirmed_epoch() { async fn protected_storage_rejects_public_modes_symlinks_and_hardlinks() { use std::os::unix::fs::{symlink, PermissionsExt}; let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); - let retained = journal(temporary.path(), fixture.record(false, false)); + let temporary = crate::container::publish::tests::temporary_directory(); + let base = temporary.path().join("new/retained"); + let retained = journal(&base, fixture.record(false, false)); + for parent in [temporary.path().join("new"), base] { + assert_eq!(std::fs::metadata(parent).unwrap().permissions().mode() & 0o777, 0o700); + } let path = retained.directory().to_owned(); let body = path.join("submission.json"); assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o700); @@ -178,7 +182,7 @@ async fn kept_schema_checks_identity_program_hash_and_bounded_v10_metadata() { async fn malformed_server_response_cannot_disclose_echoed_environment_values() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().malformed_status = true; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut retained = journal(temporary.path(), fixture.record(false, false)); let error = run_now(&fixture.client(), &mut retained).await.unwrap_err(); assert!(!format!("{error:#}").contains("secret-body-sentinel")); @@ -191,7 +195,7 @@ async fn malformed_server_response_cannot_disclose_echoed_environment_values() { async fn untrusted_writable_parent_is_rejected_before_retaining_secrets() { use std::os::unix::fs::PermissionsExt; let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let base = temporary.path().join("untrusted"); std::fs::create_dir(&base).unwrap(); std::fs::set_permissions(&base, std::fs::Permissions::from_mode(0o777)).unwrap(); diff --git a/crates/cli/src/container/tests.rs b/crates/cli/src/container/tests.rs index 6282db120f9..9dabee6c96d 100644 --- a/crates/cli/src/container/tests.rs +++ b/crates/cli/src/container/tests.rs @@ -413,6 +413,60 @@ async fn registry_import_uses_explicit_anonymous_auth_and_records_selected_diges assert_eq!(fs::read(auth).unwrap(), br#"{"auths":{}}"#); } +#[cfg(unix)] +#[tokio::test] +async fn registry_credentials_are_private_during_preparation_under_an_existing_readable_base() { + use std::os::unix::fs::PermissionsExt; + let _serial = TEST_PREPARATIONS.lock().await; + let root = crate::container::publish::tests::temporary_directory(); + let input = root.path().join("image"); + fixture(&input); + let base = root.path().join("existing"); + fs::create_dir(&base).unwrap(); + fs::set_permissions(&base, fs::Permissions::from_mode(0o755)).unwrap(); + let credential = root.path().join("registry.json"); + const AUTH: &[u8] = br#"{"auths":{"example.invalid":{"auth":"fixture-registry-secret"}}}"#; + fs::write(&credential, AUTH).unwrap(); + fs::set_permissions(&credential, fs::Permissions::from_mode(0o600)).unwrap(); + struct InspectingRunner { + inner: FakeRunner, + workspace: Mutex>, + } + impl Runner for InspectingRunner { + async fn run(&self, invocation: Invocation) -> Result { + let workspace = invocation.workspace.path(); + assert_eq!(fs::metadata(workspace).unwrap().permissions().mode() & 0o777, 0o700); + assert_eq!(fs::read(workspace.join("auth/config.json")).unwrap(), AUTH); + *self.workspace.lock().unwrap() = Some(workspace.to_path_buf()); + self.inner.run(invocation).await + } + } + let runner = InspectingRunner { + inner: FakeRunner::new(&input), + workspace: Mutex::new(None), + }; + let tools = BuildTools { + registry_auth_file: Some(credential), + ..Default::default() + }; + let prepared = prepare_container( + &declaration(json!({"oci_ref":"example.invalid/team/image:fixture"})), + root.path(), + platform(), + &tools, + &base, + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(fs::metadata(&base).unwrap().permissions().mode() & 0o777, 0o755); + let workspace = runner.workspace.lock().unwrap().clone().unwrap(); + assert!(workspace.exists()); + drop(prepared); + assert!(!workspace.exists()); +} + #[test] fn archive_rejects_links_and_cancelled_reads() { let root = tempfile::tempdir().unwrap(); diff --git a/crates/cli/src/subcommands/publish/managed.rs b/crates/cli/src/subcommands/publish/managed.rs index f08a93d57ca..983c083a466 100644 --- a/crates/cli/src/subcommands/publish/managed.rs +++ b/crates/cli/src/subcommands/publish/managed.rs @@ -306,7 +306,7 @@ async fn try_execute_with_client( .get_one::("publication_state_dir") .cloned() .unwrap_or_else(|| config_dir.unwrap_or(&cwd).join(".spacetime/publications")); - std::fs::create_dir_all(&base)?; + let _base_parents = Journal::prepare_base(&base)?; let cancel = CancellationToken::new(); let prepare = prepare_request( &client, @@ -756,13 +756,13 @@ mod tests { #[tokio::test] async fn container_only_frontend_uploads_verified_closure_with_server_reserved_identity() { let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let layout = temporary.path().join("layout"); let selected = crate::container::tests::fixture(&layout); let declaration = crate::container::tests::declaration(json!({"oci_ref":"oci:layout"})); let command = super::super::cli(); let schema = super::super::build_publish_schema(&command).unwrap(); - let state = temporary.path().join("state"); + let state = temporary.path().join("new/state"); let args = command .try_get_matches_from([ "publish", @@ -791,6 +791,13 @@ mod tests { ) .await .unwrap()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for parent in [temporary.path().join("new"), state.clone()] { + assert_eq!(std::fs::metadata(parent).unwrap().permissions().mode() & 0o777, 0o700); + } + } { let snapshot = fixture.state.lock().unwrap(); assert_eq!(snapshot.reservations.len(), 1); @@ -818,10 +825,62 @@ mod tests { } fixture.close().await; } + #[cfg(unix)] + #[tokio::test] + async fn writable_publication_parent_is_rejected_before_image_preparation_or_submission() { + use std::os::unix::fs::PermissionsExt; + let fixture = Fixture::new().await; + let temporary = crate::container::publish::tests::temporary_directory(); + let state = temporary.path().join("state"); + std::fs::create_dir(&state).unwrap(); + std::fs::set_permissions(&state, std::fs::Permissions::from_mode(0o777)).unwrap(); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--container-platform", + "linux/amd64", + "--publication-state-dir", + state.to_str().unwrap(), + ]) + .unwrap(); + // If preparation runs, this missing local source would fail first. + // No real image helper, registry, or saved configuration is used. + let declaration = crate::container::tests::declaration(json!({"oci_ref":"oci:missing-fixture"})); + let target = CommandConfig::new(&schema, HashMap::new(), &args) + .unwrap() + .with_container(Some(declaration)); + let error = try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all(), + ) + .await + .unwrap_err(); + assert!(format!("{error:#}").contains("untrusted writable ancestor")); + assert_eq!(std::fs::metadata(&state).unwrap().permissions().mode() & 0o777, 0o777); + assert_eq!(std::fs::read_dir(&state).unwrap().count(), 0); + { + let state = fixture.state.lock().unwrap(); + assert!(state.reservations.is_empty()); + assert!(state.submits.is_empty()); + assert_eq!(state.begin_count, 0); + } + fixture.close().await; + } + #[tokio::test] async fn existing_managed_module_update_keeps_container_and_preflights_without_pro() { let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let wasm = temporary.path().join("module.wasm"); std::fs::write(&wasm, deployment::system_empty::empty().bytes.as_ref()).unwrap(); let prior_request = fixture.record(false, false).request().unwrap(); @@ -888,7 +947,7 @@ mod tests { #[tokio::test] async fn remove_module_requires_authorized_preflight_before_any_upload_or_submission() { let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let request = fixture.record(false, false).request().unwrap(); { let mut state = fixture.state.lock().unwrap(); @@ -964,7 +1023,7 @@ mod tests { async fn resume_entrypoint_ignores_changed_project_and_reuses_original_bytes() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().lose_submit_before_commit = true; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut record = fixture.record(false, false); let mut request = record.request().unwrap(); request diff --git a/crates/cli/src/subcommands/publish/managed/environment_tests.rs b/crates/cli/src/subcommands/publish/managed/environment_tests.rs index 1bbad838f07..582a5615234 100644 --- a/crates/cli/src/subcommands/publish/managed/environment_tests.rs +++ b/crates/cli/src/subcommands/publish/managed/environment_tests.rs @@ -21,7 +21,7 @@ fn declared_module() -> deployment::system_empty::GeneratedModule { #[tokio::test] async fn keep_resolves_program_bound_declarations_and_retains_complete_input() { let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let module = declared_module(); let deployment = deployment::DeploymentSpec::V1(deployment::DeploymentSpecV1 { module: deployment::ModuleComponent::SystemEmpty(module.descriptor), @@ -105,7 +105,7 @@ async fn keep_resolves_program_bound_declarations_and_retains_complete_input() { #[tokio::test] async fn precompiled_exact_bytes_validate_required_invalid_unknown_and_explicit_values_before_mutation() { let fixture = Fixture::new().await; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let module = declared_module(); let wasm = temporary.path().join("exact.wasm"); std::fs::write(&wasm, &module.bytes).unwrap(); @@ -156,7 +156,7 @@ async fn precompiled_exact_bytes_validate_required_invalid_unknown_and_explicit_ async fn fresh_process_resume_replays_original_values_after_shell_changes() { let fixture = Fixture::new().await; fixture.state.lock().unwrap().lose_submit_before_commit = true; - let temporary = tempfile::tempdir().unwrap(); + let temporary = crate::container::publish::tests::temporary_directory(); let mut record = fixture.record(false, false); let mut request = record.request().unwrap(); request.environment.insert(KEY.into(), "first-secret".into()); diff --git a/crates/pg/src/pg_server.rs b/crates/pg/src/pg_server.rs index f1de39f4efd..dcdb0ecc360 100644 --- a/crates/pg/src/pg_server.rs +++ b/crates/pg/src/pg_server.rs @@ -282,13 +282,11 @@ impl claims, - Err(err) => { - // TODO: Do not log the supplied password/token; then classify credential errors separately from provider failures. - log::warn!( - "PG: Authentication failed for identity `{}` on database {database}: {err}", - pwd.password - ); - let err = ErrorInfo::new("FATAL".to_owned(), "28P01".to_owned(), err.to_string()); + Err(_) => { + // The supplied password is a bearer token. Validator + // errors can also contain untrusted claims or responses. + log::warn!("PG: Authentication failed on database {database}"); + let err = ErrorInfo::new("FATAL".to_owned(), "28P01".to_owned(), "Invalid token".to_owned()); return close_client(client, err).await; } }; diff --git a/crates/smoketests/modules/Cargo.lock b/crates/smoketests/modules/Cargo.lock index 3bfe68ba8c2..d84b1fda1cf 100644 --- a/crates/smoketests/modules/Cargo.lock +++ b/crates/smoketests/modules/Cargo.lock @@ -1221,6 +1221,7 @@ dependencies = [ "spacetimedb-bindings-macro", "spacetimedb-primitives", "spacetimedb-sats", + "thiserror", ] [[package]] From a9b6241ca9d2697830fc4b37de0e04ae7d967b82 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 05:02:42 -0400 Subject: [PATCH 14/16] Preserve subscription error contexts when container sessions close --- .../src/sdk/db_connection_impl.ts | 10 +++++++++- .../tests/node_container_session.test.ts | 17 ++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/bindings-typescript/src/sdk/db_connection_impl.ts b/crates/bindings-typescript/src/sdk/db_connection_impl.ts index 91acc700ed7..f3d7aa7034e 100644 --- a/crates/bindings-typescript/src/sdk/db_connection_impl.ts +++ b/crates/bindings-typescript/src/sdk/db_connection_impl.ts @@ -328,10 +328,18 @@ export class DbConnectionImpl call.reject( new ContainerSessionCallError(call.sent ? 'unknown' : 'not_sent') ); + const errorContext: ErrorContextInterface = { + ...this.#makeEventContext({ + id: this.#nextEventId(), + tag: 'Error', + value: error, + }), + event: error, + }; let callbackFailed = false; for (const { emitter } of subscriptions) { try { - emitter.emit('error', this, error); + emitter.emit('error', errorContext, error); } catch { callbackFailed = true; } finally { diff --git a/crates/bindings-typescript/tests/node_container_session.test.ts b/crates/bindings-typescript/tests/node_container_session.test.ts index 4c117db467d..946fe7855d9 100644 --- a/crates/bindings-typescript/tests/node_container_session.test.ts +++ b/crates/bindings-typescript/tests/node_container_session.test.ts @@ -408,12 +408,15 @@ describe('managed Node container sessions', () => { const applied = new Deferred(); let oldCall: Promise | undefined; let laterSubscriptionErrors = 0; + const callbackErrors: unknown[] = []; const session = own(fixture, connection => { connection .subscriptionBuilder() .onApplied(() => applied.resolve()) - .onError(() => { - oldCall = connection.reducers + .onError(context => { + callbackErrors.push(context.event); + expect(context.isActive).toBe(false); + oldCall = context.reducers .createPlayer({ name: 'after-seal', location: { x: 0, y: 0 } }) .catch(error => error); throw new Error('application-private-error'); @@ -421,7 +424,10 @@ describe('managed Node container sessions', () => { .subscribe('SELECT * FROM player'); connection .subscriptionBuilder() - .onError(() => laterSubscriptionErrors++) + .onError(context => { + callbackErrors.push(context.event); + laterSubscriptionErrors++; + }) .subscribe('SELECT * FROM player'); }); const run = session.run().catch(error => error); @@ -431,8 +437,13 @@ describe('managed Node container sessions', () => { expect(await oldCall).toBeInstanceOf(ContainerSessionCallError); expect(await oldCall).toMatchObject({ outcome: 'not_sent' }); expect(laterSubscriptionErrors).toBe(1); + expect(callbackErrors).toHaveLength(2); + expect(callbackErrors[0]).toBeInstanceOf(ContainerSessionError); + expect(callbackErrors[0]).toMatchObject({ code: 'terminated' }); + expect(callbackErrors[1]).toBe(callbackErrors[0]); await session.shutdown(); expect(laterSubscriptionErrors).toBe(1); + expect(callbackErrors).toHaveLength(2); await fixture.joinPeers(); expect(fixture.live).toBe(0); }); From 228371c30c9d544301da74bca8413be8fb17c1b3 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 05:15:08 -0400 Subject: [PATCH 15/16] Reconcile hosted admission in the invocation fixture --- crates/testing/tests/hosted_invocation.rs | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/testing/tests/hosted_invocation.rs b/crates/testing/tests/hosted_invocation.rs index 7c9b3c5528c..269f3c582b6 100644 --- a/crates/testing/tests/hosted_invocation.rs +++ b/crates/testing/tests/hosted_invocation.rs @@ -112,6 +112,21 @@ fn hosted_wasm_calls_preserve_authority_and_disconnect_after_revocation() { install_fence(&module, foreign, 1, true); let self_auth = authenticate(target, target, Duration::from_secs(30)); let foreign_auth = authenticate(foreign, target, Duration::from_secs(30)); + // This in-process standalone fixture has no cloud reconciliation + // service. Grant rows alone must not open hosted admission. + assert!(!module.relational_db().hosted_admission().is_open()); + assert!(module + .call_identity_connected(self_auth.clone(), self_connection) + .await + .is_err()); + assert_connection_count(&module, 0); + module + .relational_db() + .hosted_admission() + .begin() + .unwrap() + .complete() + .unwrap(); let expiring_auth = authenticate(target, target, Duration::from_secs(3)); for (auth, connection) in [ (self_auth.clone(), self_connection), @@ -197,6 +212,15 @@ fn hosted_wasm_calls_preserve_authority_and_disconnect_after_revocation() { // Tokens remain cryptographically valid, but their persisted grants are revoked. install_fence(&module, target, 2, false); install_fence(&module, foreign, 2, false); + assert!(!module.relational_db().hosted_admission().is_open()); + // Reopening the database-wide gate must not restore a denied grant. + module + .relational_db() + .hosted_admission() + .begin() + .unwrap() + .complete() + .unwrap(); assert!( call(&module, &self_auth, Some(self_connection), "internal_only", product![]) .await From 5745cdef50fcf1c8384b31426f5d4bb1607c7184 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Thu, 10 Sep 2026 05:46:20 -0400 Subject: [PATCH 16/16] Refresh module dependency snapshot for typed container errors --- ...ps__spacetimedb_bindings_dependencies.snap | 94 +++++++++---------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap b/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap index e344e1fa4f2..53786286152 100644 --- a/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap +++ b/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap @@ -1,6 +1,5 @@ --- source: crates/bindings/tests/deps.rs -assertion_line: 16 expression: "cargo tree -p spacetimedb -e no-dev --color never --target wasm32-unknown-unknown -f {lib}" --- total crates: 72 @@ -72,52 +71,53 @@ spacetimedb │ ├── hex │ ├── spacetimedb_bindings_macro (*) │ ├── spacetimedb_primitives (*) -│ └── spacetimedb_sats -│ ├── anyhow -│ ├── arrayvec -│ ├── bytemuck -│ ├── bytes -│ ├── chrono -│ │ └── num_traits -│ │ [build-dependencies] -│ │ └── autocfg -│ ├── decorum -│ │ ├── approx -│ │ │ └── num_traits (*) -│ │ └── num_traits (*) -│ ├── derive_more (*) -│ ├── enum_as_inner (*) -│ ├── ethnum -│ │ └── serde -│ │ └── serde_core -│ ├── hex -│ ├── itertools (*) -│ ├── lean_string -│ │ ├── castaway -│ │ │ └── rustversion -│ │ ├── itoa -│ │ └── ryu -│ ├── second_stack -│ ├── sha3 -│ │ ├── digest -│ │ │ ├── block_buffer -│ │ │ │ └── generic_array -│ │ │ │ └── typenum -│ │ │ │ [build-dependencies] -│ │ │ │ └── version_check -│ │ │ └── crypto_common -│ │ │ ├── generic_array (*) -│ │ │ └── typenum -│ │ └── keccak -│ ├── smallvec -│ ├── spacetimedb_bindings_macro (*) -│ ├── spacetimedb_primitives (*) -│ ├── thiserror -│ │ └── thiserror_impl -│ │ ├── proc_macro2 (*) -│ │ ├── quote (*) -│ │ └── syn (*) -│ └── uuid +│ ├── spacetimedb_sats +│ │ ├── anyhow +│ │ ├── arrayvec +│ │ ├── bytemuck +│ │ ├── bytes +│ │ ├── chrono +│ │ │ └── num_traits +│ │ │ [build-dependencies] +│ │ │ └── autocfg +│ │ ├── decorum +│ │ │ ├── approx +│ │ │ │ └── num_traits (*) +│ │ │ └── num_traits (*) +│ │ ├── derive_more (*) +│ │ ├── enum_as_inner (*) +│ │ ├── ethnum +│ │ │ └── serde +│ │ │ └── serde_core +│ │ ├── hex +│ │ ├── itertools (*) +│ │ ├── lean_string +│ │ │ ├── castaway +│ │ │ │ └── rustversion +│ │ │ ├── itoa +│ │ │ └── ryu +│ │ ├── second_stack +│ │ ├── sha3 +│ │ │ ├── digest +│ │ │ │ ├── block_buffer +│ │ │ │ │ └── generic_array +│ │ │ │ │ └── typenum +│ │ │ │ │ [build-dependencies] +│ │ │ │ │ └── version_check +│ │ │ │ └── crypto_common +│ │ │ │ ├── generic_array (*) +│ │ │ │ └── typenum +│ │ │ └── keccak +│ │ ├── smallvec +│ │ ├── spacetimedb_bindings_macro (*) +│ │ ├── spacetimedb_primitives (*) +│ │ ├── thiserror +│ │ │ └── thiserror_impl +│ │ │ ├── proc_macro2 (*) +│ │ │ ├── quote (*) +│ │ │ └── syn (*) +│ │ └── uuid +│ └── thiserror (*) ├── spacetimedb_primitives (*) └── spacetimedb_query_builder └── spacetimedb_lib (*)