From 52f973caef7d369458389a5fc97918da88a6adb4 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Mon, 7 Sep 2026 19:25:25 -0400 Subject: [PATCH 01/23] Add container deployment protocol and fenced hosted authentication --- Cargo.lock | 23 + Cargo.toml | 2 + crates/auth/src/hosted.rs | 322 ++++++++++++ crates/auth/src/identity.rs | 22 +- crates/auth/src/lib.rs | 1 + crates/bindings-cpp/README.md | 25 +- .../include/spacetimedb/abi/FFI.h | 1 + .../include/spacetimedb/abi/abi.h | 7 + .../include/spacetimedb/auth_ctx.h | 90 ++-- .../include/spacetimedb/function_visibility.h | 6 + .../autogen/FunctionVisibilityV11.g.h | 23 + .../autogen/RawMiscModuleExportV9.g.h | 2 +- .../internal/autogen/RawModuleDef.g.h | 3 +- .../autogen/RawModuleDefV10Section.g.h | 16 +- .../internal/autogen/RawModuleDefV11.g.h | 27 + .../autogen/RawModuleDefV11Section.g.h | 32 ++ .../internal/autogen/RawModuleDefV8.g.h | 4 +- .../internal/autogen/RawModuleDefV9.g.h | 4 +- .../internal/autogen/RawProcedureDefV10.g.h | 2 +- .../internal/autogen/RawProcedureDefV11.g.h | 35 ++ .../internal/autogen/RawProcedureDefV9.g.h | 2 +- .../internal/autogen/RawReducerDefV11.g.h | 37 ++ .../internal/autogen/RawReducerDefV9.g.h | 2 +- .../internal/autogen/RawTableDefV10.g.h | 6 +- .../internal/autogen/RawTableDefV8.g.h | 4 +- .../internal/autogen/RawTableDefV9.g.h | 4 +- .../spacetimedb/internal/v10_builder.h | 36 +- .../include/spacetimedb/jwt_claims.h | 12 +- .../bindings-cpp/include/spacetimedb/macros.h | 10 +- .../include/spacetimedb/procedure_context.h | 19 +- crates/bindings-cpp/src/internal/Module.cpp | 8 +- .../bindings-cpp/src/internal/v10_builder.cpp | 114 +++-- crates/bindings-cpp/tests/unit/CMakeLists.txt | 12 + .../unit/function_visibility_unit_tests.cpp | 74 +++ .../tests/unit/hosted_auth_unit_tests.cpp | 104 ++++ crates/bindings-csharp/Codegen.Tests/Tests.cs | 65 +++ .../diag/snapshots/Module#FFI.verified.cs | 32 +- .../snapshots/Module#FFI.verified.cs | 8 +- .../server/snapshots/Module#FFI.verified.cs | 24 +- crates/bindings-csharp/Codegen/Diag.cs | 9 + crates/bindings-csharp/Codegen/Module.cs | 54 +- crates/bindings-csharp/README.md | 22 +- .../Runtime.Tests/FunctionVisibilityTests.cs | 54 ++ .../Runtime.Tests/HostedAuthTests.cs | 43 ++ crates/bindings-csharp/Runtime/Attrs.cs | 12 + crates/bindings-csharp/Runtime/AuthCtx.cs | 34 +- .../Autogen/FunctionVisibilityV11.g.cs | 17 + .../Internal/Autogen/RawModuleDef.g.cs | 3 +- .../Internal/Autogen/RawModuleDefV11.g.cs | 29 ++ .../Autogen/RawModuleDefV11Section.g.cs | 27 + .../Internal/Autogen/RawProcedureDefV11.g.cs | 45 ++ .../Internal/Autogen/RawReducerDefV11.g.cs | 50 ++ .../bindings-csharp/Runtime/Internal/FFI.cs | 11 + .../Runtime/Internal/IReducer.cs | 2 +- .../Runtime/Internal/Module.cs | 80 ++- .../Runtime/Internal/Procedure.cs | 2 +- crates/bindings-csharp/Runtime/JwtClaims.cs | 4 +- crates/bindings-csharp/Runtime/Runtime.csproj | 1 + crates/bindings-csharp/Runtime/bindings.c | 4 + crates/bindings-macro/src/procedure.rs | 37 +- crates/bindings-macro/src/reducer.rs | 89 ++++ crates/bindings-sys/src/lib.rs | 15 + crates/bindings-typescript/README.md | 19 + .../src/lib/autogen/types.ts | 94 ++++ .../bindings-typescript/src/lib/reducers.ts | 4 +- crates/bindings-typescript/src/lib/schema.ts | 12 +- .../src/server/function_visibility.ts | 22 + .../bindings-typescript/src/server/index.ts | 1 + .../src/server/procedures.ts | 45 +- .../src/server/reducers.ts | 21 +- .../bindings-typescript/src/server/runtime.ts | 49 +- .../bindings-typescript/src/server/schema.ts | 20 +- .../bindings-typescript/src/server/sys.d.ts | 5 + .../tests/hosted_auth.test.ts | 225 +++++++++ crates/bindings/src/http.rs | 8 +- crates/bindings/src/lib.rs | 139 +++++- crates/bindings/src/rt.rs | 29 +- .../tests/pass/function_visibility.rs | 56 +++ crates/bindings/tests/ui/tables.stderr | 20 +- crates/cli/src/api.rs | 6 +- crates/cli/src/lib.rs | 2 - crates/cli/src/subcommands/describe.rs | 16 +- crates/cli/src/subcommands/generate.rs | 2 +- crates/cli/src/subcommands/mod.rs | 1 - crates/cli/src/subcommands/sidecar.rs | 201 -------- crates/cli/src/subcommands/subscribe.rs | 48 +- crates/cli/src/util.rs | 5 - crates/client-api/src/auth.rs | 47 +- crates/client-api/src/lib.rs | 19 +- crates/client-api/src/routes/database.rs | 26 +- crates/codegen/src/util.rs | 88 +++- crates/core/src/auth/hosted_tokens.rs | 320 ++++++++++++ crates/core/src/auth/invocation.rs | 123 +++++ crates/core/src/auth/invocation/tests.rs | 156 ++++++ crates/core/src/auth/mod.rs | 2 + crates/core/src/auth/token_validation.rs | 13 + crates/core/src/client/client_connection.rs | 465 ++++++++++++++++- crates/core/src/db/deployment.rs | 347 +++++++++++++ crates/core/src/db/deployment/tests.rs | 385 ++++++++++++++ crates/core/src/db/mod.rs | 1 + crates/core/src/db/relational_db.rs | 1 + crates/core/src/error.rs | 2 + crates/core/src/host/host_controller.rs | 19 +- crates/core/src/host/instance_env.rs | 145 +++++- crates/core/src/host/mod.rs | 1 + crates/core/src/host/module_host.rs | 174 +++++-- crates/core/src/host/v8/mod.rs | 7 +- crates/core/src/host/v8/syscall/common.rs | 2 + crates/core/src/host/v8/syscall/mod.rs | 1 + crates/core/src/host/v8/syscall/v1.rs | 2 + crates/core/src/host/v8/syscall/v2.rs | 15 + crates/core/src/host/wasm_common.rs | 1 + .../src/host/wasm_common/module_host_actor.rs | 77 +++ .../src/host/wasmtime/wasm_instance_env.rs | 17 + .../core/src/host/wasmtime/wasmtime_module.rs | 6 +- crates/core/src/sql/execute.rs | 42 +- .../subscription/module_subscription_actor.rs | 108 +++- .../module_subscription_manager.rs | 2 +- .../locking_tx_datastore/committed_state.rs | 3 + .../src/locking_tx_datastore/datastore.rs | 50 ++ .../src/locking_tx_datastore/mut_tx.rs | 15 +- crates/datastore/src/system_tables.rs | 22 +- .../datastore/src/system_tables/deployment.rs | 236 +++++++++ crates/lib/src/container.rs | 469 ++++++++++++++++++ crates/lib/src/container/tests.rs | 246 +++++++++ crates/lib/src/db/raw_def.rs | 2 + crates/lib/src/db/raw_def/v10.rs | 4 +- crates/lib/src/db/raw_def/v11.rs | 269 ++++++++++ crates/lib/src/deployment.rs | 234 +++++++++ crates/lib/src/deployment/tests.rs | 128 +++++ crates/lib/src/lib.rs | 3 + crates/oci/Cargo.toml | 20 + crates/oci/src/layers.rs | 350 +++++++++++++ crates/oci/src/layers/tests.rs | 130 +++++ crates/oci/src/lib.rs | 377 ++++++++++++++ crates/oci/src/tests.rs | 166 +++++++ crates/schema/src/auto_migrate.rs | 27 + crates/schema/src/auto_migrate/formatter.rs | 9 + .../src/auto_migrate/termcolor_formatter.rs | 9 + crates/schema/src/def.rs | 399 +++++++++++++-- crates/schema/src/def/validate.rs | 1 + crates/schema/src/def/validate/v10.rs | 5 +- crates/schema/src/def/validate/v11.rs | 353 +++++++++++++ crates/schema/src/def/validate/v9.rs | 7 +- crates/schema/src/error.rs | 8 + crates/schema/src/schema.rs | 4 +- .../tests/smoketests/http_routes.rs | 2 +- .../src/subcommands/extract_schema.rs | 4 +- crates/testing/Cargo.toml | 2 + crates/testing/tests/hosted_invocation.rs | 261 ++++++++++ modules/hosted-auth-test/Cargo.toml | 13 + modules/hosted-auth-test/src/lib.rs | 141 ++++++ modules/module-test-ts/src/index.ts | 2 +- modules/module-test/src/lib.rs | 2 +- modules/sdk-test-procedure-ts/src/index.ts | 2 +- modules/sdk-test-procedure/src/lib.rs | 2 +- .../procedure-client/src/test_handlers.rs | 12 +- 157 files changed, 8702 insertions(+), 815 deletions(-) create mode 100644 crates/auth/src/hosted.rs create mode 100644 crates/bindings-cpp/include/spacetimedb/function_visibility.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibilityV11.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11Section.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV11.g.h create mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV11.g.h create mode 100644 crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp create mode 100644 crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp create mode 100644 crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs create mode 100644 crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibilityV11.g.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11.g.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11Section.g.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/RawProcedureDefV11.g.cs create mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/RawReducerDefV11.g.cs create mode 100644 crates/bindings-typescript/src/server/function_visibility.ts create mode 100644 crates/bindings-typescript/tests/hosted_auth.test.ts create mode 100644 crates/bindings/tests/pass/function_visibility.rs delete mode 100644 crates/cli/src/subcommands/sidecar.rs create mode 100644 crates/core/src/auth/hosted_tokens.rs create mode 100644 crates/core/src/auth/invocation.rs create mode 100644 crates/core/src/auth/invocation/tests.rs create mode 100644 crates/core/src/db/deployment.rs create mode 100644 crates/core/src/db/deployment/tests.rs create mode 100644 crates/datastore/src/system_tables/deployment.rs create mode 100644 crates/lib/src/container.rs create mode 100644 crates/lib/src/container/tests.rs create mode 100644 crates/lib/src/db/raw_def/v11.rs create mode 100644 crates/lib/src/deployment.rs create mode 100644 crates/lib/src/deployment/tests.rs create mode 100644 crates/oci/Cargo.toml create mode 100644 crates/oci/src/layers.rs create mode 100644 crates/oci/src/layers/tests.rs create mode 100644 crates/oci/src/lib.rs create mode 100644 crates/oci/src/tests.rs create mode 100644 crates/schema/src/def/validate/v11.rs create mode 100644 crates/testing/tests/hosted_invocation.rs create mode 100644 modules/hosted-auth-test/Cargo.toml create mode 100644 modules/hosted-auth-test/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 862a2f4d67c..1e0387e5b94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2893,6 +2893,13 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "hosted-auth-test" +version = "0.0.0" +dependencies = [ + "spacetimedb 2.3.0", +] + [[package]] name = "hostname" version = "0.3.1" @@ -8369,6 +8376,20 @@ dependencies = [ "prometheus", ] +[[package]] +name = "spacetimedb-oci" +version = "2.3.0" +dependencies = [ + "anyhow", + "flate2", + "serde", + "serde_json", + "sha2", + "spacetimedb-lib 2.3.0", + "tar", + "zstd", +] + [[package]] name = "spacetimedb-paths" version = "2.3.0" @@ -8775,11 +8796,13 @@ dependencies = [ "serde", "serde_json", "serial_test", + "spacetimedb-auth", "spacetimedb-cli", "spacetimedb-client-api", "spacetimedb-client-api-messages", "spacetimedb-core", "spacetimedb-data-structures", + "spacetimedb-datastore", "spacetimedb-lib 2.3.0", "spacetimedb-paths", "spacetimedb-schema", diff --git a/Cargo.toml b/Cargo.toml index 2968240b375..2e1417f1322 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "crates/guard", "crates/fs-utils", "crates/lib", + "crates/oci", "crates/metrics", "crates/paths", "crates/pg", @@ -42,6 +43,7 @@ members = [ "modules/keynote-benchmarks", "modules/perf-test", "modules/module-test", + "modules/hosted-auth-test", "templates/basic-rs/spacetimedb", "templates/chat-console-rs/spacetimedb", "modules/sdk-test", diff --git a/crates/auth/src/hosted.rs b/crates/auth/src/hosted.rs new file mode 100644 index 00000000000..97e8ec32362 --- /dev/null +++ b/crates/auth/src/hosted.rs @@ -0,0 +1,322 @@ +//! Target-bound credentials for a database's hosted container. +//! +//! Decoded claims are untrusted. Only signature verification against an explicitly +//! configured platform issuer and comparison with an authoritative instance/grant +//! binding can produce [`VerifiedHostedAuth`]. Receiving hosts must additionally +//! recheck their durable target fence at each transaction and subscription admission. + +use crate::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; +use anyhow::{bail, ensure, Context}; +use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, EncodingKey, Header, Validation}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::Identity; +use std::fmt; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +pub const HOSTED_TOKEN_KIND: &str = "spacetimedb_hosted_v1"; +pub const HOSTED_TOKEN_TYPE: &str = "spacetimedb-hosted+jwt"; +pub const MAX_HOSTED_TOKEN_LIFETIME: Duration = Duration::from_secs(30); +pub const MAX_HOSTED_TOKEN_BYTES: usize = 8192; + +/// Wire claims, deliberately distinct from ordinary issuer/subject-derived Identity claims. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostedTokenClaims { + pub kind: Box, + #[serde(rename = "iss")] + pub issuer: Box, + #[serde(rename = "sub")] + pub subject: Box, + #[serde(with = "identity_hex")] + pub source_database: Identity, + /// A scalar, canonical database Identity. Lists and database names are not accepted. + #[serde(rename = "aud", with = "identity_hex")] + pub target_database: Identity, + pub generation: u64, + pub grant_revision: u64, + #[serde(rename = "iat")] + pub issued_at: u64, + #[serde(rename = "exp")] + pub expires_at: u64, + #[serde(rename = "jti")] + pub token_id: Box, +} + +/// Trusted input obtained from current source registration, placement and target grant state. +/// Never construct this by copying the incoming token's claims. Admission must be open, +/// and the assigned node/incarnation and required module capability must already be checked. +#[derive(Clone, Copy, Debug)] +pub struct HostedTokenBinding { + pub source_database: Identity, + pub target_database: Identity, + pub generation: u64, + pub grant_revision: u64, + pub lease_expires_at: SystemTime, +} + +/// Authentication proof. It cannot be deserialized or constructed from decoded claims. +#[derive(Clone)] +pub struct VerifiedHostedAuth { + claims: HostedTokenClaims, +} + +impl fmt::Debug for VerifiedHostedAuth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VerifiedHostedAuth") + .field("source_database", &self.source_database()) + .field("target_database", &self.target_database()) + .field("generation", &self.generation()) + .field("grant_revision", &self.grant_revision()) + .field("expires_at", &self.expires_at()) + .finish_non_exhaustive() + } +} + +impl VerifiedHostedAuth { + pub fn source_database(&self) -> Identity { + self.claims.source_database + } + pub fn target_database(&self) -> Identity { + self.claims.target_database + } + pub fn generation(&self) -> u64 { + self.claims.generation + } + pub fn grant_revision(&self) -> u64 { + self.claims.grant_revision + } + pub fn issued_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.claims.issued_at) + } + pub fn expires_at(&self) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(self.claims.expires_at) + } + pub fn token_id(&self) -> &str { + &self.claims.token_id + } + pub fn issuer(&self) -> &str { + &self.claims.issuer + } + pub fn is_internal(&self) -> bool { + self.source_database() == self.target_database() + } + + /// Expiry has no positive leeway. This does not replace generation/grant fencing. + pub fn check_at(&self, now: SystemTime) -> anyhow::Result<()> { + ensure!(now >= self.issued_at(), "hosted credential is not yet valid"); + ensure!(now < self.expires_at(), "hosted credential expired"); + Ok(()) + } + + pub fn into_connection_auth(self) -> anyhow::Result { + // Keep the actual claims, including the source/target/generation restrictions. + // Normalizing JSON whitespace does not alter any signed claim values. + let jwt_payload = serde_json::to_string(&self.claims)?.into_boxed_str(); + let mut extra = serde_json::to_value(&self.claims)?; + let extra = extra.as_object_mut().expect("hosted claims serialize as an object"); + for key in ["iss", "sub", "aud", "iat", "exp"] { + extra.remove(key); + } + let claims = SpacetimeIdentityClaims { + identity: self.source_database(), + subject: self.claims.subject.clone(), + issuer: self.claims.issuer.clone(), + audience: [self.target_database().to_hex().to_string().into_boxed_str()].into(), + iat: self.issued_at(), + exp: Some(self.expires_at()), + extra: Some( + extra + .iter() + .map(|(key, value)| (key.clone().into_boxed_str(), value.clone())) + .collect(), + ), + }; + Ok(ConnectionAuthCtx { + claims, + jwt_payload, + hosted: Some(self), + }) + } +} + +/// Classifies the reserved namespace only. A positive result grants no authority. +/// All reserved versions are rejected by ordinary OIDC validation and token exchange. +pub fn has_reserved_hosted_token_kind(token: &str) -> anyhow::Result { + let header = decode_header(token)?; + if header.typ.as_deref().is_some_and(is_reserved_hosted_type) { + return Ok(true); + } + let mut validation = Validation::new(Algorithm::ES256); + validation.required_spec_claims.clear(); + validation.validate_exp = false; + validation.validate_aud = false; + validation.insecure_disable_signature_validation(); + let data = decode::(token, &DecodingKey::from_secret(b"classification-only"), &validation)?; + Ok(data + .claims + .get("kind") + .and_then(serde_json::Value::as_str) + .is_some_and(is_reserved_hosted_kind)) +} + +pub fn is_reserved_hosted_kind(kind: &str) -> bool { + kind.starts_with("spacetimedb_hosted_") +} +fn is_reserved_hosted_type(kind: &str) -> bool { + kind.starts_with("spacetimedb-hosted") +} + +/// Decode routing hints only, never authentication. The caller must use these hints to +/// find trusted registration/grant state and then call [`verify_hosted_token`]. +pub fn unverified_hosted_token_claims(token: &str) -> anyhow::Result { + ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); + let mut validation = Validation::new(Algorithm::ES256); + validation.required_spec_claims.clear(); + validation.validate_exp = false; + validation.validate_aud = false; + validation.insecure_disable_signature_validation(); + Ok(decode::(token, &DecodingKey::from_secret(b"routing-only"), &validation)?.claims) +} + +/// Verify against a configured key and issuer, never a JWT-supplied key or JWKS URL. +/// `binding` must be authoritative state for that issuer's registered source database. +pub fn verify_hosted_token( + token: &str, + public_key: &DecodingKey, + trusted_issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, +) -> anyhow::Result { + ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); + let header = decode_header(token)?; + ensure!(header.alg == Algorithm::ES256, "hosted credential requires ES256"); + ensure!( + header.typ.as_deref() == Some(HOSTED_TOKEN_TYPE), + "invalid hosted credential type" + ); + let mut validation = Validation::new(Algorithm::ES256); + validation.set_required_spec_claims(&["iss", "sub", "aud", "exp"]); + validation.set_issuer(&[trusted_issuer]); + validation.set_audience(&[binding.target_database.to_hex().to_string()]); + validation.leeway = 0; + // Check time below against the caller's trusted clock, including exact expiry equality. + validation.validate_exp = false; + let claims = decode::(token, public_key, &validation)?.claims; + validate_claims(&claims, trusted_issuer, binding, now)?; + Ok(VerifiedHostedAuth { claims }) +} + +/// Mint from the broker's authoritative binding, with no guest-selected sender or generation. +pub fn sign_hosted_token( + private_key: &EncodingKey, + trusted_issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, + expires_at: SystemTime, + token_id: &str, +) -> anyhow::Result { + let claims = HostedTokenClaims { + kind: HOSTED_TOKEN_KIND.into(), + issuer: trusted_issuer.into(), + subject: binding.source_database.to_hex().to_string().into_boxed_str(), + source_database: binding.source_database, + target_database: binding.target_database, + generation: binding.generation, + grant_revision: binding.grant_revision, + issued_at: unix_seconds(now)?, + expires_at: unix_seconds(expires_at)?, + token_id: token_id.into(), + }; + validate_claims(&claims, trusted_issuer, binding, now)?; + let mut header = Header::new(Algorithm::ES256); + header.typ = Some(HOSTED_TOKEN_TYPE.into()); + Ok(jsonwebtoken::encode(&header, &claims, private_key)?) +} + +fn validate_claims( + claims: &HostedTokenClaims, + issuer: &str, + binding: &HostedTokenBinding, + now: SystemTime, +) -> anyhow::Result<()> { + ensure!( + !issuer.is_empty() && issuer.len() <= 128, + "invalid trusted hosted issuer" + ); + ensure!( + claims.kind.as_ref() == HOSTED_TOKEN_KIND, + "unsupported hosted credential kind" + ); + ensure!(claims.issuer.as_ref() == issuer, "untrusted hosted credential issuer"); + ensure!( + claims.source_database == binding.source_database, + "hosted credential source mismatch" + ); + ensure!( + claims.target_database == binding.target_database, + "hosted credential target mismatch" + ); + ensure!( + claims.subject.as_ref() == claims.source_database.to_hex().as_str(), + "hosted credential subject mismatch" + ); + ensure!( + claims.generation == binding.generation, + "hosted credential generation mismatch" + ); + ensure!( + claims.grant_revision == binding.grant_revision, + "hosted credential grant revision mismatch" + ); + ensure!( + !claims.token_id.is_empty() + && claims.token_id.len() <= 128 + && claims + .token_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_'), + "invalid hosted credential token ID" + ); + let Some(lifetime) = claims.expires_at.checked_sub(claims.issued_at) else { + bail!("invalid hosted credential lifetime") + }; + ensure!( + lifetime > 0 && lifetime <= MAX_HOSTED_TOKEN_LIFETIME.as_secs(), + "hosted credential lifetime exceeds limit" + ); + let issued_at = UNIX_EPOCH + .checked_add(Duration::from_secs(claims.issued_at)) + .context("invalid hosted issue time")?; + let expires_at = UNIX_EPOCH + .checked_add(Duration::from_secs(claims.expires_at)) + .context("invalid hosted expiry")?; + ensure!( + issued_at <= now && now < expires_at, + "hosted credential outside validity interval" + ); + ensure!( + expires_at <= binding.lease_expires_at, + "hosted credential exceeds confirmed lease" + ); + Ok(()) +} + +fn unix_seconds(time: SystemTime) -> anyhow::Result { + Ok(time.duration_since(UNIX_EPOCH)?.as_secs()) +} + +mod identity_hex { + use super::*; + pub fn serialize(value: &Identity, serializer: S) -> Result { + serializer.serialize_str(value.to_hex().as_str()) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let value = String::deserialize(deserializer)?; + if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(serde::de::Error::custom( + "expected canonical 64-character lowercase database Identity", + )); + } + Identity::from_hex(value).map_err(serde::de::Error::custom) + } +} diff --git a/crates/auth/src/identity.rs b/crates/auth/src/identity.rs index 5ecbe61dec7..960187f72b2 100644 --- a/crates/auth/src/identity.rs +++ b/crates/auth/src/identity.rs @@ -7,10 +7,20 @@ use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::Identity; use std::time::SystemTime; -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ConnectionAuthCtx { pub claims: SpacetimeIdentityClaims, pub jwt_payload: Box, + pub hosted: Option, +} + +impl std::fmt::Debug for ConnectionAuthCtx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConnectionAuthCtx") + .field("identity", &self.claims.identity) + .field("hosted", &self.hosted) + .finish_non_exhaustive() + } } impl TryFrom for ConnectionAuthCtx { @@ -20,6 +30,7 @@ impl TryFrom for ConnectionAuthCtx { Ok(ConnectionAuthCtx { claims, jwt_payload: payload.into(), + hosted: None, }) } } @@ -98,6 +109,15 @@ impl TryInto for IncomingClaims { type Error = anyhow::Error; fn try_into(self) -> anyhow::Result { + if self + .extra + .as_ref() + .and_then(|extra| extra.get("kind")) + .and_then(serde_json::Value::as_str) + .is_some_and(crate::hosted::is_reserved_hosted_kind) + { + anyhow::bail!("hosted credentials require dedicated target-bound validation"); + } // The issuer and subject must be less than 128 bytes. if self.issuer.len() > 128 { return Err(anyhow::anyhow!("Issuer too long: {:?}", self.issuer)); diff --git a/crates/auth/src/lib.rs b/crates/auth/src/lib.rs index db53a0c9064..b8b0976f40d 100644 --- a/crates/auth/src/lib.rs +++ b/crates/auth/src/lib.rs @@ -1 +1,2 @@ +pub mod hosted; pub mod identity; diff --git a/crates/bindings-cpp/README.md b/crates/bindings-cpp/README.md index ef31361c10b..bedb5f515d8 100644 --- a/crates/bindings-cpp/README.md +++ b/crates/bindings-cpp/README.md @@ -2,6 +2,30 @@ The SpacetimeDB C++ Module Library provides a modern C++20 API for building SpacetimeDB modules that run inside the database as WebAssembly. +## Function visibility and invocation authentication + +Apply `SPACETIMEDB_FUNCTION_VISIBILITY(name, Public)`, `Private`, or `Internal` +to a reducer or procedure after its definition: + +```cpp +SPACETIMEDB_REDUCER(process_jobs, ReducerContext ctx) { + return Ok(); +} +SPACETIMEDB_FUNCTION_VISIBILITY(process_jobs, Internal); +``` + +Omission means public for ordinary functions and private for scheduled functions. +An explicit choice is preserved when the function is scheduled. Lifecycle +reducers permit only omission or `Internal` and can only run for their host +lifecycle event. Internal functions require verified internal authority. Private +functions also admit the owner, and public functions admit any client. + +`ctx.sender_auth().is_internal()` captures the host's invocation authority. It is +independent of connection and JWT presence, so an internal call can have a JWT. +JWT identity is the verified sender supplied by the host. Procedures preserve +this authentication in `with_tx` and `try_with_tx`. Newly compiled modules emit +schema V11 and advertise `hosted_auth_v1`, requiring a compatible host. + ## Current State This library provides a production-ready C++ bindings for SpacetimeDB with complete type system support: @@ -274,4 +298,3 @@ See the `modules/*-cpp/src/` directory for example modules: ## Contributing This library is part of the SpacetimeDB project. Please see the main repository for contribution guidelines. - diff --git a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h index 23064a518f2..d9ce82b6aca 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h @@ -73,6 +73,7 @@ using ::identity; // ===== JWT ===== using ::get_jwt; +using ::get_call_auth_flags; // ===== Procedure Transactions ===== #ifdef SPACETIMEDB_UNSTABLE_FEATURES diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index e1aa12ac2de..91066510861 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -39,6 +39,9 @@ #define STDB_IMPORT_10_5(name) \ __attribute__((import_module("spacetime_10.5"), import_name(#name))) extern +#define STDB_IMPORT_10_6(name) \ + __attribute__((import_module("spacetime_10.6"), import_name(#name))) extern + // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; using SpacetimeDB::TableId; @@ -59,6 +62,10 @@ using SpacetimeDB::ConsoleTimerId; extern "C" { +// Verified invocation authority. Bit 0 is INTERNAL; JWT presence is independent. +STDB_IMPORT_10_6(get_call_auth_flags) +uint32_t get_call_auth_flags(); + // ===== Table and Index Management ===== STDB_IMPORT(table_id_from_name) Status table_id_from_name(const uint8_t* name_ptr, size_t name_len, TableId* out); diff --git a/crates/bindings-cpp/include/spacetimedb/auth_ctx.h b/crates/bindings-cpp/include/spacetimedb/auth_ctx.h index 00a4898e020..a8ae7fc62c9 100644 --- a/crates/bindings-cpp/include/spacetimedb/auth_ctx.h +++ b/crates/bindings-cpp/include/spacetimedb/auth_ctx.h @@ -28,22 +28,25 @@ struct ConnectionId; class AuthCtx { private: bool is_internal_; + std::optional verified_sender_; mutable std::shared_ptr> jwt_; std::function()> jwt_loader_; // Private constructor used by factory methods - AuthCtx(bool is_internal, std::function()> loader); + AuthCtx(bool is_internal, std::function()> loader, + std::optional verified_sender = std::nullopt); + static AuthCtx from_connection_with_flags(ConnectionId connection_id, Identity sender, uint32_t flags); public: /** * @brief Creates an AuthCtx from an optional ConnectionId. * * If the connection_id is present, creates an AuthCtx that will load the JWT. - * If the connection_id is absent, creates an internal AuthCtx. + * Internal authority is captured from the host, independently of connection presence. * * @param connection_id Optional connection ID - * @param sender The identity of the caller (already derived from JWT claims by the host) - * @return An AuthCtx based on the connection_id + * @param sender The verified caller Identity supplied by the host + * @return An AuthCtx with captured invocation authority and lazy JWT loading */ static AuthCtx from_connection_id_opt(std::optional connection_id, Identity sender); @@ -63,11 +66,10 @@ class AuthCtx { * This is primarily used for testing purposes, allowing you to create * an AuthCtx with specific JWT claims without needing a real connection. * - * Note: The Identity must be computed by calling the host function, - * as we cannot compute Blake3 hashes in WASM. + * The Identity must be the verified sender supplied by the host. * * @param jwt_payload The raw JWT payload (JSON claims) - * @param identity The identity derived from the JWT's issuer and subject + * @param identity The verified sender Identity * @return An AuthCtx with the provided JWT */ static AuthCtx from_jwt_payload(std::string jwt_payload, Identity identity); @@ -76,11 +78,10 @@ class AuthCtx { * @brief Creates an AuthCtx that reads the JWT for the given connection ID. * * The JWT will be lazily loaded from the host when first accessed. - * The identity parameter is the sender's identity, already derived from - * JWT claims by the host (using Blake3 hashing). + * The identity parameter is the verified sender supplied by the host. * * @param connection_id The connection ID to load the JWT for - * @param sender The identity of the caller (already derived from JWT claims by the host) + * @param sender The verified sender Identity supplied by the host * @return An AuthCtx that will load the JWT on demand */ static AuthCtx from_connection_id(ConnectionId connection_id, Identity sender); @@ -93,9 +94,9 @@ class AuthCtx { bool is_internal() const { return is_internal_; } /** - * @brief Checks if there is a JWT without loading it. + * @brief Checks if there is a JWT, loading it lazily if necessary. * - * If is_internal() returns true, this will return false. + * Independent of is_internal(). Internal calls can also have a JWT. * * @return true if a JWT is available */ @@ -113,9 +114,8 @@ class AuthCtx { /** * @brief Gets the caller's identity. * - * For internal calls, this returns the database's identity. - * For external calls, this returns the identity derived from the JWT - * (based on the issuer and subject claims). + * Returns the verified sender captured when constructing the context, + * independently of JWT presence or token claims. * * @return The caller's Identity */ @@ -126,16 +126,16 @@ class AuthCtx { // INLINE IMPLEMENTATIONS // ============================================================================ -constexpr uint16_t ERROR_BUFFER_TOO_SMALL = 11; - -inline AuthCtx::AuthCtx(bool is_internal, std::function()> loader) - : is_internal_(is_internal), jwt_loader_(std::move(loader)) {} +inline AuthCtx::AuthCtx(bool is_internal, std::function()> loader, + std::optional verified_sender) + : is_internal_(is_internal), verified_sender_(std::move(verified_sender)), jwt_loader_(std::move(loader)) {} inline AuthCtx AuthCtx::from_connection_id_opt(std::optional connection_id, Identity sender) { + const auto flags = FFI::get_call_auth_flags(); if (connection_id.has_value()) { - return from_connection_id(*connection_id, std::move(sender)); + return from_connection_with_flags(*connection_id, std::move(sender), flags); } else { - return internal(); + return AuthCtx((flags & 1) != 0, []() -> std::optional { return std::nullopt; }, sender); } } @@ -144,13 +144,17 @@ inline AuthCtx AuthCtx::internal() { } inline AuthCtx AuthCtx::from_jwt_payload(std::string jwt_payload, Identity identity) { - return AuthCtx(false, [payload = std::move(jwt_payload), id = std::move(identity)]() mutable -> std::optional { + return AuthCtx(false, [payload = std::move(jwt_payload), id = identity]() mutable -> std::optional { return JwtClaims(std::move(payload), std::move(id)); - }); + }, identity); } inline AuthCtx AuthCtx::from_connection_id(ConnectionId connection_id, Identity sender) { - return AuthCtx(false, [connection_id, sender]() -> std::optional { + return from_connection_with_flags(connection_id, std::move(sender), FFI::get_call_auth_flags()); +} + +inline AuthCtx AuthCtx::from_connection_with_flags(ConnectionId connection_id, Identity sender, uint32_t flags) { + return AuthCtx((flags & 1) != 0, [connection_id, sender]() -> std::optional { // Call the host FFI to get the JWT BytesSource jwt_source; @@ -169,35 +173,24 @@ inline AuthCtx AuthCtx::from_connection_id(ConnectionId connection_id, Identity } // Read the JWT payload from the BytesSource - std::vector buffer; - buffer.resize(4096); // Start with 4KB buffer - - size_t buffer_len = buffer.size(); - int16_t result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); - - while (result == ERROR_BUFFER_TOO_SMALL) { - buffer.resize(buffer.size() * 2); - buffer_len = buffer.size(); - result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); + std::array buffer; + std::string jwt_payload; + for (;;) { + size_t buffer_len = buffer.size(); + const auto result = bytes_source_read(jwt_source, buffer.data(), &buffer_len); + if (result != 0 && result != -1) return std::nullopt; + jwt_payload.append(reinterpret_cast(buffer.data()), buffer_len); + // -1 is successful exhaustion and may include the final payload bytes. + if (result == -1) break; + if (buffer_len == 0) return std::nullopt; } - - if (result < 0) { - return std::nullopt; - } - - // Convert bytes to string - std::string jwt_payload(buffer.begin(), buffer.begin() + buffer_len); - - // Use the provided sender identity (already computed by host from JWT claims) + if (jwt_payload.empty()) return std::nullopt; + // Token claims cannot override the verified sender, including hosted tokens. return JwtClaims(std::move(jwt_payload), sender); - }); + }, sender); } inline bool AuthCtx::has_jwt() const { - if (is_internal_) { - return false; - } - // Load the JWT if not already loaded, then check if it has a value // This ensures has_jwt() and get_jwt() are consistent return get_jwt().has_value(); @@ -211,6 +204,7 @@ inline const std::optional& AuthCtx::get_jwt() const { } inline Identity AuthCtx::get_caller_identity() const { + if (verified_sender_.has_value()) return *verified_sender_; if (is_internal_) { // Return database identity for internal calls std::array identity_bytes; diff --git a/crates/bindings-cpp/include/spacetimedb/function_visibility.h b/crates/bindings-cpp/include/spacetimedb/function_visibility.h new file mode 100644 index 00000000000..9bd36e19e48 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/function_visibility.h @@ -0,0 +1,6 @@ +#pragma once + +namespace SpacetimeDB { +// Omission preserves the host default: Public ordinarily, Private when scheduled. +enum class FunctionVisibility { Public, Private, Internal }; +} diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibilityV11.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibilityV11.g.h new file mode 100644 index 00000000000..e8f1ffadaa7 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibilityV11.g.h @@ -0,0 +1,23 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { + +enum class FunctionVisibilityV11 : uint8_t { + Private = 0, + ClientCallable = 1, + Internal = 2, +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h index 494243dd470..654260b3532 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h @@ -12,9 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawColumnDefaultValueV9.g.h" #include "RawProcedureDefV9.g.h" #include "RawViewDefV9.g.h" +#include "RawColumnDefaultValueV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h index c7f144eb07d..6987a9edd0e 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h @@ -12,11 +12,12 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "RawModuleDefV11.g.h" #include "RawModuleDefV8.g.h" #include "RawModuleDefV9.g.h" #include "RawModuleDefV10.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDef, SpacetimeDB::Internal::RawModuleDefV8, SpacetimeDB::Internal::RawModuleDefV9, SpacetimeDB::Internal::RawModuleDefV10) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDef, SpacetimeDB::Internal::RawModuleDefV8, SpacetimeDB::Internal::RawModuleDefV9, SpacetimeDB::Internal::RawModuleDefV10, SpacetimeDB::Internal::RawModuleDefV11) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index 1efcad29ed5..7b8a01cdb51 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -12,19 +12,19 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "ExplicitNames.g.h" +#include "RawHttpRouteDefV10.g.h" +#include "RawTypeDefV10.g.h" +#include "Typespace.g.h" +#include "RawReducerDefV10.g.h" #include "RawProcedureDefV10.g.h" -#include "CaseConversionPolicy.g.h" +#include "RawViewDefV10.g.h" #include "RawLifeCycleReducerDefV10.g.h" -#include "RawReducerDefV10.g.h" #include "RawHttpHandlerDefV10.g.h" -#include "RawTypeDefV10.g.h" -#include "ExplicitNames.g.h" -#include "RawViewDefV10.g.h" -#include "RawScheduleDefV10.g.h" -#include "Typespace.g.h" #include "RawTableDefV10.g.h" +#include "RawScheduleDefV10.g.h" #include "RawRowLevelSecurityDefV9.g.h" -#include "RawHttpRouteDefV10.g.h" +#include "CaseConversionPolicy.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11.g.h new file mode 100644 index 00000000000..c787ba63fb4 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11.g.h @@ -0,0 +1,27 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" +#include "RawModuleDefV11Section.g.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawModuleDefV11) { + std::vector sections; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, sections); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(sections) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11Section.g.h new file mode 100644 index 00000000000..359648e5304 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11Section.g.h @@ -0,0 +1,32 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" +#include "RawProcedureDefV11.g.h" +#include "RawTableDefV10.g.h" +#include "RawScheduleDefV10.g.h" +#include "CaseConversionPolicy.g.h" +#include "ExplicitNames.g.h" +#include "RawHttpHandlerDefV10.g.h" +#include "RawHttpRouteDefV10.g.h" +#include "RawTypeDefV10.g.h" +#include "RawReducerDefV11.g.h" +#include "Typespace.g.h" +#include "RawLifeCycleReducerDefV10.g.h" +#include "RawViewDefV10.g.h" +#include "RawRowLevelSecurityDefV9.g.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV11Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector) +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h index 6936f2f32c5..f9c5ec18db2 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h @@ -12,10 +12,10 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "MiscModuleExport.g.h" +#include "Typespace.g.h" #include "ReducerDef.g.h" #include "TableDesc.g.h" -#include "Typespace.g.h" +#include "MiscModuleExport.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h index cf6881a9bb2..a64a4bf380c 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h @@ -13,11 +13,11 @@ #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" #include "RawTableDefV9.g.h" -#include "RawRowLevelSecurityDefV9.g.h" #include "Typespace.g.h" #include "RawReducerDefV9.g.h" -#include "RawTypeDefV9.g.h" +#include "RawRowLevelSecurityDefV9.g.h" #include "RawMiscModuleExportV9.g.h" +#include "RawTypeDefV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h index dc84b35e602..f316264fc5c 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h @@ -12,9 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "AlgebraicType.g.h" #include "FunctionVisibility.g.h" #include "ProductType.g.h" +#include "AlgebraicType.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV11.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV11.g.h new file mode 100644 index 00000000000..2ac5d359352 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV11.g.h @@ -0,0 +1,35 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" +#include "FunctionVisibilityV11.g.h" +#include "ProductType.g.h" +#include "AlgebraicType.g.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawProcedureDefV11) { + std::string source_name; + SpacetimeDB::Internal::ProductType params; + std::optional declared_visibility; + SpacetimeDB::Internal::AlgebraicType return_type; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, source_name); + ::SpacetimeDB::bsatn::serialize(writer, params); + ::SpacetimeDB::bsatn::serialize(writer, declared_visibility); + ::SpacetimeDB::bsatn::serialize(writer, return_type); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(source_name, params, declared_visibility, return_type) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h index 667d9864a2a..a49d9d78970 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h @@ -12,8 +12,8 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "ProductType.g.h" #include "AlgebraicType.g.h" +#include "ProductType.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV11.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV11.g.h new file mode 100644 index 00000000000..1eee0f8fb76 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV11.g.h @@ -0,0 +1,37 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" +#include "ProductType.g.h" +#include "FunctionVisibilityV11.g.h" +#include "AlgebraicType.g.h" + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawReducerDefV11) { + std::string source_name; + SpacetimeDB::Internal::ProductType params; + std::optional declared_visibility; + SpacetimeDB::Internal::AlgebraicType ok_return_type; + SpacetimeDB::Internal::AlgebraicType err_return_type; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, source_name); + ::SpacetimeDB::bsatn::serialize(writer, params); + ::SpacetimeDB::bsatn::serialize(writer, declared_visibility); + ::SpacetimeDB::bsatn::serialize(writer, ok_return_type); + ::SpacetimeDB::bsatn::serialize(writer, err_return_type); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(source_name, params, declared_visibility, ok_return_type, err_return_type) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV9.g.h index 8121773a40d..964ed98df12 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV9.g.h @@ -12,8 +12,8 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "Lifecycle.g.h" #include "ProductType.g.h" +#include "Lifecycle.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h index 46fc7ca6ed1..a2db62eb7d1 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h @@ -13,11 +13,11 @@ #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" #include "TableAccess.g.h" -#include "RawSequenceDefV10.g.h" -#include "RawConstraintDefV10.g.h" +#include "RawColumnDefaultValueV10.g.h" #include "RawIndexDefV10.g.h" +#include "RawConstraintDefV10.g.h" #include "TableType.g.h" -#include "RawColumnDefaultValueV10.g.h" +#include "RawSequenceDefV10.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV8.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV8.g.h index 4a85aabd2f3..6b050914df0 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV8.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV8.g.h @@ -12,10 +12,10 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawIndexDefV8.g.h" +#include "RawColumnDefV8.g.h" #include "RawSequenceDefV8.g.h" #include "RawConstraintDefV8.g.h" -#include "RawColumnDefV8.g.h" +#include "RawIndexDefV8.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h index e817785f690..619da3f8d24 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h @@ -12,11 +12,11 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "RawConstraintDefV9.g.h" #include "TableType.g.h" +#include "RawIndexDefV9.g.h" #include "RawSequenceDefV9.g.h" #include "TableAccess.g.h" -#include "RawIndexDefV9.g.h" -#include "RawConstraintDefV9.g.h" #include "RawScheduleDefV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h index f746093c574..2c4dbdda684 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h @@ -13,6 +13,7 @@ #include #include "../bsatn/bsatn.h" #include "../database.h" +#include "../function_visibility.h" #include "autogen/CaseConversionPolicy.g.h" #include "autogen/ExplicitNameEntry.g.h" #include "autogen/NameMapping.g.h" @@ -20,11 +21,11 @@ #include "autogen/SumType.g.h" #include "autogen/ProductType.g.h" #include "autogen/ProductTypeElement.g.h" -#include "autogen/RawModuleDefV10.g.h" +#include "autogen/RawModuleDefV11.g.h" #include "autogen/Typespace.g.h" #include "autogen/RawTableDefV10.g.h" -#include "autogen/RawReducerDefV10.g.h" -#include "autogen/RawProcedureDefV10.g.h" +#include "autogen/RawReducerDefV11.g.h" +#include "autogen/RawProcedureDefV11.g.h" #include "autogen/RawViewDefV10.g.h" #include "autogen/RawScheduleDefV10.g.h" #include "autogen/RawLifeCycleReducerDefV10.g.h" @@ -47,6 +48,7 @@ void fail_reducer(std::string message); namespace Internal { +// The historical facade name is retained; newly compiled modules serialize V11. class V10Builder { public: V10Builder() = default; @@ -382,10 +384,10 @@ class V10Builder { }(std::make_index_sequence{}, params, param_names, type_reg); } - RawReducerDefV10 reducer_def{ + RawReducerDefV11 reducer_def{ reducer_name, std::move(params), - FunctionVisibility::ClientCallable, + std::nullopt, MakeUnitAlgebraicType(), MakeStringAlgebraicType(), }; @@ -432,10 +434,10 @@ class V10Builder { } RegisterReducerHandler(reducer_name, handler, lifecycle); - RawReducerDefV10 reducer_def{ + RawReducerDefV11 reducer_def{ reducer_name, ProductType{}, - FunctionVisibility::Private, + std::nullopt, MakeUnitAlgebraicType(), MakeStringAlgebraicType(), }; @@ -579,11 +581,11 @@ class V10Builder { }(std::make_index_sequence{}, params, param_names, type_reg); } - RawProcedureDefV10 procedure_def{ + RawProcedureDefV11 procedure_def{ procedure_name, std::move(params), + std::nullopt, return_type, - FunctionVisibility::ClientCallable, }; UpsertProcedure(procedure_def); } @@ -623,17 +625,18 @@ class V10Builder { void RegisterExplicitTableName(const std::string& source_name, const std::string& canonical_name); void RegisterExplicitFunctionName(const std::string& source_name, const std::string& canonical_name); + void SetFunctionVisibility(const std::string& source_name, ::SpacetimeDB::FunctionVisibility visibility); void RegisterExplicitIndexName(const std::string& source_name, const std::string& canonical_name); - RawModuleDefV10 BuildModuleDef() const; + RawModuleDefV11 BuildModuleDef() const; Typespace& GetTypespace() { return typespace_; } const Typespace& GetTypespace() const { return typespace_; } std::vector& GetTypeDefs() { return types_; } const std::vector& GetTypeDefs() const { return types_; } std::vector& GetTables() { return tables_; } const std::vector& GetTables() const { return tables_; } - std::vector& GetReducers() { return reducers_; } - const std::vector& GetReducers() const { return reducers_; } + std::vector& GetReducers() { return reducers_; } + const std::vector& GetReducers() const { return reducers_; } const std::optional& GetCaseConversionPolicy() const { return case_conversion_policy_; } const std::vector& GetExplicitNames() const { return explicit_names_; } const std::vector& GetHttpHandlers() const { return http_handlers_; } @@ -645,8 +648,8 @@ class V10Builder { } void UpsertTable(const RawTableDefV10& table); void UpsertLifecycleReducer(const RawLifeCycleReducerDefV10& lifecycle); - void UpsertReducer(const RawReducerDefV10& reducer); - void UpsertProcedure(const RawProcedureDefV10& procedure); + void UpsertReducer(const RawReducerDefV11& reducer); + void UpsertProcedure(const RawProcedureDefV11& procedure); void UpsertView(const RawViewDefV10& view); void UpsertHttpHandler(const RawHttpHandlerDefV10& handler); RawIndexDefV10 CreateBTreeIndex(const std::string& table_name, @@ -664,8 +667,8 @@ class V10Builder { std::vector explicit_names_; std::unordered_map> column_defaults_by_table_; std::vector tables_; - std::vector reducers_; - std::vector procedures_; + std::vector reducers_; + std::vector procedures_; std::vector views_; std::vector http_handlers_; std::vector http_routes_; @@ -685,4 +688,3 @@ V10Builder& getV10Builder(); } // namespace SpacetimeDB #endif // SPACETIMEDB_V10_BUILDER_H - diff --git a/crates/bindings-cpp/include/spacetimedb/jwt_claims.h b/crates/bindings-cpp/include/spacetimedb/jwt_claims.h index cdc9aef511d..6a72e973633 100644 --- a/crates/bindings-cpp/include/spacetimedb/jwt_claims.h +++ b/crates/bindings-cpp/include/spacetimedb/jwt_claims.h @@ -15,8 +15,8 @@ namespace SpacetimeDB { * This class provides lazy parsing of JWT claims, parsing specific fields * on demand. It follows the same pattern as the Rust and C# implementations. * - * The Identity is provided in the constructor because computing it requires - * Blake3 hashing, which is done on the host side. + * The Identity is the verified sender supplied by the host. Token claims + * cannot override it, including for hosted container credentials. */ class JwtClaims { private: @@ -36,11 +36,10 @@ class JwtClaims { /** * @brief Constructs a JwtClaims from a JWT payload and its associated Identity. * - * The Identity must be provided because computing it requires Blake3 hashing, - * which is performed on the host side. + * The Identity must be the verified sender supplied by the host. * * @param jwt_payload The raw JWT payload (JSON claims) - * @param identity The identity derived from the JWT's issuer and subject + * @param identity The verified sender Identity */ JwtClaims(std::string jwt_payload, Identity identity); @@ -71,8 +70,7 @@ class JwtClaims { /** * @brief Returns the identity for these credentials. * - * The identity is based on the 'iss' and 'sub' claims and is computed - * using Blake3 hashing on the host side. + * This is the verified sender supplied by the host, independently of claims. * * @return The identity */ diff --git a/crates/bindings-cpp/include/spacetimedb/macros.h b/crates/bindings-cpp/include/spacetimedb/macros.h index b0ca63b5336..2d7a23b8317 100644 --- a/crates/bindings-cpp/include/spacetimedb/macros.h +++ b/crates/bindings-cpp/include/spacetimedb/macros.h @@ -542,6 +542,15 @@ inline std::vector parseParameterNames(const std::string& param_lis // VISIBILITY FILTER MACRO // ============================================================================= +// Apply to a registered reducer or procedure. Runs after function registration; +// lifecycle reducers only accept Internal. Scheduling preserves this choice. +#define SPACETIMEDB_FUNCTION_VISIBILITY(function_name, visibility) \ + extern "C" __attribute__((export_name("__preinit__40_visibility_" #function_name))) \ + void CONCAT(__spacetimedb_function_visibility_, function_name)() { \ + ::SpacetimeDB::Internal::getV10Builder().SetFunctionVisibility( \ + #function_name, ::SpacetimeDB::FunctionVisibility::visibility); \ + } + /** * @brief Set module case conversion policy using a fixed preinit registration symbol. * @@ -782,4 +791,3 @@ inline std::vector parseParameterNames(const std::string& param_lis #endif // SPACETIMEDB_MACROS_H - diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index ea189f8ef12..8a146a82706 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -57,6 +57,7 @@ struct ProcedureContext { private: // Caller's identity - who invoked this procedure Identity sender_; + AuthCtx sender_auth_ = AuthCtx::internal(); public: // Timestamp when the procedure was invoked @@ -84,7 +85,11 @@ struct ProcedureContext { ProcedureContext() = default; ProcedureContext(Identity s, Timestamp t, ConnectionId conn_id) - : sender_(s), timestamp(t), connection_id(conn_id) {} + : sender_(s), sender_auth_(AuthCtx::from_connection_id_opt( + conn_id.id.low == 0 && conn_id.id.high == 0 ? std::nullopt : std::optional(conn_id), s)), + timestamp(t), connection_id(conn_id) {} + + const AuthCtx& sender_auth() const { return sender_auth_; } Identity sender() const { return sender_; @@ -100,7 +105,7 @@ struct ProcedureContext { * @code * auto module_id = ctx.database_identity(); * std::string url = "http://localhost:3000/v1/database/" + - * module_id.to_hex() + "/schema?version=9"; + * module_id.to_hex_string() + "/schema?version=11"; * @endcode */ Identity database_identity() const { @@ -200,8 +205,9 @@ struct ProcedureContext { auto make_reducer_ctx = [this](Timestamp tx_timestamp) { return ReducerContext( sender(), - std::optional(connection_id), - tx_timestamp + connection_id.id.low == 0 && connection_id.id.high == 0 ? std::nullopt : std::optional(connection_id), + tx_timestamp, + sender_auth_ ); }; return Internal::with_tx(make_reducer_ctx, body); @@ -232,8 +238,9 @@ struct ProcedureContext { auto make_reducer_ctx = [this](Timestamp tx_timestamp) { return ReducerContext( sender(), - std::optional(connection_id), - tx_timestamp + connection_id.id.low == 0 && connection_id.id.high == 0 ? std::nullopt : std::optional(connection_id), + tx_timestamp, + sender_auth_ ); }; return Internal::try_with_tx(make_reducer_ctx, body); diff --git a/crates/bindings-cpp/src/internal/Module.cpp b/crates/bindings-cpp/src/internal/Module.cpp index 901bb5e3e86..d1a751264c1 100644 --- a/crates/bindings-cpp/src/internal/Module.cpp +++ b/crates/bindings-cpp/src/internal/Module.cpp @@ -5,7 +5,7 @@ #include "spacetimedb/internal/Module.h" #include "spacetimedb/internal/buffer_pool.h" #include "spacetimedb/internal/autogen/RawModuleDef.g.h" -#include "spacetimedb/internal/autogen/RawModuleDefV10.g.h" +#include "spacetimedb/internal/autogen/RawModuleDefV11.g.h" #include "spacetimedb/internal/autogen/RawTypeDefV10.g.h" #include "spacetimedb/internal/v9_builder.h" #include "spacetimedb/internal/v10_builder.h" @@ -373,9 +373,9 @@ void __preinit__99_validate_types() { std::vector Internal::Module::SerializeModuleDef() { - RawModuleDefV10 v10_module = getV10Builder().BuildModuleDef(); + RawModuleDefV11 v11_module = getV10Builder().BuildModuleDef(); RawModuleDef versioned_module; - versioned_module.set<2>(std::move(v10_module)); + versioned_module.set<3>(std::move(v11_module)); std::vector buffer; bsatn::Writer writer(buffer); @@ -383,7 +383,7 @@ std::vector Internal::Module::SerializeModuleDef() { return buffer; } -// FFI export - V10 serialization +// FFI export - V11 serialization void Internal::Module::__describe_module__(BytesSink sink) { // The preinit functions should have already been called by SpacetimeDB // Including our validation preinit which checks for recursive types diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index eb22114e8b9..336a8151478 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -2,10 +2,10 @@ #include "spacetimedb/internal/autogen/AlgebraicType.g.h" #include "spacetimedb/internal/autogen/ProductType.g.h" #include "spacetimedb/internal/autogen/ProductTypeElement.g.h" -#include "spacetimedb/internal/autogen/RawModuleDefV10Section.g.h" +#include "spacetimedb/internal/autogen/RawModuleDefV11Section.g.h" #include "spacetimedb/internal/autogen/RawTypeDefV10.g.h" #include "spacetimedb/internal/autogen/RawScopedTypeNameV10.g.h" -#include "spacetimedb/internal/autogen/FunctionVisibility.g.h" +#include "spacetimedb/internal/autogen/FunctionVisibilityV11.g.h" #include "spacetimedb/internal/autogen/ExplicitNames.g.h" #include "spacetimedb/router.h" #include @@ -120,7 +120,7 @@ void V10Builder::UpsertLifecycleReducer(const RawLifeCycleReducerDefV10& lifecyc } } -void V10Builder::UpsertReducer(const RawReducerDefV10& reducer) { +void V10Builder::UpsertReducer(const RawReducerDefV11& reducer) { auto it = std::find_if(reducers_.begin(), reducers_.end(), [&](const auto& existing) { return existing.source_name == reducer.source_name; }); @@ -131,7 +131,7 @@ void V10Builder::UpsertReducer(const RawReducerDefV10& reducer) { } } -void V10Builder::UpsertProcedure(const RawProcedureDefV10& procedure) { +void V10Builder::UpsertProcedure(const RawProcedureDefV11& procedure) { auto it = std::find_if(procedures_.begin(), procedures_.end(), [&](const auto& existing) { return existing.source_name == procedure.source_name; }); @@ -209,98 +209,108 @@ RawConstraintDefV10 V10Builder::CreateUniqueConstraint(const std::string& table_ }; } -RawModuleDefV10 V10Builder::BuildModuleDef() const { - RawModuleDefV10 v10_module; - - std::vector types = types_; - - std::vector reducers = reducers_; - std::vector procedures = procedures_; - - std::unordered_set internal_functions; - for (const auto& lifecycle : lifecycle_reducers_) { - internal_functions.insert(lifecycle.function_name); - } - for (const auto& schedule : schedules_) { - internal_functions.insert(schedule.function_name); +void V10Builder::SetFunctionVisibility(const std::string& name, ::SpacetimeDB::FunctionVisibility visibility) { + FunctionVisibilityV11 declared; + switch (visibility) { + case ::SpacetimeDB::FunctionVisibility::Public: declared = FunctionVisibilityV11::ClientCallable; break; + case ::SpacetimeDB::FunctionVisibility::Private: declared = FunctionVisibilityV11::Private; break; + case ::SpacetimeDB::FunctionVisibility::Internal: declared = FunctionVisibilityV11::Internal; break; + default: + SetConstraintRegistrationError("INVALID_FUNCTION_VISIBILITY", "function='" + name + "'"); + return; } - for (auto& reducer : reducers) { - if (internal_functions.find(reducer.source_name) != internal_functions.end()) { - reducer.visibility = FunctionVisibility::Private; + for (const auto& lifecycle : lifecycle_reducers_) { + if (lifecycle.function_name == name && declared != FunctionVisibilityV11::Internal) { + SetConstraintRegistrationError("INVALID_LIFECYCLE_VISIBILITY", "function='" + name + "' must be Internal"); + return; } } - for (auto& procedure : procedures) { - if (internal_functions.find(procedure.source_name) != internal_functions.end()) { - procedure.visibility = FunctionVisibility::Private; - } + for (auto& reducer : reducers_) { + if (reducer.source_name == name) { reducer.declared_visibility = declared; return; } } + for (auto& procedure : procedures_) { + if (procedure.source_name == name) { procedure.declared_visibility = declared; return; } + } + SetConstraintRegistrationError("UNKNOWN_FUNCTION_VISIBILITY", "function='" + name + "' is not a reducer or procedure"); +} + +RawModuleDefV11 V10Builder::BuildModuleDef() const { + RawModuleDefV11 v11_module; + + std::vector types = types_; + + std::vector reducers = reducers_; + std::vector procedures = procedures_; - RawModuleDefV10Section section_typespace; + RawModuleDefV11Section section_typespace; section_typespace.set<0>(typespace_); - v10_module.sections.push_back(section_typespace); + v11_module.sections.push_back(section_typespace); + RawModuleDefV11Section capabilities; + capabilities.set<13>(std::vector{"hosted_auth_v1"}); + v11_module.sections.push_back(std::move(capabilities)); if (!types.empty()) { - RawModuleDefV10Section section_types; + RawModuleDefV11Section section_types; section_types.set<1>(std::move(types)); - v10_module.sections.push_back(std::move(section_types)); + v11_module.sections.push_back(std::move(section_types)); } if (!tables_.empty()) { - RawModuleDefV10Section section_tables; + RawModuleDefV11Section section_tables; section_tables.set<2>(tables_); - v10_module.sections.push_back(std::move(section_tables)); + v11_module.sections.push_back(std::move(section_tables)); } if (!reducers.empty()) { - RawModuleDefV10Section section_reducers; + RawModuleDefV11Section section_reducers; section_reducers.set<3>(std::move(reducers)); - v10_module.sections.push_back(std::move(section_reducers)); + v11_module.sections.push_back(std::move(section_reducers)); } if (!procedures.empty()) { - RawModuleDefV10Section section_procedures; + RawModuleDefV11Section section_procedures; section_procedures.set<4>(std::move(procedures)); - v10_module.sections.push_back(std::move(section_procedures)); + v11_module.sections.push_back(std::move(section_procedures)); } if (!views_.empty()) { - RawModuleDefV10Section section_views; + RawModuleDefV11Section section_views; section_views.set<5>(views_); - v10_module.sections.push_back(std::move(section_views)); + v11_module.sections.push_back(std::move(section_views)); } if (!schedules_.empty()) { - RawModuleDefV10Section section_schedules; + RawModuleDefV11Section section_schedules; section_schedules.set<6>(schedules_); - v10_module.sections.push_back(std::move(section_schedules)); + v11_module.sections.push_back(std::move(section_schedules)); } if (!lifecycle_reducers_.empty()) { - RawModuleDefV10Section section_lifecycle; + RawModuleDefV11Section section_lifecycle; section_lifecycle.set<7>(lifecycle_reducers_); - v10_module.sections.push_back(std::move(section_lifecycle)); + v11_module.sections.push_back(std::move(section_lifecycle)); } if (case_conversion_policy_.has_value()) { - RawModuleDefV10Section section_case_policy; + RawModuleDefV11Section section_case_policy; section_case_policy.set<9>(case_conversion_policy_.value()); - v10_module.sections.push_back(std::move(section_case_policy)); + v11_module.sections.push_back(std::move(section_case_policy)); } if (!explicit_names_.empty()) { - RawModuleDefV10Section section_explicit_names; + RawModuleDefV11Section section_explicit_names; section_explicit_names.set<10>(ExplicitNames{explicit_names_}); - v10_module.sections.push_back(std::move(section_explicit_names)); + v11_module.sections.push_back(std::move(section_explicit_names)); } if (!http_handlers_.empty()) { - RawModuleDefV10Section section_http_handlers; + RawModuleDefV11Section section_http_handlers; section_http_handlers.set<11>(http_handlers_); - v10_module.sections.push_back(std::move(section_http_handlers)); + v11_module.sections.push_back(std::move(section_http_handlers)); } if (!http_routes_.empty()) { - RawModuleDefV10Section section_http_routes; + RawModuleDefV11Section section_http_routes; section_http_routes.set<12>(http_routes_); - v10_module.sections.push_back(std::move(section_http_routes)); + v11_module.sections.push_back(std::move(section_http_routes)); } if (!row_level_security_.empty()) { - RawModuleDefV10Section section_rls; + RawModuleDefV11Section section_rls; section_rls.set<8>(row_level_security_); - v10_module.sections.push_back(std::move(section_rls)); + v11_module.sections.push_back(std::move(section_rls)); } - return v10_module; + return v11_module; } } // namespace Internal diff --git a/crates/bindings-cpp/tests/unit/CMakeLists.txt b/crates/bindings-cpp/tests/unit/CMakeLists.txt index 0ced4e0194c..2a540f96f31 100644 --- a/crates/bindings-cpp/tests/unit/CMakeLists.txt +++ b/crates/bindings-cpp/tests/unit/CMakeLists.txt @@ -11,6 +11,18 @@ endif() add_executable(bindings_cpp_unit_tests main.cpp http_unit_tests.cpp + hosted_auth_unit_tests.cpp + function_visibility_unit_tests.cpp +) + +# Exercise the real module builder without the standalone WASI shims, which +# replace the Node test runner's standard I/O and process lifecycle functions. +target_sources(bindings_cpp_unit_tests PRIVATE + ../../src/internal/Module.cpp + ../../src/internal/AlgebraicType.cpp + ../../src/internal/v9_builder.cpp + ../../src/internal/v10_builder.cpp + ../../src/internal/module_type_registration.cpp ) target_include_directories(bindings_cpp_unit_tests PRIVATE diff --git a/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp new file mode 100644 index 00000000000..7365c046baa --- /dev/null +++ b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp @@ -0,0 +1,74 @@ +#include "test_harness.h" +#include "spacetimedb/reducer_error.h" +#include "spacetimedb/procedure_context.h" +#include "spacetimedb/internal/v10_builder.h" +#include "spacetimedb/internal/autogen/RawModuleDef.g.h" +#include "spacetimedb/macros.h" + +using namespace SpacetimeDB; +using namespace SpacetimeDB::Internal; + +namespace { +ReducerResult noop(ReducerContext) { return Ok(); } +uint32_t procedure(ProcedureContext) { return 7; } +} + +SPACETIMEDB_FUNCTION_VISIBILITY(visibility_macro_target, Internal); + +TEST_CASE(visibility_macro_applies_after_function_registration) { + auto& builder = getV10Builder(); + builder.RegisterReducer("visibility_macro_target", &noop, {}); + __spacetimedb_function_visibility_visibility_macro_target(); + bool found = false; + for (const auto& section : builder.BuildModuleDef().sections) { + if (section.get_tag() != 3) continue; + for (const auto& reducer : section.get<3>()) { + if (reducer.source_name != "visibility_macro_target") continue; + ASSERT_EQ(FunctionVisibilityV11::Internal, *reducer.declared_visibility); + found = true; + } + } + ASSERT_TRUE(found); +} + +TEST_CASE(v11_retains_explicit_visibility_and_schedule_default_omission) { + V10Builder builder; + builder.RegisterReducer("omitted", &noop, {}); + builder.RegisterReducer("public", &noop, {}); + builder.RegisterReducer("private", &noop, {}); + builder.RegisterReducer("internal", &noop, {}); + builder.SetFunctionVisibility("public", SpacetimeDB::FunctionVisibility::Public); + builder.SetFunctionVisibility("private", SpacetimeDB::FunctionVisibility::Private); + builder.SetFunctionVisibility("internal", SpacetimeDB::FunctionVisibility::Internal); + builder.RegisterSchedule("jobs", 0, "public"); + builder.RegisterSchedule("other_jobs", 0, "omitted"); + builder.RegisterProcedure("procedure", &procedure); + builder.SetFunctionVisibility("procedure", SpacetimeDB::FunctionVisibility::Internal); + + RawModuleDef versioned; + versioned.set<3>(builder.BuildModuleDef()); + std::vector bytes; + bsatn::Writer writer(bytes); + bsatn::serialize(writer, versioned); + ASSERT_EQ(uint8_t{3}, bytes.at(0)); + ASSERT_EQ(uint8_t{3}, versioned.get_tag()); + bool saw_reducers = false, saw_procedure = false, saw_capability = false; + for (const auto& section : versioned.get<3>().sections) { + if (section.get_tag() == 3) { + const auto& reducers = section.get<3>(); + ASSERT_EQ(size_t{4}, reducers.size()); + ASSERT_TRUE(!reducers[0].declared_visibility.has_value()); + ASSERT_EQ(FunctionVisibilityV11::ClientCallable, *reducers[1].declared_visibility); + ASSERT_EQ(FunctionVisibilityV11::Private, *reducers[2].declared_visibility); + ASSERT_EQ(FunctionVisibilityV11::Internal, *reducers[3].declared_visibility); + saw_reducers = true; + } else if (section.get_tag() == 4) { + ASSERT_EQ(FunctionVisibilityV11::Internal, *section.get<4>().at(0).declared_visibility); + saw_procedure = true; + } else if (section.get_tag() == 13) { + ASSERT_EQ(std::vector{"hosted_auth_v1"}, section.get<13>()); + saw_capability = true; + } + } + ASSERT_TRUE(saw_reducers && saw_procedure && saw_capability); +} diff --git a/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp b/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp new file mode 100644 index 00000000000..914261554e2 --- /dev/null +++ b/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp @@ -0,0 +1,104 @@ +#include "test_harness.h" +#include "spacetimedb/procedure_context.h" + +#include +#include + +using namespace SpacetimeDB; + +namespace { +uint32_t auth_flags; +size_t flag_reads; +size_t jwt_reads; +size_t payload_offset; +std::string jwt_payload; + +Identity verified_sender() { + std::array bytes{}; + bytes[0] = 42; + return Identity(bytes); +} + +void reset_host(uint32_t flags, std::string payload = {}) { + auth_flags = flags; + flag_reads = jwt_reads = payload_offset = 0; + jwt_payload = std::move(payload); +} +} + +extern "C" uint32_t get_call_auth_flags() { + ++flag_reads; + return auth_flags; +} + +extern "C" Status get_jwt(const uint8_t*, BytesSource* out) { + ++jwt_reads; + payload_offset = 0; + *out = BytesSource{jwt_payload.empty() ? 0u : 1u}; + return Status{0}; +} + +extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* len) { + *len = std::min(*len, jwt_payload.size() - payload_offset); + std::memcpy(out, jwt_payload.data() + payload_offset, *len); + payload_offset += *len; + // Successful exhaustion can return the last bytes together with -1. + return payload_offset == jwt_payload.size() ? -1 : 0; +} + +extern "C" void identity(uint8_t* out) { std::memset(out, 0, 32); } +extern "C" Status procedure_start_mut_tx(int64_t* out) { *out = 0; return Status{0}; } +extern "C" Status procedure_commit_mut_tx() { return Status{0}; } +extern "C" Status procedure_abort_mut_tx() { return Status{0}; } +extern "C" void console_log(LogLevel, const uint8_t*, size_t, const uint8_t*, size_t, + uint32_t, const uint8_t*, size_t) {} + +TEST_CASE(authority_without_connection_is_captured_from_host) { + for (uint32_t flags : {0u, 1u}) { + reset_host(flags); + auto ctx = AuthCtx::from_connection_id_opt(std::nullopt, verified_sender()); + auth_flags = flags ^ 1; + ASSERT_EQ(size_t{1}, flag_reads); + ASSERT_EQ(flags == 1, ctx.is_internal()); + ASSERT_TRUE(!ctx.has_jwt()); + ASSERT_EQ(verified_sender(), ctx.get_caller_identity()); + ASSERT_EQ(size_t{0}, jwt_reads); + } +} + +TEST_CASE(internal_call_retains_lazy_jwt_and_verified_identity) { + reset_host(1, R"({"iss":"other","sub":"other","identity":"untrusted"})"); + auto ctx = AuthCtx::from_connection_id(ConnectionId(5), verified_sender()); + auth_flags = 0; + ASSERT_TRUE(ctx.is_internal()); + ASSERT_EQ(size_t{0}, jwt_reads); + ASSERT_TRUE(ctx.has_jwt()); + ASSERT_EQ(verified_sender(), ctx.get_jwt()->get_identity()); + ASSERT_EQ(std::string("other"), ctx.get_jwt()->subject()); + ASSERT_EQ(size_t{1}, jwt_reads); +} + +TEST_CASE(jwt_source_reads_all_chunks_including_final_exhausted_bytes) { + reset_host(0, "{\"padding\":\"" + std::string(8192, 'x') + "\",\"sub\":\"last\"}"); + auto ctx = AuthCtx::from_connection_id(ConnectionId(5), verified_sender()); + ASSERT_TRUE(ctx.has_jwt()); + ASSERT_EQ(std::string("last"), ctx.get_jwt()->subject()); + ASSERT_EQ(jwt_payload.size(), payload_offset); + ASSERT_EQ(size_t{1}, jwt_reads); +} + +TEST_CASE(procedure_transactions_preserve_authority_connection_and_sender) { + for (uint64_t connection : {0u, 5u}) { + reset_host(1, R"({"sub":"worker"})"); + ProcedureContext ctx(verified_sender(), Timestamp::from_micros_since_epoch(0), ConnectionId(connection)); + auth_flags = 0; + ctx.with_tx([&](TxContext& tx) { + ASSERT_TRUE(tx.sender_auth().is_internal()); + ASSERT_EQ(verified_sender(), tx.sender()); + ASSERT_EQ(connection != 0, tx.connection_id.has_value()); + ASSERT_EQ(connection != 0, tx.sender_auth().has_jwt()); + if (connection) ASSERT_EQ(verified_sender(), tx.sender_auth().get_jwt()->get_identity()); + }); + ASSERT_EQ(size_t{1}, flag_reads); + } +} diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index f3933229c16..40e8c051ff4 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -347,6 +347,71 @@ public static void @params(ProcedureContext ctx) Assert.Empty(GetCompilationErrors(compilationAfterGen)); } + [Fact] + public static async Task ExplicitFunctionVisibilityCompilesAndRejectsExternalLifecycle() + { + var fixture = await Fixture.Compile("server"); + const string source = """ + using SpacetimeDB; + public static partial class VisibilityFunctions + { + [Reducer(Visibility = FunctionVisibility.Public)] + public static void PublicJob(ReducerContext ctx) {} + [Reducer(Visibility = FunctionVisibility.Private)] + public static void PrivateJob(ReducerContext ctx) {} + [Reducer(Visibility = FunctionVisibility.Internal)] + public static void InternalJob(ReducerContext ctx) {} + [Procedure(Visibility = FunctionVisibility.Internal)] + public static int InternalProcedure(ProcedureContext ctx) => 1; + } + """; + var parseOptions = new CSharpParseOptions(fixture.SampleCompilation.LanguageVersion); + var tree = CSharpSyntaxTree.ParseText(source, parseOptions); + var compilation = fixture.SampleCompilation.AddSyntaxTrees(tree); + var driver = CSharpGeneratorDriver.Create( + [ + new SpacetimeDB.Codegen.Type().AsSourceGenerator(), + new SpacetimeDB.Codegen.Module().AsSourceGenerator(), + ], + parseOptions: parseOptions + ); + var result = driver.RunGenerators(compilation).GetRunResult(); + Assert.Empty(result.Diagnostics.Where(d => d.Severity == DiagnosticSeverity.Error)); + Assert.Empty(GetCompilationErrors(compilation.AddSyntaxTrees(result.GeneratedTrees))); + var generated = string.Join("\n", result.GeneratedTrees.Select(t => t.ToString())); + Assert.Contains( + "DeclaredVisibility: SpacetimeDB.Internal.FunctionVisibilityV11.ClientCallable", + generated + ); + Assert.Contains( + "DeclaredVisibility: SpacetimeDB.Internal.FunctionVisibilityV11.Private", + generated + ); + Assert.Contains( + "DeclaredVisibility: SpacetimeDB.Internal.FunctionVisibilityV11.Internal", + generated + ); + + var invalid = CSharpSyntaxTree.ParseText( + """ + using SpacetimeDB; + public static partial class BadVisibility + { + [Reducer(ReducerKind.Init, Visibility = FunctionVisibility.Public)] + public static void InvalidLifecycle(ReducerContext ctx) {} + } + """, + parseOptions + ); + var rejected = driver + .RunGenerators(fixture.SampleCompilation.AddSyntaxTrees(invalid)) + .GetRunResult(); + Assert.Contains( + rejected.Diagnostics, + diagnostic => diagnostic.GetMessage().Contains("Lifecycle reducers only permit") + ); + } + [Fact] public static async Task TestDiagnostics() { diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 2504cc013bf..dd48f15e495 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -3023,13 +3023,13 @@ static class ModuleRegistration { class __ReducerWithReservedPrefix : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(__ReducerWithReservedPrefix), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3046,13 +3046,13 @@ class DummyScheduledReducer : SpacetimeDB.Internal.IReducer { private static readonly TestScheduleIssues.BSATN tableRW = new(); - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(DummyScheduledReducer), Params: [new("table", tableRW.GetAlgebraicType(registrar))], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3070,13 +3070,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class OnReducerWithReservedPrefix : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(OnReducerWithReservedPrefix), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3091,13 +3091,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestDuplicateReducerKind1 : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestDuplicateReducerKind1), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3112,13 +3112,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestDuplicateReducerKind2 : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestDuplicateReducerKind2), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3133,13 +3133,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestDuplicateReducerName : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestDuplicateReducerName), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3154,13 +3154,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestReducerReturnType : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestReducerReturnType), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3175,13 +3175,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestReducerWithoutContext : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestReducerWithoutContext), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index b46de3189b4..126924858f3 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -519,13 +519,13 @@ class DemoReducer : SpacetimeDB.Internal.IReducer { private static readonly SpacetimeDB.BSATN.I32 valueRW = new(); - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(DemoReducer), Params: [new("value", valueRW.GetAlgebraicType(registrar))], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -540,14 +540,14 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class DemoProcedure : SpacetimeDB.Internal.IProcedure { - public SpacetimeDB.Internal.RawProcedureDefV10 MakeProcedureDef( + public SpacetimeDB.Internal.RawProcedureDefV11 MakeProcedureDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(DemoProcedure), Params: [], ReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable + DeclaredVisibility: null ); public byte[] Invoke(BinaryReader reader, SpacetimeDB.Internal.IProcedureContext ctx) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index edc7d5f2af4..3a8d63aabb0 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -2328,13 +2328,13 @@ static class ModuleRegistration { class Init : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(Init), Params: [], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2351,13 +2351,13 @@ class InsertData : SpacetimeDB.Internal.IReducer { private static readonly PublicTable.BSATN dataRW = new(); - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(InsertData), Params: [new("data", dataRW.GetAlgebraicType(registrar))], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2374,13 +2374,13 @@ class InsertData2 : SpacetimeDB.Internal.IReducer { private static readonly PublicTable.BSATN dataRW = new(); - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(InsertData2), Params: [new("data", dataRW.GetAlgebraicType(registrar))], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2400,13 +2400,13 @@ class InsertMultiData : SpacetimeDB.Internal.IReducer { private static readonly MultiTableRow.BSATN dataRW = new(); - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(InsertMultiData), Params: [new("data", dataRW.GetAlgebraicType(registrar))], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2423,13 +2423,13 @@ class ScheduleImmediate : SpacetimeDB.Internal.IReducer { private static readonly PublicTable.BSATN dataRW = new(); - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(ScheduleImmediate), Params: [new("data", dataRW.GetAlgebraicType(registrar))], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2446,13 +2446,13 @@ class SendScheduledMessage : SpacetimeDB.Internal.IReducer { private static readonly Timers.SendMessageTimer.BSATN argRW = new(); - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(SendScheduledMessage), Params: [new("arg", argRW.GetAlgebraicType(registrar))], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: null, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen/Diag.cs b/crates/bindings-csharp/Codegen/Diag.cs index 928b7a71635..fb70f672f85 100644 --- a/crates/bindings-csharp/Codegen/Diag.cs +++ b/crates/bindings-csharp/Codegen/Diag.cs @@ -333,4 +333,13 @@ string prefix $"HTTP handler method {method.Identifier} must be non-generic and take exactly two parameters.", method => method.ParameterList ); + + public static readonly ErrorDescriptor InvalidFunctionVisibility = + new( + group, + "Invalid function visibility", + _ => + $"Visibility must be Default, Public, Private, or Internal. Lifecycle reducers only permit Default or Internal.", + method => method.Identifier + ); } diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index 870cadce093..4ac8ca7139d 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -1405,13 +1405,44 @@ public byte[] Invoke( } /// -/// Represents a reducer method declaration in a module. +/// Validates a declared function visibility and maps it to the V11 schema. /// +static class FunctionVisibilityDeclaration +{ + internal static string Resolve( + FunctionVisibility visibility, + bool lifecycle, + MethodDeclarationSyntax method, + DiagReporter diag + ) + { + if ( + ( + lifecycle + && visibility is not (FunctionVisibility.Default or FunctionVisibility.Internal) + ) || !Enum.IsDefined(typeof(FunctionVisibility), visibility) + ) + { + diag.Report(ErrorDescriptor.InvalidFunctionVisibility, method); + return "null"; + } + return visibility switch + { + FunctionVisibility.Public => + "SpacetimeDB.Internal.FunctionVisibilityV11.ClientCallable", + FunctionVisibility.Private => "SpacetimeDB.Internal.FunctionVisibilityV11.Private", + FunctionVisibility.Internal => "SpacetimeDB.Internal.FunctionVisibilityV11.Internal", + _ => "null", + }; + } +} + record ReducerDeclaration { public readonly string Name; public readonly string? CanonicalName; public readonly ReducerKind Kind; + public readonly string DeclaredVisibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1450,6 +1481,12 @@ public ReducerDeclaration(GeneratorAttributeSyntaxContext context, DiagReporter } Kind = attr.Kind; + DeclaredVisibility = FunctionVisibilityDeclaration.Resolve( + attr.Visibility, + Kind != ReducerKind.UserDefined, + methodSyntax, + diag + ); CanonicalName = attr.Name; FullName = SymbolToName(method); Args = new( @@ -1475,10 +1512,10 @@ public string GenerateClass() class {{Identifier}}: SpacetimeDB.Internal.IReducer { {{MemberDeclaration.GenerateBsatnFields(Accessibility.Private, Args)}} - public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new ( + public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new ( SourceName: nameof({{Identifier}}), Params: [{{MemberDeclaration.GenerateDefs(Args)}}], - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, + DeclaredVisibility: {{DeclaredVisibility}}, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -1535,6 +1572,7 @@ record ProcedureDeclaration { public readonly string Name; public readonly string? CanonicalName; + public readonly string DeclaredVisibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1551,6 +1589,12 @@ public ProcedureDeclaration(GeneratorAttributeSyntaxContext context, DiagReporte var methodSyntax = (MethodDeclarationSyntax)context.TargetNode; var method = (IMethodSymbol)context.TargetSymbol; var attr = context.Attributes.Single().ParseAs(); + DeclaredVisibility = FunctionVisibilityDeclaration.Resolve( + attr.Visibility, + false, + methodSyntax, + diag + ); if ( method.Parameters.FirstOrDefault()?.Type @@ -1706,11 +1750,11 @@ public string GenerateClass() class {{{Identifier}}} : SpacetimeDB.Internal.IProcedure { {{{classFields}}} - public SpacetimeDB.Internal.RawProcedureDefV10 MakeProcedureDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new( + public SpacetimeDB.Internal.RawProcedureDefV11 MakeProcedureDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new( SourceName: nameof({{{Identifier}}}), Params: [{{{MemberDeclaration.GenerateDefs(Args)}}}], ReturnType: {{{returnTypeExpr}}}, - Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable + DeclaredVisibility: {{{DeclaredVisibility}}} ); public byte[] Invoke(BinaryReader reader, SpacetimeDB.Internal.IProcedureContext ctx) { diff --git a/crates/bindings-csharp/README.md b/crates/bindings-csharp/README.md index 289bd570ff0..d75b17677b1 100644 --- a/crates/bindings-csharp/README.md +++ b/crates/bindings-csharp/README.md @@ -6,6 +6,27 @@ See the [C# module library reference](https://spacetimedb.com/docs/modules/c-sha ## Internal documentation +### Function visibility and invocation authentication + +Reducers and procedures can declare `Visibility = FunctionVisibility.Public`, +`Private`, or `Internal` in their attributes. Omission (`Default`) means public +for ordinary functions and private for scheduled functions. An explicit choice +is preserved when the function is scheduled. Lifecycle reducers permit only +omission or `Internal` and can only run for their host lifecycle event. + +Internal functions require verified internal authority. Private functions also +admit the owner, and public functions admit any client. For example: + +```csharp +[Reducer(Visibility = FunctionVisibility.Internal)] +public static void ProcessJobs(ReducerContext ctx) { } +``` + +`ctx.SenderAuth.IsInternal` comes from the host's invocation authority. It is +independent of connection and JWT presence, so an internal call can have a JWT. +JWT identity is the verified sender supplied by the host. Newly compiled modules +emit schema V11 and advertise `hosted_auth_v1`, requiring a compatible host. + These projects contain the SpacetimeDB SATS typesystem, codegen and runtime bindings for SpacetimeDB WebAssembly modules. It also contains serialization code for SpacetimeDB C# clients. @@ -19,4 +40,3 @@ The [`Codegen`](./Codegen/) and [`Runtime`](./Runtime/) libraries are used: - only by C# Modules. They provide all of the functionality needed to write SpacetimeDB modules in C#. See their READMEs for more information. - diff --git a/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs new file mode 100644 index 00000000000..544391af3da --- /dev/null +++ b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs @@ -0,0 +1,54 @@ +namespace Runtime.Tests; + +using SpacetimeDB.BSATN; +using SpacetimeDB.Internal; + +public class FunctionVisibilityTests +{ + [Theory] + [InlineData(null)] + [InlineData(FunctionVisibilityV11.ClientCallable)] + [InlineData(FunctionVisibilityV11.Private)] + [InlineData(FunctionVisibilityV11.Internal)] + public void SchedulingPreservesDeclaredVisibility(FunctionVisibilityV11? visibility) + { + var module = new RawModuleDefV11(); + var reducer = new RawReducerDefV11( + "run_job", + [], + visibility, + AlgebraicType.Unit, + new AlgebraicType.String(default) + ); + module.RegisterReducer(reducer, null); + module.RegisterTable( + new RawTableDefV10 { SourceName = "jobs" }, + new RawScheduleDefV10(null, "jobs", 0, "run_job") + ); + var raw = module.BuildModuleDefinition(); + var reducers = Assert.Single(raw.Sections.OfType()); + Assert.Equal(visibility, Assert.Single(reducers.Reducers_).DeclaredVisibility); + var capabilities = Assert.Single( + raw.Sections.OfType() + ); + Assert.Contains("hosted_auth_v1", capabilities.Capabilities_); + } + + [Theory] + [InlineData(FunctionVisibilityV11.ClientCallable)] + [InlineData(FunctionVisibilityV11.Private)] + public void LifecycleRejectsExternalVisibility(FunctionVisibilityV11 visibility) + { + var module = new RawModuleDefV11(); + var reducer = new RawReducerDefV11( + "initialize", + [], + visibility, + AlgebraicType.Unit, + new AlgebraicType.String(default) + ); + Assert.Throws( + () => module.RegisterReducer(reducer, Lifecycle.Init) + ); + } +} diff --git a/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs b/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs new file mode 100644 index 00000000000..ada932fb631 --- /dev/null +++ b/crates/bindings-csharp/Runtime.Tests/HostedAuthTests.cs @@ -0,0 +1,43 @@ +namespace Runtime.Tests; + +using SpacetimeDB; + +public class HostedAuthTests +{ + [Theory] + [InlineData(0u, false)] + [InlineData(1u, true)] + public void NoJwtCallsPreserveVerifiedInternalFlag(uint flags, bool expectedInternal) + { + var auth = AuthCtx.FromVerifiedCall(flags, () => null); + Assert.Equal(expectedInternal, auth.IsInternal); + Assert.False(auth.HasJwt); + Assert.Null(auth.Jwt); + } + + [Fact] + public void InternalCallCanRetainJwtAndVerifiedSenderIdentity() + { + var sender = Identity.FromHexString(new string('a', 64)); + var reads = 0; + var flags = 1u; + var auth = AuthCtx.FromVerifiedCall( + flags, + () => + { + reads++; + return new JwtClaims( + "{\"iss\":\"different-issuer\",\"sub\":\"different-subject\",\"identity\":\"untrusted\"}", + sender + ); + } + ); + flags = 0; + Assert.True(auth.IsInternal); + Assert.Equal(0, reads); + Assert.True(auth.HasJwt); + Assert.Equal(sender, auth.Jwt!.Identity); + Assert.Equal("different-subject", auth.Jwt.Subject); + Assert.Equal(1, reads); + } +} diff --git a/crates/bindings-csharp/Runtime/Attrs.cs b/crates/bindings-csharp/Runtime/Attrs.cs index 46daec5eec9..d321c6a6d9e 100644 --- a/crates/bindings-csharp/Runtime/Attrs.cs +++ b/crates/bindings-csharp/Runtime/Attrs.cs @@ -190,18 +190,30 @@ public enum ReducerKind ClientDisconnected, } + /// Invocation admission for reducers and procedures. + public enum FunctionVisibility + { + /// Public for ordinary functions, Private for scheduled functions. + Default, + Public, + Private, + Internal, + } + [AttributeUsage(AttributeTargets.Method, Inherited = false)] public sealed class ReducerAttribute(ReducerKind kind = ReducerKind.UserDefined) : Attribute { public ReducerKind Kind => kind; public string? Name { get; init; } + public FunctionVisibility Visibility { get; init; } } [AttributeUsage(AttributeTargets.Method, Inherited = false)] public sealed class ProcedureAttribute() : Attribute { public string? Name { get; init; } + public FunctionVisibility Visibility { get; init; } } [AttributeUsage(AttributeTargets.Method, Inherited = false)] diff --git a/crates/bindings-csharp/Runtime/AuthCtx.cs b/crates/bindings-csharp/Runtime/AuthCtx.cs index a99ebe7d21c..db7ba86eb23 100644 --- a/crates/bindings-csharp/Runtime/AuthCtx.cs +++ b/crates/bindings-csharp/Runtime/AuthCtx.cs @@ -14,12 +14,10 @@ private AuthCtx(bool isInternal, Func jwtFactory) } /// - /// Create an AuthCtx for an internal call, with no JWT. + /// Capture verified invocation authority independently from lazy JWT loading. /// - private static AuthCtx Internal() - { - return new AuthCtx(isInternal: true, jwtFactory: () => null); - } + internal static AuthCtx FromVerifiedCall(uint callAuthFlags, Func jwtFactory) => + new(isInternal: (callAuthFlags & 1) != 0, jwtFactory); /// /// Create an AuthCtx by looking up the credentials for a connection id in system tables. @@ -29,20 +27,27 @@ private static AuthCtx Internal() /// public static AuthCtx BuildFromSystemTables(ConnectionId? connectionId, Identity identity) { + // Read synchronously while this invocation is active. Neither connection + // presence nor token claims determine internal authority. + var callAuthFlags = SpacetimeDB.Internal.FFI.get_call_auth_flags(); if (connectionId == null) { - return Internal(); + return FromVerifiedCall(callAuthFlags, () => null); } - return FromConnectionId(connectionId.Value, identity); + return FromConnectionId(connectionId.Value, identity, callAuthFlags); } /// /// Create an AuthCtx that reads JWT for a given connection ID. /// - private static AuthCtx FromConnectionId(ConnectionId connectionId, Identity identity) + private static AuthCtx FromConnectionId( + ConnectionId connectionId, + Identity identity, + uint callAuthFlags + ) { - return new AuthCtx( - isInternal: false, + return FromVerifiedCall( + callAuthFlags, jwtFactory: () => { var result = SpacetimeDB.Internal.FFI.get_jwt(ref connectionId, out var source); @@ -59,23 +64,18 @@ private static AuthCtx FromConnectionId(ConnectionId connectionId, Identity iden } /// - /// True if this reducer was spawned from inside the database. + /// True if the host verified internal authority for this invocation. /// public bool IsInternal => _isInternal; /// /// Check if there is a JWT present. - /// If IsInternal is true, this will be false. + /// Independent of IsInternal. An internal call may also have a JWT. /// public bool HasJwt { get { - if (_isInternal) - { - return false; - } - // At this point we do load the bytes. return _jwtLazy.Value != null; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibilityV11.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibilityV11.g.cs new file mode 100644 index 00000000000..fcfe2b32368 --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibilityV11.g.cs @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + public enum FunctionVisibilityV11 + { + Private, + ClientCallable, + Internal, + } +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDef.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDef.g.cs index d6d5d9f52a4..8995d4d3b86 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDef.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDef.g.cs @@ -11,6 +11,7 @@ namespace SpacetimeDB.Internal public partial record RawModuleDef : SpacetimeDB.TaggedEnum<( RawModuleDefV8 V8BackCompat, RawModuleDefV9 V9, - RawModuleDefV10 V10 + RawModuleDefV10 V10, + RawModuleDefV11 V11 )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11.g.cs new file mode 100644 index 00000000000..db3c58f7bb8 --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11.g.cs @@ -0,0 +1,29 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class RawModuleDefV11 + { + [DataMember(Name = "sections")] + public System.Collections.Generic.List Sections; + + public RawModuleDefV11(System.Collections.Generic.List Sections) + { + this.Sections = Sections; + } + + public RawModuleDefV11() + { + this.Sections = new(); + } + } +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11Section.g.cs new file mode 100644 index 00000000000..7b5d51e2d98 --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11Section.g.cs @@ -0,0 +1,27 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + public partial record RawModuleDefV11Section : SpacetimeDB.TaggedEnum<( + Typespace Typespace, + System.Collections.Generic.List Types, + System.Collections.Generic.List Tables, + System.Collections.Generic.List Reducers, + System.Collections.Generic.List Procedures, + System.Collections.Generic.List Views, + System.Collections.Generic.List Schedules, + System.Collections.Generic.List LifeCycleReducers, + System.Collections.Generic.List RowLevelSecurity, + SpacetimeDB.CaseConversionPolicy CaseConversionPolicy, + ExplicitNames ExplicitNames, + System.Collections.Generic.List HttpHandlers, + System.Collections.Generic.List HttpRoutes, + System.Collections.Generic.List Capabilities + )>; +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawProcedureDefV11.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawProcedureDefV11.g.cs new file mode 100644 index 00000000000..ca4de3f0c5d --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawProcedureDefV11.g.cs @@ -0,0 +1,45 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class RawProcedureDefV11 + { + [DataMember(Name = "source_name")] + public string SourceName; + [DataMember(Name = "params")] + public List Params; + [DataMember(Name = "declared_visibility")] + public FunctionVisibilityV11? DeclaredVisibility; + [DataMember(Name = "return_type")] + public SpacetimeDB.BSATN.AlgebraicType ReturnType; + + public RawProcedureDefV11( + string SourceName, + List Params, + FunctionVisibilityV11? DeclaredVisibility, + SpacetimeDB.BSATN.AlgebraicType ReturnType + ) + { + this.SourceName = SourceName; + this.Params = Params; + this.DeclaredVisibility = DeclaredVisibility; + this.ReturnType = ReturnType; + } + + public RawProcedureDefV11() + { + this.SourceName = ""; + this.Params = new(); + this.ReturnType = null!; + } + } +} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawReducerDefV11.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawReducerDefV11.g.cs new file mode 100644 index 00000000000..86990c74376 --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawReducerDefV11.g.cs @@ -0,0 +1,50 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class RawReducerDefV11 + { + [DataMember(Name = "source_name")] + public string SourceName; + [DataMember(Name = "params")] + public List Params; + [DataMember(Name = "declared_visibility")] + public FunctionVisibilityV11? DeclaredVisibility; + [DataMember(Name = "ok_return_type")] + public SpacetimeDB.BSATN.AlgebraicType OkReturnType; + [DataMember(Name = "err_return_type")] + public SpacetimeDB.BSATN.AlgebraicType ErrReturnType; + + public RawReducerDefV11( + string SourceName, + List Params, + FunctionVisibilityV11? DeclaredVisibility, + SpacetimeDB.BSATN.AlgebraicType OkReturnType, + SpacetimeDB.BSATN.AlgebraicType ErrReturnType + ) + { + this.SourceName = SourceName; + this.Params = Params; + this.DeclaredVisibility = DeclaredVisibility; + this.OkReturnType = OkReturnType; + this.ErrReturnType = ErrReturnType; + } + + public RawReducerDefV11() + { + this.SourceName = ""; + this.Params = new(); + this.OkReturnType = null!; + this.ErrReturnType = null!; + } + } +} diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index 261233303dc..946d66ad2db 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -98,6 +98,17 @@ internal static partial class FFI #endif ; + const string StdbNamespace10_6 = +#if EXPERIMENTAL_WASM_AOT + "spacetime_10.6" +#else + "bindings" +#endif + ; + + [LibraryImport(StdbNamespace10_6)] + public static partial uint get_call_auth_flags(); + [NativeMarshalling(typeof(Marshaller))] public struct CheckedStatus { diff --git a/crates/bindings-csharp/Runtime/Internal/IReducer.cs b/crates/bindings-csharp/Runtime/Internal/IReducer.cs index 878c98a2a2e..af068b38253 100644 --- a/crates/bindings-csharp/Runtime/Internal/IReducer.cs +++ b/crates/bindings-csharp/Runtime/Internal/IReducer.cs @@ -18,7 +18,7 @@ public static Identity GetDatabaseIdentity() public interface IReducer { - RawReducerDefV10 MakeReducerDef(ITypeRegistrar registrar); + RawReducerDefV11 MakeReducerDef(ITypeRegistrar registrar); Lifecycle? Lifecycle { get; } diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 530cc20b402..1b72234b70e 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -7,15 +7,15 @@ namespace SpacetimeDB.Internal; using SpacetimeDB; using SpacetimeDB.BSATN; -partial class RawModuleDefV10 +partial class RawModuleDefV11 { private readonly Typespace typespace = new(); private readonly List typeDefs = []; private readonly List tableDefs = []; private readonly List scheduleDefs = []; - private readonly List reducerDefs = []; + private readonly List reducerDefs = []; private readonly List lifecycleReducerDefs = []; - private readonly List procedureDefs = []; + private readonly List procedureDefs = []; private readonly List httpHandlerDefs = []; private readonly List httpRouteDefs = []; private readonly List viewDefs = []; @@ -53,19 +53,27 @@ internal AlgebraicType.Ref RegisterType(Func procedureDefs.Add(procedure); + internal void RegisterProcedure(RawProcedureDefV11 procedure) => procedureDefs.Add(procedure); internal void RegisterHttpHandler(RawHttpHandlerDefV10 handler) => httpHandlerDefs.Add(handler); @@ -109,7 +117,7 @@ internal void RegisterExplicitFunctionName(string sourceName, string canonicalNa internal void RegisterExplicitIndexName(string sourceName, string canonicalName) => explicitNames.Add(new ExplicitNameEntry.Index(new NameMapping(sourceName, canonicalName))); - internal RawModuleDefV10 BuildModuleDefinition() + internal RawModuleDefV11 BuildModuleDefinition() { var builtTables = new List(tableDefs.Count); foreach (var table in tableDefs) @@ -133,84 +141,64 @@ internal RawModuleDefV10 BuildModuleDefinition() ); } - var internalFunctions = lifecycleReducerDefs - .Select(l => l.FunctionName) - .Concat(scheduleDefs.Select(s => s.FunctionName)) - .ToHashSet(StringComparer.Ordinal); - - foreach (var reducer in reducerDefs) - { - if (internalFunctions.Contains(reducer.SourceName)) - { - reducer.Visibility = FunctionVisibility.Private; - } - } - - foreach (var procedure in procedureDefs) - { - if (internalFunctions.Contains(procedure.SourceName)) - { - procedure.Visibility = FunctionVisibility.Private; - } - } - - var sections = new List + var sections = new List { - new RawModuleDefV10Section.Typespace(typespace), + new RawModuleDefV11Section.Typespace(typespace), + new RawModuleDefV11Section.Capabilities(["hosted_auth_v1"]), }; if (typeDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.Types(typeDefs)); + sections.Add(new RawModuleDefV11Section.Types(typeDefs)); } if (builtTables.Count > 0) { - sections.Add(new RawModuleDefV10Section.Tables(builtTables)); + sections.Add(new RawModuleDefV11Section.Tables(builtTables)); } if (reducerDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.Reducers(reducerDefs)); + sections.Add(new RawModuleDefV11Section.Reducers(reducerDefs)); } if (procedureDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.Procedures(procedureDefs)); + sections.Add(new RawModuleDefV11Section.Procedures(procedureDefs)); } if (httpHandlerDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.HttpHandlers(httpHandlerDefs)); + sections.Add(new RawModuleDefV11Section.HttpHandlers(httpHandlerDefs)); } if (httpRouteDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.HttpRoutes(httpRouteDefs)); + sections.Add(new RawModuleDefV11Section.HttpRoutes(httpRouteDefs)); } if (viewDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.Views(viewDefs)); + sections.Add(new RawModuleDefV11Section.Views(viewDefs)); } if (scheduleDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.Schedules(scheduleDefs)); + sections.Add(new RawModuleDefV11Section.Schedules(scheduleDefs)); } if (lifecycleReducerDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.LifeCycleReducers(lifecycleReducerDefs)); + sections.Add(new RawModuleDefV11Section.LifeCycleReducers(lifecycleReducerDefs)); } // TODO: Add sections for Event tables and Case conversion policy (mirrors Rust `raw_def/v10.rs` TODO). if (caseConversionPolicy is { } policy) { - sections.Add(new RawModuleDefV10Section.CaseConversionPolicy(policy)); + sections.Add(new RawModuleDefV11Section.CaseConversionPolicy(policy)); } if (explicitNames.Count > 0) { sections.Add( - new RawModuleDefV10Section.ExplicitNames( + new RawModuleDefV11Section.ExplicitNames( new ExplicitNames(new List(explicitNames)) ) ); } if (rowLevelSecurityDefs.Count > 0) { - sections.Add(new RawModuleDefV10Section.RowLevelSecurity(rowLevelSecurityDefs)); + sections.Add(new RawModuleDefV11Section.RowLevelSecurity(rowLevelSecurityDefs)); } Sections = sections; @@ -240,8 +228,8 @@ private static void EnsureNativeAotTypeRoots() { _ = new RawIndexAlgorithm.BTree(null!); _ = new RawConstraintDataV9.Unique(null!); - _ = new RawModuleDef.V10(null!); - _ = new RawModuleDefV10Section.Typespace(null!); + _ = new RawModuleDef.V11(null!); + _ = new RawModuleDefV11Section.Typespace(null!); _ = new ExplicitNameEntry.Table(null!); _ = new MiscModuleExport.TypeAlias(null!); _ = new RawMiscModuleExportV9.ColumnDefaultValue(null!); @@ -250,7 +238,7 @@ private static void EnsureNativeAotTypeRoots() } } - private static readonly RawModuleDefV10 moduleDef = new(); + private static readonly RawModuleDefV11 moduleDef = new(); private static readonly List reducers = []; private static readonly List procedures = []; @@ -508,7 +496,7 @@ public static void __describe_module__(BytesSink description) try { var module = moduleDef.BuildModuleDefinition(); - RawModuleDef versioned = new RawModuleDef.V10(module); + RawModuleDef versioned = new RawModuleDef.V11(module); var moduleBytes = IStructuralReadWrite.ToBytes(new RawModuleDef.BSATN(), versioned); description.Write(moduleBytes); } diff --git a/crates/bindings-csharp/Runtime/Internal/Procedure.cs b/crates/bindings-csharp/Runtime/Internal/Procedure.cs index 81d2be9a749..010c8d9e192 100644 --- a/crates/bindings-csharp/Runtime/Internal/Procedure.cs +++ b/crates/bindings-csharp/Runtime/Internal/Procedure.cs @@ -13,7 +13,7 @@ public interface IProcedure /// /// Creates a procedure definition for registration with the module system. /// - RawProcedureDefV10 MakeProcedureDef(ITypeRegistrar registrar); + RawProcedureDefV11 MakeProcedureDef(ITypeRegistrar registrar); /// /// Invokes the procedure with the given arguments and context. diff --git a/crates/bindings-csharp/Runtime/JwtClaims.cs b/crates/bindings-csharp/Runtime/JwtClaims.cs index 6e656cca202..554ef4d02d0 100644 --- a/crates/bindings-csharp/Runtime/JwtClaims.cs +++ b/crates/bindings-csharp/Runtime/JwtClaims.cs @@ -16,8 +16,8 @@ public sealed class JwtClaims /// /// Create a JwtClaims from a raw JWT payload (JSON claims) and its associated Identity. /// - /// This only takes an Identity because the Blake3 hash package on nuget wraps rust code. - /// We should not expose this constructor publicly, but it is needed for AuthCtx. + /// Identity is the verified sender provided by the host. Claims cannot + /// override it, including for hosted database credentials. /// internal JwtClaims(string jwt, Identity identity) { diff --git a/crates/bindings-csharp/Runtime/Runtime.csproj b/crates/bindings-csharp/Runtime/Runtime.csproj index d0f71c34de0..e7d1a35c149 100644 --- a/crates/bindings-csharp/Runtime/Runtime.csproj +++ b/crates/bindings-csharp/Runtime/Runtime.csproj @@ -45,6 +45,7 @@ + diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index 57ed816d939..1ee1d745f7e 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -135,6 +135,10 @@ IMPORT(Status, datastore_clear, (table_id, count)); #undef SPACETIME_MODULE_VERSION +#define SPACETIME_MODULE_VERSION "spacetime_10.6" +IMPORT(uint32_t, get_call_auth_flags, (void), ()); +#undef SPACETIME_MODULE_VERSION + #ifndef EXPERIMENTAL_WASM_AOT static MonoClass* ffi_class; diff --git a/crates/bindings-macro/src/procedure.rs b/crates/bindings-macro/src/procedure.rs index 9f76e5b547f..129b32cc2fe 100644 --- a/crates/bindings-macro/src/procedure.rs +++ b/crates/bindings-macro/src/procedure.rs @@ -1,4 +1,5 @@ use crate::reducer::{assert_only_lifetime_generics, extract_typed_args, generate_explicit_names_impl}; +use crate::reducer::{parse_visibility, DeclaredVisibility}; use crate::sym; use crate::util::{check_duplicate, ident_to_litstr, match_meta}; use proc_macro2::TokenStream; @@ -10,12 +11,16 @@ use syn::{ItemFn, LitStr}; pub(crate) struct ProcedureArgs { /// For consistency with reducers: allow specifying a different export name than the Rust function name. name: Option, + visibility: Option, } impl ProcedureArgs { pub(crate) fn parse(input: TokenStream) -> syn::Result { let mut args = Self::default(); syn::meta::parser(|meta| { + if parse_visibility(&meta, &mut args.visibility)? { + return Ok(()); + } match_meta!(match meta { sym::name => { check_duplicate(&args.name, &meta)?; @@ -29,10 +34,11 @@ impl ProcedureArgs { } } -pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) -> syn::Result { +pub(crate) fn procedure_impl(args: ProcedureArgs, original_function: &ItemFn) -> syn::Result { let func_name = &original_function.sig.ident; let vis = &original_function.vis; - let explicit_name = _args.name.as_ref(); + let explicit_name = args.name.as_ref(); + let visibility = args.visibility.map(DeclaredVisibility::tokens).into_iter(); let procedure_name = ident_to_litstr(func_name); @@ -117,6 +123,7 @@ pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) - /// The name of this function const NAME: &'static str = #procedure_name; + #(const DECLARED_VISIBILITY: Option = Some(#visibility);)* /// The parameter names of this function const ARG_NAMES: &'static [Option<&'static str>] = &[#(#opt_arg_names),*]; @@ -133,3 +140,29 @@ pub(crate) fn procedure_impl(_args: ProcedureArgs, original_function: &ItemFn) - #generate_explicit_names }) } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn procedure_visibility_rejects_duplicates_and_emits_selection() { + assert!(ProcedureArgs::parse(quote!(private, public)).is_err()); + assert!(ProcedureArgs::parse(quote!(internal, internal)).is_err()); + let function: ItemFn = syn::parse_quote!( + fn example(ctx: &mut ProcedureContext) -> u64 { + 0 + } + ); + for (input, expected) in [ + (quote!(internal), "Internal"), + (quote!(private), "Private"), + (quote!(public), "ClientCallable"), + ] { + let tokens = procedure_impl(ProcedureArgs::parse(input).unwrap(), &function) + .unwrap() + .to_string(); + assert!(tokens.contains("DECLARED_VISIBILITY")); + assert!(tokens.contains(&format!("FunctionVisibility :: {expected}"))); + } + } +} diff --git a/crates/bindings-macro/src/reducer.rs b/crates/bindings-macro/src/reducer.rs index ac261ced35f..3093a51397a 100644 --- a/crates/bindings-macro/src/reducer.rs +++ b/crates/bindings-macro/src/reducer.rs @@ -10,6 +10,44 @@ use syn::{FnArg, Ident, ItemFn, LitStr, PatType}; pub(crate) struct ReducerArgs { name: Option, lifecycle: Option, + visibility: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DeclaredVisibility { + Internal, + Private, + Public, +} + +impl DeclaredVisibility { + pub(crate) fn tokens(self) -> TokenStream { + let variant = match self { + Self::Internal => "Internal", + Self::Private => "Private", + Self::Public => "ClientCallable", + }; + let variant = Ident::new(variant, Span::call_site()); + quote!(spacetimedb::rt::FunctionVisibility::#variant) + } +} + +pub(crate) fn parse_visibility( + meta: &syn::meta::ParseNestedMeta<'_>, + visibility: &mut Option, +) -> syn::Result { + let value = if meta.path.is_ident("internal") { + DeclaredVisibility::Internal + } else if meta.path.is_ident("private") { + DeclaredVisibility::Private + } else if meta.path.is_ident("public") { + DeclaredVisibility::Public + } else { + return Ok(false); + }; + check_duplicate_msg(visibility, meta, "already specified a function visibility")?; + *visibility = Some(value); + Ok(true) } enum LifecycleReducer { @@ -37,6 +75,9 @@ impl ReducerArgs { pub(crate) fn parse(input: TokenStream) -> syn::Result { let mut args = Self::default(); syn::meta::parser(|meta| { + if parse_visibility(&meta, &mut args.visibility)? { + return Ok(()); + } let mut set_lifecycle = |kind: fn(Span) -> _| -> syn::Result<()> { check_duplicate_msg(&args.lifecycle, &meta, "already specified a lifecycle reducer kind")?; args.lifecycle = Some(kind(meta.path.span())); @@ -55,6 +96,12 @@ impl ReducerArgs { Ok(()) }) .parse2(input)?; + if args.lifecycle.is_some() && args.visibility.is_some_and(|v| v != DeclaredVisibility::Internal) { + return Err(syn::Error::new( + Span::call_site(), + "lifecycle reducers must have internal visibility", + )); + } Ok(args) } } @@ -101,6 +148,7 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn assert_only_lifetime_generics(original_function, "reducers")?; let lifecycle = args.lifecycle.iter().filter_map(|lc| lc.to_lifecycle_value()); + let visibility = args.visibility.map(DeclaredVisibility::tokens).into_iter(); let typed_args = extract_typed_args(original_function)?; @@ -165,6 +213,7 @@ pub(crate) fn reducer_impl(args: ReducerArgs, original_function: &ItemFn) -> syn /// The function kind, which will cause scheduled tables to accept reducers. type FnKind = spacetimedb::rt::FnKindReducer; const NAME: &'static str = #reducer_name; + #(const DECLARED_VISIBILITY: Option = Some(#visibility);)* #(const LIFECYCLE: Option = Some(#lifecycle);)* const ARG_NAMES: &'static [Option<&'static str>] = &[#(#opt_arg_names),*]; const INVOKE: Self::Invoke = #func_name::invoke; @@ -202,3 +251,43 @@ pub(crate) fn generate_explicit_names_impl( } } } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn visibility_declarations_are_unambiguous() { + for input in [ + quote!(private, public), + quote!(internal, internal), + quote!(init, private), + quote!(public, client_connected), + ] { + assert!(ReducerArgs::parse(input).is_err()); + } + for input in [ + quote!(), + quote!(public), + quote!(private), + quote!(internal), + quote!(init, internal), + ] { + assert!(ReducerArgs::parse(input).is_ok()); + } + } + #[test] + fn rust_item_visibility_does_not_select_database_visibility() { + let function: ItemFn = syn::parse_quote!( + pub fn example(ctx: &ReducerContext) {} + ); + let implicit = reducer_impl(ReducerArgs::parse(quote!()).unwrap(), &function) + .unwrap() + .to_string(); + assert!(!implicit.contains("DECLARED_VISIBILITY")); + let explicit = reducer_impl(ReducerArgs::parse(quote!(internal)).unwrap(), &function) + .unwrap() + .to_string(); + assert!(explicit.contains("DECLARED_VISIBILITY")); + assert!(explicit.contains("FunctionVisibility :: Internal")); + } +} diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index b31c4d0a451..c629595ddfb 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -885,6 +885,14 @@ pub mod raw { pub fn datastore_clear(table_id: TableId, out: *mut u64) -> u16; } + #[link(wasm_import_module = "spacetime_10.6")] + unsafe extern "C" { + /// Authentication flags for the active invocation. Bit 0 is INTERNAL. + /// Read at context construction; neither a missing connection ID nor JWT + /// claims imply internal authority. Unknown bits must be ignored. + pub fn get_call_auth_flags() -> u32; + } + /// What strategy does the database index use? /// /// See also: @@ -1661,3 +1669,10 @@ pub mod procedure { } } } + +/// Read host-verified authentication flags for the active invocation. +/// Bit 0 is INTERNAL; all other bits are reserved. +pub fn get_call_auth_flags() -> u32 { + // SAFETY: no pointers or guest-provided values are passed to the host. + unsafe { raw::get_call_auth_flags() } +} diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index 5eb0acc19a4..d4f2d609599 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -18,6 +18,25 @@ You can use the package in the browser, using a bundler like vite/parcel/rsbuild ### Usage +#### Module function visibility and invocation authentication + +Reducer and procedure options accept `visibility: 'public'`, `'private'`, or +`'internal'`. For example, `spacetime.reducer({ visibility: 'internal' }, ctx => {})` +declares an internal reducer. Omission means public for ordinary functions and +private for scheduled functions. An explicit choice is preserved when the +function is scheduled. Lifecycle reducers permit only omission or `'internal'` +and can only run for their host lifecycle event. + +Internal functions require verified internal authority. Private functions also +admit the owner, and public functions admit any client. `ctx.senderAuth.isInternal` +captures the host's invocation authority independently of connection and JWT +presence, so an internal call can have a JWT. `ctx.senderAuth.jwt.identity` is the +verified sender supplied by the host. Procedure transactions preserve this +authentication. Newly compiled modules emit schema V11 and advertise +`hosted_auth_v1`, requiring a compatible host. + +#### Client SDK + In order to connect to a database you have to generate module bindings for your database. ```ts diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index c0855af29bf..f8a25bf55bf 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -79,6 +79,14 @@ export const FunctionVisibility = __t.enum('FunctionVisibility', { }); export type FunctionVisibility = __Infer; +// The tagged union or sum type for the algebraic type `FunctionVisibilityV11`. +export const FunctionVisibilityV11 = __t.enum('FunctionVisibilityV11', { + Private: __t.unit(), + ClientCallable: __t.unit(), + Internal: __t.unit(), +}); +export type FunctionVisibilityV11 = __Infer; + export const HttpHeaderPair = __t.object('HttpHeaderPair', { name: __t.string(), value: __t.byteArray(), @@ -336,6 +344,9 @@ export const RawModuleDef = __t.enum('RawModuleDef', { get V10() { return RawModuleDefV10; }, + get V11() { + return RawModuleDefV11; + }, }); export type RawModuleDef = __Infer; @@ -390,6 +401,58 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { }); export type RawModuleDefV10Section = __Infer; +export const RawModuleDefV11 = __t.object('RawModuleDefV11', { + get sections() { + return __t.array(RawModuleDefV11Section); + }, +}); +export type RawModuleDefV11 = __Infer; + +// The tagged union or sum type for the algebraic type `RawModuleDefV11Section`. +export const RawModuleDefV11Section = __t.enum('RawModuleDefV11Section', { + get Typespace() { + return Typespace; + }, + get Types() { + return __t.array(RawTypeDefV10); + }, + get Tables() { + return __t.array(RawTableDefV10); + }, + get Reducers() { + return __t.array(RawReducerDefV11); + }, + get Procedures() { + return __t.array(RawProcedureDefV11); + }, + get Views() { + return __t.array(RawViewDefV10); + }, + get Schedules() { + return __t.array(RawScheduleDefV10); + }, + get LifeCycleReducers() { + return __t.array(RawLifeCycleReducerDefV10); + }, + get RowLevelSecurity() { + return __t.array(RawRowLevelSecurityDefV9); + }, + get CaseConversionPolicy() { + return CaseConversionPolicy; + }, + get ExplicitNames() { + return ExplicitNames; + }, + get HttpHandlers() { + return __t.array(RawHttpHandlerDefV10); + }, + get HttpRoutes() { + return __t.array(RawHttpRouteDefV10); + }, + Capabilities: __t.array(__t.string()), +}); +export type RawModuleDefV11Section = __Infer; + export const RawModuleDefV8 = __t.object('RawModuleDefV8', { get typespace() { return Typespace; @@ -442,6 +505,20 @@ export const RawProcedureDefV10 = __t.object('RawProcedureDefV10', { }); export type RawProcedureDefV10 = __Infer; +export const RawProcedureDefV11 = __t.object('RawProcedureDefV11', { + sourceName: __t.string(), + get params() { + return ProductType; + }, + get declaredVisibility() { + return __t.option(FunctionVisibilityV11); + }, + get returnType() { + return AlgebraicType; + }, +}); +export type RawProcedureDefV11 = __Infer; + export const RawProcedureDefV9 = __t.object('RawProcedureDefV9', { name: __t.string(), get params() { @@ -470,6 +547,23 @@ export const RawReducerDefV10 = __t.object('RawReducerDefV10', { }); export type RawReducerDefV10 = __Infer; +export const RawReducerDefV11 = __t.object('RawReducerDefV11', { + sourceName: __t.string(), + get params() { + return ProductType; + }, + get declaredVisibility() { + return __t.option(FunctionVisibilityV11); + }, + get okReturnType() { + return AlgebraicType; + }, + get errReturnType() { + return AlgebraicType; + }, +}); +export type RawReducerDefV11 = __Infer; + export const RawReducerDefV9 = __t.object('RawReducerDefV9', { name: __t.string(), get params() { diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index 0eae2adc2a9..ebf386261f5 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -60,7 +60,7 @@ export type Reducer = ( * Authentication information for the caller of a reducer. */ export type AuthCtx = Readonly<{ - /** Whether the caller is an internal system process. */ + /** Whether the host verified internal invocation authority. Independent of JWT presence. */ isInternal: boolean; /** Whether the caller has authenticated with a JWT token. */ hasJWT: boolean; @@ -92,7 +92,7 @@ export interface JwtClaims { readonly issuer: string; /** The audience of the JWT token ('aud') */ readonly audience: readonly string[]; - /** The identity associated with the JWT token, which is based on the sub and iss */ + /** The verified sender Identity provided by the host, including hosted credentials. */ readonly identity: Identity; /** The full payload as a JsonObject */ readonly fullPayload: JsonObject; diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index ab480c93db5..229fc25b1ad 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -7,8 +7,8 @@ import { } from './algebraic_type'; import type { CaseConversionPolicy, - RawModuleDefV10, - RawModuleDefV10Section, + RawModuleDefV11, + RawModuleDefV11Section, RawScopedTypeNameV10, RawTableDefV10, } from './autogen/types'; @@ -174,10 +174,10 @@ type CompoundTypeCache = Map< >; export type ModuleDef = { - [S in RawModuleDefV10Section as Uncapitalize]: S['value']; + [S in RawModuleDefV11Section as Uncapitalize]: S['value']; }; -type Section = RawModuleDefV10Section; +type Section = RawModuleDefV11Section; export class ModuleContext { #compoundTypes: CompoundTypeCache = new Map(); @@ -197,6 +197,7 @@ export class ModuleContext { lifeCycleReducers: [], httpHandlers: [], httpRoutes: [], + capabilities: ['hosted_auth_v1'], caseConversionPolicy: { tag: 'SnakeCase' }, explicitNames: { entries: [], @@ -207,7 +208,7 @@ export class ModuleContext { return this.#moduleDef; } - rawModuleDefV10(): RawModuleDefV10 { + rawModuleDefV11(): RawModuleDefV11 { const sections: Section[] = []; const push = (s: T | undefined) => { @@ -217,6 +218,7 @@ export class ModuleContext { const module = this.#moduleDef; push(module.typespace && { tag: 'Typespace', value: module.typespace }); + push({ tag: 'Capabilities', value: module.capabilities }); push(module.types && { tag: 'Types', value: module.types }); push(module.tables && { tag: 'Tables', value: module.tables }); push(module.reducers && { tag: 'Reducers', value: module.reducers }); diff --git a/crates/bindings-typescript/src/server/function_visibility.ts b/crates/bindings-typescript/src/server/function_visibility.ts new file mode 100644 index 00000000000..d12d9692e74 --- /dev/null +++ b/crates/bindings-typescript/src/server/function_visibility.ts @@ -0,0 +1,22 @@ +import { FunctionVisibilityV11 } from '../lib/autogen/types'; + +/** Internal functions require verified internal authority. Private functions also + * admit the owner. Public functions admit any authenticated client. */ +export type FunctionVisibility = 'public' | 'private' | 'internal'; + +export function declaredVisibility( + visibility: FunctionVisibility | undefined +): FunctionVisibilityV11 | undefined { + switch (visibility) { + case undefined: + return undefined; + case 'public': + return FunctionVisibilityV11.ClientCallable; + case 'private': + return FunctionVisibilityV11.Private; + case 'internal': + return FunctionVisibilityV11.Internal; + default: + throw new TypeError('Invalid function visibility'); + } +} diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index a840be4a59d..604d9f60ea7 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -10,6 +10,7 @@ export { table } from '../lib/table'; export { SenderError, SpacetimeHostError, errors } from './errors'; export type { Reducer, ReducerCtx, JwtClaims, AuthCtx } from '../lib/reducers'; export type { ReducerExport } from './reducers'; +export type { FunctionVisibility } from './function_visibility'; export { type DbView } from './db_view'; export * from './query'; export type { diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index d07b71f5185..0867c1dfbc6 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -4,12 +4,15 @@ import { type Deserializer, type Serializer, } from '../lib/algebraic_type'; -import { FunctionVisibility } from '../lib/autogen/types'; +import { + declaredVisibility, + type FunctionVisibility, +} from './function_visibility'; import BinaryReader from '../lib/binary_reader'; import BinaryWriter from '../lib/binary_writer'; import type { ConnectionId } from '../lib/connection_id'; import { Identity } from '../lib/identity'; -import type { ParamsObj, ReducerCtx } from '../lib/reducers'; +import type { AuthCtx, ParamsObj, ReducerCtx } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; import { Timestamp } from '../lib/timestamp'; import { @@ -22,7 +25,13 @@ import { Uuid } from '../lib/uuid'; import { httpClient, type HttpClient } from './http_internal'; import type { DbView } from './db_view'; import { makeRandom, type Random } from './rng'; -import { callUserFunction, ReducerCtxImpl, runWithTx, sys } from './runtime'; +import { + AuthCtxImpl, + callUserFunction, + ReducerCtxImpl, + runWithTx, + sys, +} from './runtime'; import { exportContext, registerExport, @@ -47,16 +56,14 @@ export function makeProcedureExport< ret: Ret, fn: ProcedureFn ): ProcedureExport { - const name = opts?.name; - const procedureExport: ProcedureExport = (...args) => fn(...args); procedureExport[exportContext] = ctx; procedureExport[registerExport] = (ctx, exportName) => { - registerProcedure(ctx, name ?? exportName, params, ret, fn); + registerProcedure(ctx, exportName, params, ret, fn, opts); ctx.functionExports.set( procedureExport as ProcedureExport, - name ?? exportName + exportName ); }; @@ -70,7 +77,9 @@ export type ProcedureFn< > = (ctx: ProcedureCtx, args: InferTypeOfRow) => Infer; export interface ProcedureOpts { - name: string; + name?: string; + /** Defaults to public, or private when scheduled. */ + visibility?: FunctionVisibility; } export interface ProcedureCtx { @@ -80,6 +89,7 @@ export interface ProcedureCtx { readonly identity: Identity; readonly timestamp: Timestamp; readonly connectionId: ConnectionId | null; + readonly senderAuth: AuthCtx; readonly http: HttpClient; readonly random: Random; withTx(body: (ctx: TransactionCtx) => T): T; @@ -91,12 +101,6 @@ export interface ProcedureCtx { export interface TransactionCtx extends ReducerCtx {} -type ITransactionCtx = TransactionCtx; - -const TransactionCtxImpl = class TransactionCtx - extends ReducerCtxImpl - implements ITransactionCtx {}; - function registerProcedure< S extends UntypedSchemaDef, Params extends ParamsObj, @@ -124,7 +128,7 @@ function registerProcedure< sourceName: exportName, params: paramsType, returnType, - visibility: FunctionVisibility.ClientCallable, + declaredVisibility: declaredVisibility(opts?.visibility), }); if (opts?.name != null) { @@ -187,6 +191,7 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; + readonly senderAuth: AuthCtx; constructor( readonly sender: Identity, @@ -195,6 +200,11 @@ const ProcedureCtxImpl = class ProcedureCtx dbView: () => DbView ) { this.#dbView = dbView; + this.senderAuth = AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); } get databaseIdentity() { @@ -216,11 +226,12 @@ const ProcedureCtxImpl = class ProcedureCtx withTx(body: (ctx: TransactionCtx) => T): T { return runWithTx( timestamp => - new TransactionCtxImpl( + new ReducerCtxImpl( this.sender, timestamp, this.connectionId, - this.#dbView() + this.#dbView(), + this.senderAuth ) as TransactionCtx, body ); diff --git a/crates/bindings-typescript/src/server/reducers.ts b/crates/bindings-typescript/src/server/reducers.ts index f8aa1c390bf..a43f1821955 100644 --- a/crates/bindings-typescript/src/server/reducers.ts +++ b/crates/bindings-typescript/src/server/reducers.ts @@ -1,5 +1,9 @@ import { AlgebraicType } from '../lib/algebraic_type'; -import { FunctionVisibility, type Lifecycle } from '../lib/autogen/types'; +import { type Lifecycle } from '../lib/autogen/types'; +import { + declaredVisibility, + type FunctionVisibility, +} from './function_visibility'; import type { ParamsObj, Reducer } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; import { RowBuilder, type RowObj } from '../lib/type_builders'; @@ -18,7 +22,9 @@ export interface ReducerExport< ModuleExport {} export interface ReducerOpts { - name: string; + name?: string; + /** Defaults to public, or private when scheduled. Lifecycle hooks are internal. */ + visibility?: FunctionVisibility; } export function makeReducerExport< @@ -73,12 +79,19 @@ export function registerReducer( const ref = ctx.registerTypesRecursively(params); const paramsType = ctx.resolveType(ref).value; const isLifecycle = lifecycle != null; + if ( + isLifecycle && + opts?.visibility != null && + opts.visibility !== 'internal' + ) { + throw new TypeError('Lifecycle reducers only support internal visibility'); + } ctx.moduleDef.reducers.push({ sourceName: exportName, params: paramsType, - //ModuleDef validation code is responsible to mark private reducers - visibility: FunctionVisibility.ClientCallable, + // Preserve omission so the host can apply the scheduled private default. + declaredVisibility: declaredVisibility(opts?.visibility), //Hardcoded for now - reducers do not return values yet okReturnType: AlgebraicType.Product({ elements: [] }), errReturnType: AlgebraicType.String, diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index b3121dc2085..e6fe0ea1841 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,5 +1,6 @@ import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; +import * as _syscalls2_2 from 'spacetime:sys@2.2'; import type { ModuleHooks, u128, u16, u256, u32 } from 'spacetime:sys@2.0'; import { @@ -59,7 +60,7 @@ import { HttpRequest, HttpResponse } from '../lib/autogen/types'; const { freeze } = Object; -export const sys = { ..._syscalls2_0, ..._syscalls2_1 }; +export const sys = { ..._syscalls2_0, ..._syscalls2_1, ..._syscalls2_2 }; function requestFromWire(request: HttpRequest, body: Uint8Array): Request { return Request[makeRequest](body, { @@ -104,7 +105,8 @@ class JwtClaimsImpl implements JwtClaims { /** * Creates a new JwtClaims instance. * @param rawPayload The JWT payload as a raw JSON string. - * @param identity The identity for this JWT. We are only taking this because we don't have a blake3 implementation (which we need to compute it). + * @param identity The verified sender Identity supplied by the host. Claims + * cannot override it, including for hosted database credentials. */ constructor( public readonly rawPayload: string, @@ -132,7 +134,7 @@ class JwtClaimsImpl implements JwtClaims { } } -class AuthCtxImpl implements AuthCtx { +export class AuthCtxImpl implements AuthCtx { public readonly isInternal: boolean; // Source of the JWT payload string, if there is one. @@ -178,29 +180,21 @@ class AuthCtxImpl implements AuthCtx { return this._jwtClaims!; } - /** Create a context representing internal (non-user) requests. */ - static internal(): AuthCtx { - return new AuthCtxImpl({ - isInternal: true, - jwtSource: () => null, - senderIdentity: Identity.zero(), - }); - } - /** If there is a connection id, look up the JWT payload from the system tables. */ static fromSystemTables( connectionId: ConnectionId | null, - sender: Identity + sender: Identity, + callAuthFlags: number ): AuthCtx { if (connectionId === null) { return new AuthCtxImpl({ - isInternal: false, + isInternal: (callAuthFlags & 1) !== 0, jwtSource: () => null, senderIdentity: sender, }); } return new AuthCtxImpl({ - isInternal: false, + isInternal: (callAuthFlags & 1) !== 0, jwtSource: () => { const payloadBuf = sys.get_jwt_payload(connectionId.__connection_id__); if (payloadBuf.length === 0) return null; @@ -219,7 +213,7 @@ export const ReducerCtxImpl = class ReducerCtx< > implements IReducerCtx { #identity: Identity | undefined; - #senderAuth: AuthCtx | undefined; + #senderAuth: AuthCtx; #uuidCounter: { value: number } | undefined; #random: Random | undefined; sender: Identity; @@ -231,13 +225,21 @@ export const ReducerCtxImpl = class ReducerCtx< sender: Identity, timestamp: Timestamp, connectionId: ConnectionId | null, - dbView: DbView + dbView: DbView, + senderAuth?: AuthCtx ) { Object.seal(this); this.sender = sender; this.timestamp = timestamp; this.connectionId = connectionId; this.db = dbView; + this.#senderAuth = + senderAuth ?? + AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); } /** Reset the `ReducerCtx` to be used for a new transaction */ @@ -251,7 +253,11 @@ export const ReducerCtxImpl = class ReducerCtx< me.timestamp = timestamp; me.connectionId = connectionId; me.#uuidCounter = undefined; - me.#senderAuth = undefined; + me.#senderAuth = AuthCtxImpl.fromSystemTables( + connectionId, + sender, + sys.get_call_auth_flags() + ); } get databaseIdentity() { @@ -263,10 +269,7 @@ export const ReducerCtxImpl = class ReducerCtx< } get senderAuth() { - return (this.#senderAuth ??= AuthCtxImpl.fromSystemTables( - this.connectionId, - this.sender - )); + return this.#senderAuth; } get random() { @@ -378,7 +381,7 @@ class ModuleHooksImpl implements ModuleHooks { const writer = new BinaryWriter(128); RawModuleDef.serialize( writer, - RawModuleDef.V10(this.#schema.rawModuleDefV10()) + RawModuleDef.V11(this.#schema.rawModuleDefV11()) ); return writer.getBuffer(); } diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index d9f20be3025..97ddd4da26a 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -277,7 +277,11 @@ export class Schema implements ModuleDefaultExport { case 2: { let arg1; [arg1, fn] = args; - if (typeof arg1.name === 'string') opts = arg1 as ReducerOpts; + if ( + typeof arg1.name === 'string' || + typeof arg1.visibility === 'string' + ) + opts = arg1 as ReducerOpts; else params = arg1 as Params; break; } @@ -461,22 +465,22 @@ export class Schema implements ModuleDefaultExport { params: Params, ret: Ret, fn: ProcedureFn - ): ProcedureFn; + ): ProcedureExport; procedure>( ret: Ret, fn: ProcedureFn - ): ProcedureFn; + ): ProcedureExport; procedure>( opts: ProcedureOpts, params: Params, ret: Ret, fn: ProcedureFn - ): ProcedureFn; + ): ProcedureExport; procedure>( opts: ProcedureOpts, ret: Ret, fn: ProcedureFn - ): ProcedureFn; + ): ProcedureExport; procedure>( ...args: | [Params, Ret, ProcedureFn] @@ -495,7 +499,11 @@ export class Schema implements ModuleDefaultExport { case 3: { let arg1; [arg1, ret, fn] = args; - if (typeof arg1.name === 'string') opts = arg1 as ProcedureOpts; + if ( + typeof arg1.name === 'string' || + typeof arg1.visibility === 'string' + ) + opts = arg1 as ProcedureOpts; else params = arg1 as Params; break; } diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index f0315867cb3..97282ae189e 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -123,3 +123,8 @@ declare module 'spacetime:sys@2.0' { declare module 'spacetime:sys@2.1' { export function datastore_clear(table_id: u32): u64; } + +declare module 'spacetime:sys@2.2' { + /** Verified invocation flags. Bit 0 is INTERNAL; JWT presence is independent. */ + export function get_call_auth_flags(): number; +} diff --git a/crates/bindings-typescript/tests/hosted_auth.test.ts b/crates/bindings-typescript/tests/hosted_auth.test.ts new file mode 100644 index 00000000000..db7d14d8433 --- /dev/null +++ b/crates/bindings-typescript/tests/hosted_auth.test.ts @@ -0,0 +1,225 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const host = vi.hoisted(() => ({ + flags: 0, + payload: '', + jwtReads: 0, + flagReads: 0, +})); +vi.mock('spacetime:sys@2.0', () => ({ + moduleHooks: Symbol('moduleHooks'), + identity: () => 1n, + row_iter_bsatn_close: () => {}, + procedure_start_mut_tx: () => 0n, + procedure_commit_mut_tx: () => {}, + procedure_abort_mut_tx: () => {}, + get_jwt_payload: () => { + host.jwtReads++; + return new TextEncoder().encode(host.payload); + }, +})); +vi.mock('spacetime:sys@2.1', () => ({})); +vi.mock('spacetime:sys@2.2', () => ({ + get_call_auth_flags: () => { + host.flagReads++; + return host.flags; + }, +})); + +import { ReducerCtxImpl } from '../src/server/runtime'; +import { ConnectionId } from '../src/lib/connection_id'; +import { Identity } from '../src/lib/identity'; +import { Timestamp } from '../src/lib/timestamp'; +import { schema, exportContext, registerExport } from '../src/server/schema'; +import { callProcedure } from '../src/server/procedures'; +import { t } from '../src/lib/type_builders'; +import { RawModuleDef } from '../src/lib/autogen/types'; +import BinaryReader from '../src/lib/binary_reader'; +import BinaryWriter from '../src/lib/binary_writer'; + +beforeEach(() => { + Object.assign(host, { flags: 0, payload: '', jwtReads: 0, flagReads: 0 }); +}); + +describe('verified invocation authentication', () => { + it.each([0, 1])( + 'preserves flag %s for calls without a connection or JWT', + flags => { + host.flags = flags; + const ctx = new ReducerCtxImpl( + new Identity(1n), + Timestamp.UNIX_EPOCH, + null, + {} + ); + host.flags = flags ^ 1; + expect(host.flagReads).toBe(1); + expect(ctx.senderAuth.isInternal).toBe(Boolean(flags)); + expect(ctx.senderAuth.hasJWT).toBe(false); + expect(ctx.senderAuth.jwt).toBeNull(); + expect(host.jwtReads).toBe(0); + } + ); + + it('retains an internal connection and JWT independently, using the verified sender Identity', () => { + host.flags = 1; + host.payload = JSON.stringify({ + iss: 'unrelated-issuer', + sub: 'unrelated-subject', + identity: 'untrusted-claim', + }); + const sender = new Identity(123n); + const connection = new ConnectionId(7n); + const ctx = new ReducerCtxImpl( + sender, + Timestamp.UNIX_EPOCH, + connection, + {} + ); + host.flags = 0; + expect(ctx.connectionId).toBe(connection); + expect(ctx.senderAuth.isInternal).toBe(true); + expect(host.jwtReads).toBe(0); + expect(ctx.senderAuth.hasJWT).toBe(true); + expect(ctx.senderAuth.jwt?.identity).toBe(sender); + expect(ctx.senderAuth.jwt?.subject).toBe('unrelated-subject'); + expect(host.jwtReads).toBe(1); + }); + + it('refreshes captured flags and sender when a cached reducer context is reused', () => { + host.flags = 1; + const ctx = new ReducerCtxImpl( + new Identity(1n), + Timestamp.UNIX_EPOCH, + null, + {} + ); + const firstAuth = ctx.senderAuth; + host.flags = 0; + ReducerCtxImpl.reset( + ctx, + new Identity(2n), + Timestamp.UNIX_EPOCH, + new ConnectionId(8n) + ); + host.flags = 1; + expect(firstAuth.isInternal).toBe(true); + expect(ctx.senderAuth.isInternal).toBe(false); + expect(ctx.senderAuth.hasJWT).toBe(false); + }); + + it('preserves procedure auth inside a transaction after the host flags change', () => { + host.flags = 1; + const module = schema({}); + const proc = module.procedure(t.unit(), ctx => { + host.flags = 0; + ctx.withTx(tx => { + expect(tx.senderAuth).toBe(ctx.senderAuth); + expect(tx.senderAuth.isInternal).toBe(true); + expect(tx.connectionId).toBe(ctx.connectionId); + }); + return {}; + }); + const inner = proc[exportContext]!; + proc[registerExport](inner, 'procedure_auth'); + callProcedure( + inner, + 0, + new Identity(9n), + new ConnectionId(8n), + Timestamp.UNIX_EPOCH, + new Uint8Array(), + () => ({}) + ); + expect(host.flagReads).toBe(1); + }); +}); + +describe('V11 explicit function visibility', () => { + it('serializes omission separately from explicit visibility and advertises hosted auth', () => { + const module = schema({}); + const omitted = module.reducer(() => {}); + const explicitlyPublic = module.reducer({ visibility: 'public' }, () => {}); + const privateReducer = module.reducer({ visibility: 'private' }, () => {}); + const internalReducer = module.reducer( + { visibility: 'internal' }, + () => {} + ); + const inner = omitted[exportContext]!; + for (const [name, reducer] of Object.entries({ + omitted, + explicitlyPublic, + privateReducer, + internalReducer, + })) { + reducer[registerExport](inner, name); + } + // Being scheduled must not erase a public choice or manufacture an explicit + // choice for the default. The host resolves the latter to Private. + inner.moduleDef.schedules.push({ + sourceName: undefined, + tableName: 'jobs', + scheduleAtCol: 0, + functionName: 'explicitlyPublic', + }); + const raw = RawModuleDef.V11(inner.rawModuleDefV11()); + const writer = new BinaryWriter(128); + RawModuleDef.serialize(writer, raw); + expect(writer.getBuffer()[0]).toBe(3); + const decoded = RawModuleDef.deserialize( + new BinaryReader(writer.getBuffer()) + ); + const roundTrip = new BinaryWriter(128); + RawModuleDef.serialize(roundTrip, decoded); + expect(roundTrip.getBuffer()).toEqual(writer.getBuffer()); + expect(decoded.tag).toBe('V11'); + if (decoded.tag !== 'V11') throw new Error('Expected V11'); + const reducers = decoded.value.sections.find( + section => section.tag === 'Reducers' + ); + expect( + reducers?.value.map(reducer => reducer.declaredVisibility?.tag) + ).toEqual([undefined, 'ClientCallable', 'Private', 'Internal']); + expect( + inner.moduleDef.reducers.map(reducer => reducer.declaredVisibility?.tag) + ).toEqual([undefined, 'ClientCallable', 'Private', 'Internal']); + expect(inner.moduleDef.capabilities).toEqual(['hosted_auth_v1']); + }); + + it('retains procedure names and explicit visibility, including a visibility parameter', () => { + const module = schema({}); + const proc = module.procedure( + { name: 'public_name', visibility: 'internal' }, + t.unit(), + () => ({}) + ); + const reducer = module.reducer({ visibility: t.string() }, () => {}); + const inner = proc[exportContext]!; + proc[registerExport](inner, 'source_name'); + reducer[registerExport](inner, 'accept_visibility'); + expect(inner.moduleDef.procedures[0].sourceName).toBe('source_name'); + expect(inner.moduleDef.procedures[0].declaredVisibility?.tag).toBe( + 'Internal' + ); + expect(inner.moduleDef.explicitNames.entries).toContainEqual({ + tag: 'Function', + value: { sourceName: 'source_name', canonicalName: 'public_name' }, + }); + expect(inner.moduleDef.reducers[0].params.elements[0].name).toBe( + 'visibility' + ); + }); + + it('rejects externally callable lifecycle declarations', () => { + const module = schema({}); + const invalid = module.init({ visibility: 'private' }, () => {}); + expect(() => + invalid[registerExport](invalid[exportContext]!, 'invalid_init') + ).toThrow('Lifecycle reducers only support internal visibility'); + const valid = module.init({ visibility: 'internal' }, () => {}); + valid[registerExport](valid[exportContext]!, 'valid_init'); + expect( + valid[exportContext]!.moduleDef.reducers[0].declaredVisibility?.tag + ).toBe('Internal'); + }); +}); diff --git a/crates/bindings/src/http.rs b/crates/bindings/src/http.rs index a0ee89819ba..f3fc86af45c 100644 --- a/crates/bindings/src/http.rs +++ b/crates/bindings/src/http.rs @@ -7,7 +7,7 @@ use crate::{ rt::{read_bytes_source_as, read_bytes_source_into}, - try_with_tx, with_tx, IterBuf, StdbRng, Timestamp, TxContext, + try_with_tx, with_tx, AuthCtx, IterBuf, StdbRng, Timestamp, TxContext, }; use bytes::Bytes; #[cfg(feature = "rand")] @@ -88,6 +88,7 @@ pub struct HandlerContext { /// Methods for performing HTTP requests. pub http: HttpClient, + sender_auth: AuthCtx, #[cfg(feature = "rand08")] pub(crate) rng: OnceCell, @@ -103,6 +104,7 @@ impl HandlerContext { Self { timestamp, http: HttpClient {}, + sender_auth: AuthCtx::from_invocation(Identity::ZERO, None), #[cfg(feature = "rand08")] rng: OnceCell::new(), #[cfg(feature = "rand")] @@ -117,12 +119,12 @@ impl HandlerContext { /// Acquire a mutable transaction and execute `body` with read-write access. pub fn with_tx(&mut self, body: impl Fn(&TxContext) -> T) -> T { - with_tx(body) + with_tx(Identity::ZERO, None, &self.sender_auth, body) } /// Acquire a mutable transaction and execute `body` with read-write access. pub fn try_with_tx(&mut self, body: impl Fn(&TxContext) -> Result) -> Result { - try_with_tx(body) + try_with_tx(Identity::ZERO, None, &self.sender_auth, body) } /// Create a new random [`Uuid`] `v4` using the built-in RNG. diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index abc972748dd..2f108f9dd9c 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -1049,12 +1049,23 @@ impl ReducerContext { #[doc(hidden)] fn new(db: Local, sender: Identity, connection_id: Option, timestamp: Timestamp) -> Self { + let sender_auth = AuthCtx::from_invocation(sender, connection_id); + Self::new_with_auth(db, sender, connection_id, timestamp, sender_auth) + } + + fn new_with_auth( + db: Local, + sender: Identity, + connection_id: Option, + timestamp: Timestamp, + sender_auth: AuthCtx, + ) -> Self { Self { db, sender, timestamp, connection_id, - sender_auth: AuthCtx::from_connection_id_opt(connection_id), + sender_auth, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), #[cfg(feature = "rand")] @@ -1184,7 +1195,12 @@ impl Deref for TxContext { } #[cfg(feature = "unstable")] -fn try_with_tx(body: impl Fn(&TxContext) -> Result) -> Result { +fn try_with_tx( + sender: Identity, + connection_id: Option, + sender_auth: &AuthCtx, + body: impl Fn(&TxContext) -> Result, +) -> Result { let abort = || { crate::sys::procedure::procedure_abort_mut_tx() .expect("should have a pending mutable anon tx as `procedure_start_mut_tx` preceded") @@ -1195,8 +1211,8 @@ fn try_with_tx(body: impl Fn(&TxContext) -> Result) -> Result .expect("holding `&mut HandlerContext`, so should not be in a tx already; called manually elsewhere?"); let timestamp = Timestamp::from_micros_since_unix_epoch(timestamp); - // Use the internal auth context (no external caller identity). - let tx = ReducerContext::new(crate::Local {}, Identity::ZERO, None, timestamp); + // Every retry retains the original invocation's identity and authority. + let tx = ReducerContext::new_with_auth(crate::Local {}, sender, connection_id, timestamp, sender_auth.clone()); let tx = TxContext(tx); struct DoOnDrop(F); @@ -1230,9 +1246,14 @@ fn try_with_tx(body: impl Fn(&TxContext) -> Result) -> Result } #[cfg(feature = "unstable")] -fn with_tx(body: impl Fn(&TxContext) -> T) -> T { +fn with_tx( + sender: Identity, + connection_id: Option, + sender_auth: &AuthCtx, + body: impl Fn(&TxContext) -> T, +) -> T { use core::convert::Infallible; - match try_with_tx::(|tx| Ok(body(tx))) { + match try_with_tx::(sender, connection_id, sender_auth, |tx| Ok(body(tx))) { Ok(v) => v, Err(e) => match e {}, } @@ -1257,6 +1278,7 @@ pub struct ProcedureContext { /// /// Will be `None` for certain scheduled procedures. connection_id: Option, + sender_auth: AuthCtx, /// Methods for performing HTTP requests. pub http: crate::http::HttpClient, @@ -1279,6 +1301,7 @@ impl ProcedureContext { sender, timestamp, connection_id, + sender_auth: AuthCtx::from_invocation(sender, connection_id), http: http::HttpClient {}, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), @@ -1287,6 +1310,11 @@ impl ProcedureContext { } } + /// Host-verified authentication for this invocation, retained in transactions. + pub fn sender_auth(&self) -> &AuthCtx { + &self.sender_auth + } + /// The `Identity` of the client that invoked the procedure. pub fn sender(&self) -> Identity { self.sender @@ -1368,7 +1396,7 @@ impl ProcedureContext { /// This includes interior mutability through types like [`std::cell::Cell`]. #[cfg(feature = "unstable")] pub fn with_tx(&mut self, body: impl Fn(&TxContext) -> T) -> T { - with_tx(body) + with_tx(self.sender, self.connection_id, &self.sender_auth, body) } /// Acquire a mutable transaction @@ -1402,7 +1430,7 @@ impl ProcedureContext { /// This includes interior mutability through types like [`std::cell::Cell`]. #[cfg(feature = "unstable")] pub fn try_with_tx(&mut self, body: impl Fn(&TxContext) -> Result) -> Result { - try_with_tx(body) + try_with_tx(self.sender, self.connection_id, &self.sender_auth, body) } /// Create a new random [`Uuid`] `v4` using the built-in RNG. @@ -1547,6 +1575,7 @@ impl Local { /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token #[non_exhaustive] pub struct JwtClaims { + identity: Identity, payload: String, parsed: OnceCell, audience: OnceCell>, @@ -1562,10 +1591,16 @@ pub struct AuthCtx { } impl AuthCtx { - /// Creates an [`AuthCtx`] both for cases where there's a [`ConnectionId`] - /// and for when there isn't. - fn from_connection_id_opt(conn_id: Option) -> Self { - conn_id.map(Self::from_connection_id).unwrap_or_else(Self::internal) + /// Capture host authority immediately. JWT loading remains independent and lazy. + fn from_invocation(sender: Identity, connection_id: Option) -> Self { + let flags = spacetimedb_bindings_sys::get_call_auth_flags(); + Self::from_host_auth(sender, flags, move || connection_id.and_then(rt::get_jwt)) + } + + fn from_host_auth(sender: Identity, flags: u32, jwt_fn: impl FnOnce() -> Option + 'static) -> Self { + Self::new(flags & 1 != 0, move || { + jwt_fn().map(|payload| JwtClaims::new(payload, sender)) + }) } fn new(is_internal: bool, jwt_fn: impl FnOnce() -> Option + 'static) -> Self { @@ -1588,14 +1623,18 @@ impl AuthCtx { /// /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token pub fn from_jwt_payload(jwt_payload: String) -> AuthCtx { - Self::new(false, move || Some(JwtClaims::new(jwt_payload))) + let parsed: serde_json::Value = serde_json::from_str(&jwt_payload).expect("invalid test JWT payload"); + let sender = Identity::from_claims( + parsed["iss"].as_str().expect("missing test issuer"), + parsed["sub"].as_str().expect("missing test subject"), + ); + Self::from_jwt_payload_for_sender(jwt_payload, sender, false) } - /// Creates an [`AuthCtx`] that reads the [JWT] for the given connection id. - /// - /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token - fn from_connection_id(connection_id: ConnectionId) -> AuthCtx { - Self::new(false, move || rt::get_jwt(connection_id).map(JwtClaims::new)) + /// Create test authentication with an explicit effective sender and authority. + /// Production contexts receive these values from the host, not JWT claims. + pub fn from_jwt_payload_for_sender(jwt_payload: String, sender: Identity, is_internal: bool) -> AuthCtx { + Self::new(is_internal, move || Some(JwtClaims::new(jwt_payload, sender))) } /// Returns whether this reducer was spawned from inside the database. @@ -1603,8 +1642,8 @@ impl AuthCtx { self.is_internal } - /// Checks if there is a [JWT] without loading it. - /// If [`AuthCtx::is_internal`] returns true, this will return false. + /// Returns whether this invocation has a [JWT]. Internal invocations may + /// also carry a JWT; internal authority and credential presence are independent. /// /// [JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token pub fn has_jwt(&self) -> bool { @@ -1620,8 +1659,9 @@ impl AuthCtx { } impl JwtClaims { - fn new(jwt: String) -> Self { + fn new(jwt: String, identity: Identity) -> Self { Self { + identity, payload: jwt, parsed: OnceCell::new(), audience: OnceCell::new(), @@ -1663,10 +1703,10 @@ impl JwtClaims { self.audience.get_or_init(|| self.extract_audience()) } - /// Returns the identity for these credentials, which is - /// based on the iss and sub claims. + /// The effective sender verified by the host for this invocation. + /// Hosted database credentials need not derive this Identity from iss/sub. pub fn identity(&self) -> Identity { - Identity::from_claims(self.issuer(), self.subject()) + self.identity } /// Get the whole JWT payload as a json string. @@ -1819,3 +1859,54 @@ mod tests { assert_eq!(audience, &["my-project-id".to_string()]); } } + +#[cfg(test)] +mod hosted_auth_tests { + use super::*; + + #[test] + fn host_authority_is_independent_of_jwt_presence() { + for internal in [false, true] { + for has_jwt in [false, true] { + let flags = u32::from(internal) | (1 << 31); + let auth = AuthCtx::from_host_auth(Identity::ONE, flags, move || { + has_jwt.then(|| r#"{"iss":"hosted","sub":"generation","hex_identity":"untrusted"}"#.to_string()) + }); + assert_eq!(auth.is_internal(), internal); + assert_eq!(auth.has_jwt(), has_jwt); + if let Some(jwt) = auth.jwt() { + assert_eq!(jwt.identity(), Identity::ONE); + assert_ne!(jwt.identity(), Identity::from_claims(jwt.issuer(), jwt.subject())); + } + } + } + } + + #[test] + fn verified_sender_wins_over_signed_payload_identity_claims() { + let payload = format!( + r#"{{"iss":"hosted","sub":"generation","hex_identity":"{}"}}"#, + Identity::ZERO.to_hex() + ); + let auth = AuthCtx::from_host_auth(Identity::ONE, 1, move || Some(payload)); + assert_eq!(auth.jwt().unwrap().identity(), Identity::ONE); + assert!(auth.is_internal()); + assert!(auth.has_jwt()); + } + + #[test] + fn transaction_context_preserves_identity_connection_and_authentication() { + let sender = Identity::ONE; + let connection = Some(ConnectionId::from_u128(123)); + let auth = AuthCtx::from_host_auth(sender, 1, || Some(r#"{"iss":"hosted","sub":"generation"}"#.into())); + // Procedure transactions/retries call this same constructor with the + // original captured AuthCtx rather than manufacturing internal authority. + for timestamp in [Timestamp::UNIX_EPOCH, Timestamp::from_micros_since_unix_epoch(1)] { + let context = ReducerContext::new_with_auth(Local {}, sender, connection, timestamp, auth.clone()); + assert_eq!(context.sender(), sender); + assert_eq!(context.connection_id(), connection); + assert!(context.sender_auth().is_internal()); + assert_eq!(context.sender_auth().jwt().unwrap().identity(), sender); + } + } +} diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index 79543f4a925..e66b70cd96c 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1,12 +1,13 @@ #![deny(unsafe_op_in_unsafe_fn)] +pub use spacetimedb_lib::db::raw_def::v11::FunctionVisibility; +use spacetimedb_lib::db::raw_def::v11::RawModuleDefV11Builder; + use crate::query_builder::{FromWhere, HasCols, LeftSemiJoin, RawQuery, RightSemiJoin, Table as QbTable}; use crate::table::IndexAlgo; use crate::{sys, AnonymousViewContext, IterBuf, ReducerContext, ReducerResult, SpacetimeType, Table, ViewContext}; use spacetimedb_lib::bsatn::EncodeError; -use spacetimedb_lib::db::raw_def::v10::{ - CaseConversionPolicy, ExplicitNames as RawExplicitNames, RawModuleDefV10Builder, -}; +use spacetimedb_lib::db::raw_def::v10::{CaseConversionPolicy, ExplicitNames as RawExplicitNames}; pub use spacetimedb_lib::db::raw_def::v9::Lifecycle as LifecycleReducer; use spacetimedb_lib::db::raw_def::v9::{RawIndexAlgorithm, TableType, ViewResultHeader}; use spacetimedb_lib::de::{self, Deserialize, DeserializeOwned, Error as _, SeqProductAccess}; @@ -166,6 +167,9 @@ pub trait FnInfo: ExplicitNames { /// The lifecycle of the function, if there is one. const LIFECYCLE: Option = None; + /// Explicit SpacetimeDB visibility; Rust item visibility is independent. + const DECLARED_VISIBILITY: Option = None; + /// A description of the parameter names of the function. const ARG_NAMES: &'static [Option<&'static str>]; @@ -819,9 +823,13 @@ pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl register_describer(|module| { let params = A::schema::(&mut module.inner); if let Some(lifecycle) = I::LIFECYCLE { - module.inner.add_lifecycle_reducer(lifecycle, I::NAME, params); + module + .inner + .add_lifecycle_reducer_with_visibility(lifecycle, I::NAME, params, I::DECLARED_VISIBILITY); } else { - module.inner.add_reducer(I::NAME, params); + module + .inner + .add_reducer_with_visibility(I::NAME, params, I::DECLARED_VISIBILITY); } module.reducers.push(I::INVOKE); @@ -839,7 +847,9 @@ where register_describer(|module| { let params = A::schema::(&mut module.inner); let ret_ty = ::make_type(&mut module.inner); - module.inner.add_procedure(I::NAME, params, ret_ty); + module + .inner + .add_procedure_with_visibility(I::NAME, params, ret_ty, I::DECLARED_VISIBILITY); module.procedures.push(I::INVOKE); module.inner.add_explicit_names(I::explicit_names()); @@ -931,7 +941,7 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { #[derive(Default)] pub struct ModuleBuilder { /// The module definition. - inner: RawModuleDefV10Builder, + inner: RawModuleDefV11Builder, /// The reducers of the module. reducers: Vec, /// The procedures of the module. @@ -995,9 +1005,12 @@ extern "C" fn __describe_module__(description: BytesSink) { describer(&mut module) } + // These bindings capture host flags and preserve the verified sender in JWT claims. + module.inner.add_capability("hosted_auth_v1"); + // Serialize the module to bsatn. let module_def = module.inner.finish(); - let module_def = RawModuleDef::V10(module_def); + let module_def = RawModuleDef::V11(module_def); let bytes = bsatn::to_vec(&module_def).expect("unable to serialize typespace"); // Write the sets of reducers, procedures and views. diff --git a/crates/bindings/tests/pass/function_visibility.rs b/crates/bindings/tests/pass/function_visibility.rs new file mode 100644 index 00000000000..6fe9cfa03f3 --- /dev/null +++ b/crates/bindings/tests/pass/function_visibility.rs @@ -0,0 +1,56 @@ +#![deny(warnings)] + +use spacetimedb::rt::{FnInfo, FunctionVisibility}; +use spacetimedb::{ProcedureContext, ReducerContext}; + +#[spacetimedb::reducer(internal)] +pub fn internal_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(private)] +fn private_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(public)] +fn public_reducer(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer(init, internal)] +fn initialize(_ctx: &ReducerContext) {} + +#[spacetimedb::procedure(internal)] +fn internal_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +#[spacetimedb::procedure(private)] +fn private_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +#[spacetimedb::procedure(public)] +fn public_procedure(_ctx: &mut ProcedureContext) -> u64 { + 0 +} + +fn main() { + assert_eq!( + internal_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + ); + assert_eq!(private_reducer::DECLARED_VISIBILITY, Some(FunctionVisibility::Private)); + assert_eq!( + public_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::ClientCallable) + ); + assert_eq!(initialize::DECLARED_VISIBILITY, Some(FunctionVisibility::Internal)); + assert_eq!( + internal_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + ); + assert_eq!( + private_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::Private) + ); + assert_eq!( + public_procedure::DECLARED_VISIBILITY, + Some(FunctionVisibility::ClientCallable) + ); +} diff --git a/crates/bindings/tests/ui/tables.stderr b/crates/bindings/tests/ui/tables.stderr index 7609d9ba378..e6f65b174bb 100644 --- a/crates/bindings/tests/ui/tables.stderr +++ b/crates/bindings/tests/ui/tables.stderr @@ -209,13 +209,13 @@ error[E0277]: `&'a Alpha` cannot appear as an argument to an index filtering ope = note: The allowed set of types are limited to integers, bool, strings, `Identity`, `Uuid`, `Timestamp`, `ConnectionId`, `Hash` and no-payload enums which derive `SpacetimeType`, = help: the following other types implement trait `FilterableValue`: &ConnectionId - &FunctionVisibility + &ContainerMode + &FunctionVisibilityV11 &Identity &Lifecycle - &TableAccess - &TableType - &bool - ðnum::int::I256 + &PortExposure + &PortProtocol + &RestartPolicy and $N others note: required by a bound in `UniqueColumn::::ColType, Col>::find` --> src/table.rs @@ -241,13 +241,13 @@ help: the trait `FilterableValue` is not implemented for `Alpha` | ^^^^^^^^^^^^ = help: the following other types implement trait `FilterableValue`: &ConnectionId - &FunctionVisibility + &ContainerMode + &FunctionVisibilityV11 &Identity &Lifecycle - &TableAccess - &TableType - &bool - ðnum::int::I256 + &PortExposure + &PortProtocol + &RestartPolicy and $N others = note: required for `Alpha` to implement `IndexScanRangeBounds<(Alpha,), SingleBound>` note: required by a bound in `RangedIndex::::filter` diff --git a/crates/cli/src/api.rs b/crates/cli/src/api.rs index d40b03ee87e..53fdc28a39a 100644 --- a/crates/cli/src/api.rs +++ b/crates/cli/src/api.rs @@ -4,7 +4,7 @@ use std::ops::Add; use reqwest::{header, Client, RequestBuilder}; use serde::Deserialize; -use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; +use spacetimedb_lib::db::raw_def::v11::RawModuleDefV11; use spacetimedb_lib::de::serde::DeserializeWrapper; use spacetimedb_lib::Identity; @@ -61,11 +61,11 @@ impl ClientApi { } /// Reads the `ModuleDef` from the `schema` endpoint. - pub async fn module_def(&self) -> anyhow::Result { + pub async fn module_def(&self) -> anyhow::Result { let res = self .client .get(self.con.db_uri("schema")) - .query(&[("version", "9")]) + .query(&[("version", "11")]) .send() .await?; let DeserializeWrapper(module_def) = res.json_or_error().await?; diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index e38542595c8..c44b3d2a41e 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -36,7 +36,6 @@ pub fn get_subcommands() -> Vec { init::cli(), build::cli(), server::cli(), - sidecar::cli(), subscribe::cli(), start::cli(), subcommands::version::cli(), @@ -64,7 +63,6 @@ pub async fn exec_subcommand( "init" => init::exec(config, args).await.map(|_| ()), "build" => build::exec(config, args).await.map(drop), "server" => server::exec(config, paths, args).await, - "sidecar" => sidecar::exec(config, args).await, "subscribe" => subscribe::exec(config, args).await, "start" => return start::exec(config, paths, args).await, "login" => login::exec(config, args).await, diff --git a/crates/cli/src/subcommands/describe.rs b/crates/cli/src/subcommands/describe.rs index e774224855c..735968a818d 100644 --- a/crates/cli/src/subcommands/describe.rs +++ b/crates/cli/src/subcommands/describe.rs @@ -98,6 +98,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error let api = ClientApi::new(conn); let module_def = api.module_def().await?; + let canonical = spacetimedb_schema::def::ModuleDef::try_from(module_def.clone())?; if json { fn sats_to_json(v: &T) -> serde_json::Result { @@ -105,18 +106,21 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error } let json = match entity { Some((EntityType::Reducer, reducer_name)) => { + let source_name = &canonical + .reducer(reducer_name) + .context("no such reducer")? + .accessor_name; let reducer = module_def - .reducers - .iter() - .find(|r| *r.name == *reducer_name) + .reducers() + .find(|r| *r.source_name == **source_name) .context("no such reducer")?; sats_to_json(reducer)? } Some((EntityType::Table, table_name)) => { + let source_name = &canonical.table(table_name).context("no such table")?.accessor_name; let table = module_def - .tables - .iter() - .find(|t| *t.name == *table_name) + .tables() + .find(|t| *t.source_name == **source_name) .context("no such table")?; sats_to_json(table)? } diff --git a/crates/cli/src/subcommands/generate.rs b/crates/cli/src/subcommands/generate.rs index 6e5378fede5..234cd3eef82 100644 --- a/crates/cli/src/subcommands/generate.rs +++ b/crates/cli/src/subcommands/generate.rs @@ -260,7 +260,7 @@ pub fn cli() -> clap::Command { .long("include-private") .action(SetTrue) .default_value("false") - .help("Include private tables and functions in generated code (types are always included)."), + .help("Include private tables and private/internal non-lifecycle functions (types are always included)."), ) .arg(common_args::yes()) .arg( diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index 5d993a88b3e..58456274469 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -14,7 +14,6 @@ pub mod logs; pub mod publish; pub mod repl; pub mod server; -pub mod sidecar; pub mod sql; pub mod start; pub mod subscribe; diff --git a/crates/cli/src/subcommands/sidecar.rs b/crates/cli/src/subcommands/sidecar.rs deleted file mode 100644 index 68a422275a7..00000000000 --- a/crates/cli/src/subcommands/sidecar.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! `spacetime sidecar` (PROTOTYPE) -//! -//! Manage sidecars: long-running programs (e.g. agents) hosted beside a -//! database. This command is a thin client: it records the desired state by -//! calling the control database's `create_sidecar` / `delete_sidecar` reducers. -//! The actual container is launched and reconciled by the SpacetimeDB Cloud -//! node hosting the database (see `private/crates/cloud`); SpacetimeDB -//! Standalone has no control plane and does not support sidecars. -//! -//! See `proposals/00XX-agent-hosting.md`. - -use crate::api::{ClientApi, Connection}; -use crate::common_args; -use crate::config::Config; -use crate::util::{database_identity, get_auth_header, AuthHeader, UNSTABLE_WARNING}; -use anyhow::{bail, Context}; -use clap::{Arg, ArgAction, ArgMatches}; -use spacetimedb_lib::Identity; - -pub fn cli() -> clap::Command { - clap::Command::new("sidecar") - .about(format!( - "Manage sidecars (long-running programs hosted beside a database). {UNSTABLE_WARNING}" - )) - .args_conflicts_with_subcommands(true) - .subcommand_required(true) - .subcommands([ - clap::Command::new("run") - .about("Declare a sidecar for a database; the hosting node launches it") - .arg(Arg::new("database").required(true).help("The database name or identity")) - .arg(Arg::new("image").long("image").required(true).help("The OCI image to run")) - .arg( - Arg::new("name") - .long("name") - .default_value("default") - .help("A name for this sidecar, allowing several per database"), - ) - .arg( - Arg::new("env") - .long("env") - .short('e') - .value_name("KEY=VALUE") - .action(ArgAction::Append) - .help("Extra environment variables to inject (repeatable)"), - ) - .arg( - Arg::new("command") - .num_args(0..) - .last(true) - .help("Optional command to run in the container, after `--`"), - ) - .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) - .arg(common_args::anonymous()) - .arg(common_args::yes()), - clap::Command::new("ls") - .about("List declared sidecars") - .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")), - clap::Command::new("stop") - .about("Remove a sidecar from a database; the hosting node stops it") - .arg(Arg::new("database").required(true).help("The database name or identity")) - .arg( - Arg::new("name") - .long("name") - .default_value("default") - .help("The sidecar name to remove"), - ) - .arg(common_args::server().help("The nickname, host name or URL of the server hosting the database")) - .arg(common_args::anonymous()) - .arg(common_args::yes()), - ]) - .after_help("Run `spacetime help sidecar` for more detailed information.\n") -} - -pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { - eprintln!("{UNSTABLE_WARNING}\n"); - let (cmd, sub) = args.subcommand().expect("subcommand required"); - match cmd { - "run" => exec_run(config, sub).await, - "ls" => exec_ls(config, sub).await, - "stop" => exec_stop(config, sub).await, - unknown => bail!("Invalid subcommand: {unknown}"), - } -} - -/// The control database is addressed by the all-zero identity. -fn control_api(config: &Config, server: Option<&str>, auth: AuthHeader) -> anyhow::Result { - Ok(ClientApi::new(Connection { - host: config.get_host_url(server)?, - database_identity: Identity::ZERO, - database: "control".to_string(), - auth_header: auth, - })) -} - -async fn exec_run(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { - let server = args.get_one::("server").map(|s| s.as_str()); - let force = args.get_flag("force"); - let anon = args.get_flag("anon_identity"); - let database = args.get_one::("database").unwrap(); - let image = args.get_one::("image").unwrap(); - let name = args.get_one::("name").unwrap(); - let mut env: Vec = args - .get_many::("env") - .map(|v| v.cloned().collect()) - .unwrap_or_default(); - let command: Vec = args - .get_many::("command") - .map(|v| v.cloned().collect()) - .unwrap_or_default(); - - let host = config.get_host_url(server)?; - let db_identity = database_identity(&config, database, server).await?; - let auth = get_auth_header(&mut config, anon, server, !force).await?; - let token = auth - .token() - .context("No auth token available; run `spacetime login` first (the sidecar connects back with it)")? - .to_string(); - - // Connection details for the sidecar to reach this database, made reachable - // from inside a container (Docker Desktop maps host.docker.internal). - let db_hex = db_identity.to_hex().to_string(); - let mut connect_env = vec![ - format!("SPACETIMEDB_URI={}", to_container_host(&http_to_ws(&host))), - format!("SPACETIMEDB_HTTP_URI={}", to_container_host(&host)), - format!("SPACETIMEDB_TOKEN={token}"), - format!("SPACETIMEDB_DB={db_hex}"), - ]; - connect_env.append(&mut env); - - let api = control_api(&config, server, auth)?; - let arg_json = serde_json::json!([db_hex, name, image, command, connect_env]).to_string(); - let res = api.call("create_sidecar", arg_json).await?; - let status = res.status(); - let body = res.text().await.unwrap_or_default(); - if !status.is_success() { - bail!("failed to declare sidecar ({status}): {body}"); - } - - println!("Declared sidecar `{name}` for database `{database}`."); - println!("The node hosting the database will launch it shortly. Check with:"); - println!(" spacetime sidecar ls"); - Ok(()) -} - -async fn exec_stop(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { - let server = args.get_one::("server").map(|s| s.as_str()); - let force = args.get_flag("force"); - let anon = args.get_flag("anon_identity"); - let database = args.get_one::("database").unwrap(); - let name = args.get_one::("name").unwrap(); - - let db_identity = database_identity(&config, database, server).await?; - let auth = get_auth_header(&mut config, anon, server, !force).await?; - let db_hex = db_identity.to_hex().to_string(); - - let api = control_api(&config, server, auth)?; - let arg_json = serde_json::json!([db_hex, name]).to_string(); - let res = api.call("delete_sidecar", arg_json).await?; - let status = res.status(); - let body = res.text().await.unwrap_or_default(); - if !status.is_success() { - bail!("failed to remove sidecar ({status}): {body}"); - } - - println!("Removed sidecar `{name}` from database `{database}`. The hosting node will stop it shortly."); - Ok(()) -} - -async fn exec_ls(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { - let server = args.get_one::("server").map(|s| s.as_str()); - // Reading the control db requires a token; reuse the logged-in identity. - let auth = get_auth_header(&mut config, false, server, true).await?; - let api = control_api(&config, server, auth)?; - let res = api - .sql() - .body("SELECT id, database_id, name, image, desired_state FROM sidecar") - .send() - .await?; - let status = res.status(); - let body = res.text().await.unwrap_or_default(); - if !status.is_success() { - bail!("failed to list sidecars ({status}): {body}"); - } - println!("{body}"); - Ok(()) -} - -fn http_to_ws(url: &str) -> String { - if let Some(rest) = url.strip_prefix("https://") { - format!("wss://{rest}") - } else if let Some(rest) = url.strip_prefix("http://") { - format!("ws://{rest}") - } else { - url.to_string() - } -} - -fn to_container_host(url: &str) -> String { - url.replace("localhost", "host.docker.internal") - .replace("127.0.0.1", "host.docker.internal") -} diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index 6586d8edd5f..e7a32bc5bdb 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -7,12 +7,12 @@ use reqwest::Url; use serde_json::Value; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2, v3 as ws_v3}; use spacetimedb_data_structures::map::HashMap; -use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::{DeserializeWrapper, SeedWrapper}; use spacetimedb_lib::de::DeserializeSeed as BsatnDeserializeSeed; use spacetimedb_lib::sats::WithTypespace; use spacetimedb_lib::ser::serde::SerializeWrapper; use spacetimedb_lib::{bsatn, AlgebraicType}; +use spacetimedb_schema::def::ModuleDef; use std::collections::VecDeque; use std::io; use std::time::Duration; @@ -115,7 +115,7 @@ impl SubscribeConnection { } /// Wait for the initial subscription result and optionally print it. - async fn await_initial_update(&mut self, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> { + async fn await_initial_update(&mut self, module_def: Option<&ModuleDef>) -> Result<(), Error> { match self { Self::V3 { ws, pending } => await_initial_update_v3(ws, pending, module_def).await, Self::V1 { ws } => await_initial_update_v1(ws, module_def).await, @@ -123,11 +123,7 @@ impl SubscribeConnection { } /// Print transaction updates until the requested count is reached. - async fn consume_transaction_updates( - &mut self, - num: Option, - module_def: &RawModuleDefV9, - ) -> Result<(), Error> { + async fn consume_transaction_updates(&mut self, num: Option, module_def: &ModuleDef) -> Result<(), Error> { match self { Self::V3 { ws, pending } => consume_transaction_updates_v3(ws, pending, num, module_def).await, Self::V1 { ws } => consume_transaction_updates_v1(ws, num, module_def).await, @@ -179,7 +175,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error database: resolved.database.clone(), }; let api = ClientApi::new(conn); - let module_def = api.module_def().await?; + let module_def: ModuleDef = api.module_def().await?.try_into()?; let mut conn = connect_with_fallback(&api, confirmed).await?; let task = async { @@ -398,7 +394,7 @@ fn parse_msg_json(msg: &WsMessage) -> Option(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> +async fn await_initial_update_v1(ws: &mut S, module_def: Option<&ModuleDef>) -> Result<(), Error> where S: TryStream + Unpin, { @@ -442,7 +438,7 @@ where async fn await_initial_update_v3( ws: &mut S, pending: &mut VecDeque, - module_def: Option<&RawModuleDefV9>, + module_def: Option<&ModuleDef>, ) -> Result<(), Error> where S: TryStream + Unpin, @@ -475,11 +471,7 @@ where /// Print `num` v1 [`ws_v1::ServerMessage::TransactionUpdate`] messages as JSON. /// If `num` is `None`, keep going indefinitely. -async fn consume_transaction_updates_v1( - ws: &mut S, - num: Option, - module_def: &RawModuleDefV9, -) -> Result<(), Error> +async fn consume_transaction_updates_v1(ws: &mut S, num: Option, module_def: &ModuleDef) -> Result<(), Error> where S: TryStream + Unpin, { @@ -526,7 +518,7 @@ async fn consume_transaction_updates_v3( ws: &mut S, pending: &mut VecDeque, num: Option, - module_def: &RawModuleDefV9, + module_def: &ModuleDef, ) -> Result<(), Error> where S: TryStream + Unpin, @@ -619,25 +611,19 @@ fn decode_server_payload(msg: Bytes, pending: &mut VecDeque, - schema: &RawModuleDefV9, -) -> Result { +fn format_output_json_v1(msg: &ws_v1::DatabaseUpdate, schema: &ModuleDef) -> Result { let formatted = reformat_update_v1(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } /// Format initial v3 subscription rows using the CLI's existing JSON output shape. -fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &RawModuleDefV9) -> Result { +fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &ModuleDef) -> Result { let formatted = reformat_query_rows(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } /// Format a v3 transaction update using the CLI's existing JSON output shape. -fn format_output_json_transaction_update( - msg: &ws_v2::TransactionUpdate, - schema: &RawModuleDefV9, -) -> Result { +fn format_output_json_transaction_update(msg: &ws_v2::TransactionUpdate, schema: &ModuleDef) -> Result { let formatted = reformat_transaction_update(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } @@ -651,12 +637,12 @@ fn format_output_json_from_tables(formatted: &HashMap<&str, SubscriptionTable>) /// Convert a v1 JSON-format database update to the normalized table output map. fn reformat_update_v1<'a>( msg: &'a ws_v1::DatabaseUpdate, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { msg.tables .iter() .map(|upd| { - let table_ty = schema.typespace.resolve( + let table_ty = schema.typespace().resolve( schema .type_ref_for_table_like(&upd.table_name) .context("table not found in schema")?, @@ -690,12 +676,12 @@ fn reformat_update_v1<'a>( /// Convert v3 initial subscription rows to the normalized table output map. fn reformat_query_rows<'a>( msg: &'a ws_v2::QueryRows, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { let mut formatted = HashMap::default(); for table in &msg.tables { - let table_ty = schema.typespace.resolve( + let table_ty = schema.typespace().resolve( schema .type_ref_for_table_like(&table.table) .context("table not found in schema")?, @@ -713,13 +699,13 @@ fn reformat_query_rows<'a>( /// Convert a v3 transaction update to the normalized table output map. fn reformat_transaction_update<'a>( msg: &'a ws_v2::TransactionUpdate, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { let mut formatted = HashMap::default(); for query_set in &msg.query_sets { for table in &query_set.tables { - let table_ty = schema.typespace.resolve( + let table_ty = schema.typespace().resolve( schema .type_ref_for_table_like(&table.table_name) .context("table not found in schema")?, diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 5fd05be8320..d2202566a1a 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -198,11 +198,6 @@ impl AuthHeader { }) } - /// The raw bearer token, if any. Used by `spacetime sidecar` to pass the - /// caller's credentials into a sidecar container. - pub fn token(&self) -> Option<&str> { - self.token.as_deref() - } } pub const VALID_PROTOCOLS: [&str; 2] = ["http", "https"]; diff --git a/crates/client-api/src/auth.rs b/crates/client-api/src/auth.rs index 03c122510aa..493c91f0994 100644 --- a/crates/client-api/src/auth.rs +++ b/crates/client-api/src/auth.rs @@ -88,6 +88,7 @@ pub struct SpacetimeAuth { pub claims: SpacetimeIdentityClaims, /// The JWT payload as a json string (after base64 decoding). pub jwt_payload: Box, + pub hosted: Option, } impl SpacetimeAuth { @@ -100,6 +101,7 @@ impl SpacetimeAuth { creds, claims, jwt_payload: payload, + hosted: None, }) } } @@ -109,6 +111,7 @@ impl From for ConnectionAuthCtx { ConnectionAuthCtx { claims: auth.claims, jwt_payload: auth.jwt_payload.clone(), + hosted: auth.hosted, } } } @@ -214,6 +217,9 @@ impl SpacetimeAuth { signer: &impl TokenSigner, expiry: Duration, ) -> Result<(SpacetimeIdentityClaims, String), JwtError> { + if self.hosted.is_some() { + return Err(JwtErrorKind::InvalidToken.into()); + } TokenClaims::from(self.clone()).encode_and_sign_with_expiry(signer, Some(expiry)) } } @@ -401,13 +407,47 @@ pub struct SpacetimeAuthHeader { } #[async_trait::async_trait] -impl axum::extract::FromRequestParts for SpacetimeAuthHeader { +impl axum::extract::FromRequestParts for SpacetimeAuthHeader { type Rejection = AuthorizationRejection; async fn from_request_parts(parts: &mut request::Parts, state: &S) -> Result { let Some(creds) = SpacetimeCreds::from_request_parts(parts)? else { return Ok(Self { auth: None }); }; + if spacetimedb::auth::hosted_tokens::has_reserved_hosted_token_kind(&creds.token) + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))? + { + // Use Axum's matched route parameters, never a client header or a + // hand-parsed URL suffix. Tokens are ineligible for root publishing, + // identity allocation, and generic token exchange routes. + #[derive(Deserialize)] + struct HostedTargetPath { + name_or_identity: crate::util::NameOrIdentity, + } + let axum::extract::Path(params) = axum::extract::Path::::from_request_parts(parts, state) + .await + .map_err(|_| AuthorizationRejection::Required)?; + let target = params + .name_or_identity + .resolve(state) + .await + .map_err(|_| AuthorizationRejection::Required)?; + let verified = state + .authenticate_hosted_token(&creds.token, target) + .await + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))?; + let connection = verified + .into_connection_auth() + .map_err(|error| AuthorizationRejection::Custom(TokenValidationError::Other(error)))?; + let auth = SpacetimeAuth { + creds, + claims: connection.claims, + jwt_payload: connection.jwt_payload, + hosted: connection.hosted, + }; + return Ok(Self { auth: Some(auth) }); + } + let claims = validate_token(state, &creds.token) .await .map_err(AuthorizationRejection::Custom)?; @@ -419,6 +459,7 @@ impl axum::extract::FromRequestParts for Space creds, claims, jwt_payload: payload.into(), + hosted: None, }; Ok(Self { auth: Some(auth) }) } @@ -477,7 +518,9 @@ impl SpacetimeAuthHeader { pub struct SpacetimeAuthRequired(pub SpacetimeAuth); #[async_trait::async_trait] -impl axum::extract::FromRequestParts for SpacetimeAuthRequired { +impl axum::extract::FromRequestParts + for SpacetimeAuthRequired +{ type Rejection = AuthorizationRejection; async fn from_request_parts(parts: &mut request::Parts, state: &S) -> Result { let auth = SpacetimeAuthHeader::from_request_parts(parts, state).await?; diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 7ec13a670de..b80799e7c83 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -55,6 +55,15 @@ pub trait NodeDelegate: Send + Sync { type JwtAuthProviderT: auth::JwtAuthProvider; fn jwt_auth_provider(&self) -> &Self::JwtAuthProviderT; + /// Authenticate a platform-issued hosted credential for the exact resolved + /// database in this request. Editions without container hosting fail closed. + async fn authenticate_hosted_token( + &self, + _token: &str, + _target: Identity, + ) -> anyhow::Result { + anyhow::bail!("Hosted database credentials are not enabled on this server") + } /// Return the leader [`Host`] of `database_id`. /// /// The [`Host`] is spawned implicitly if not already running. @@ -136,7 +145,7 @@ impl Host { pub async fn exec_sql( &self, - auth: AuthCtx, + auth: impl Into, _database: Database, confirmed_read: bool, body: String, @@ -466,6 +475,14 @@ impl NodeDelegate for Arc { (**self).jwt_auth_provider() } + async fn authenticate_hosted_token( + &self, + token: &str, + target: Identity, + ) -> anyhow::Result { + (**self).authenticate_hosted_token(token, target).await + } + async fn leader(&self, database_id: u64) -> Result { (**self).leader(database_id).await } diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index 60576cf752b..c5ec4f9dc8d 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -42,6 +42,7 @@ use spacetimedb_client_api_messages::name::{ PrePublishResult, PrettyPrintStyle, PublishOp, PublishResult, }; use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; +use spacetimedb_lib::db::raw_def::v11::RawModuleDefV11; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::http as st_http; use spacetimedb_lib::{sats, AlgebraicValue, Hash, ProductValue, Timestamp}; @@ -158,16 +159,19 @@ pub async fn call( let (module, Database { owner_identity, .. }) = find_module_and_database(&worker_ctx, name_or_identity).await?; + let caller_auth: ConnectionAuthCtx = auth.into(); + let caller = spacetimedb::auth::invocation::InvocationCaller::from(&caller_auth); + // Call the database's `client_connected` reducer, if any. // If it fails or rejects the connection, bail. module - .call_identity_connected(auth.into(), connection_id) + .call_identity_connected(caller_auth, connection_id) .await .map_err(client_connected_error_to_response)?; let result = match module .call_reducer( - caller_identity, + caller.clone(), Some(connection_id), None, None, @@ -181,7 +185,7 @@ pub async fn call( Err(ReducerCallError::NoSuchReducer | ReducerCallError::ScheduleReducerNotFound) => { // Not a reducer — try procedure instead match module - .call_procedure(caller_identity, Some(connection_id), None, &reducer, args) + .call_procedure(caller, Some(connection_id), None, &reducer, args) .await .result { @@ -526,6 +530,8 @@ enum SchemaVersion { V9, #[serde(rename = "10")] V10, + #[serde(rename = "11")] + V11, } pub async fn schema( @@ -549,11 +555,17 @@ where let module_def = &module.info.module_def; let response_json = match version { SchemaVersion::V9 => { - let raw = RawModuleDefV9::from(module_def.as_ref().clone()); + let raw = RawModuleDefV9::try_from(module_def.as_ref().clone()) + .map_err(|err| bad_request(err.to_string().into()))?; axum::Json(sats::serde::SerdeWrapper(raw)).into_response() } SchemaVersion::V10 => { - let raw = RawModuleDefV10::from(module_def.as_ref().clone()); + let raw = RawModuleDefV10::try_from(module_def.as_ref().clone()) + .map_err(|err| bad_request(err.to_string().into()))?; + axum::Json(sats::serde::SerdeWrapper(raw)).into_response() + } + SchemaVersion::V11 => { + let raw = RawModuleDefV11::from(module_def.as_ref().clone()); axum::Json(sats::serde::SerdeWrapper(raw)).into_response() } }; @@ -728,7 +740,7 @@ where // If it rejects the connection, bail before executing SQL. let module = host.module().await.map_err(log_and_500)?; module - .call_identity_connected(caller_auth, connection_id) + .call_identity_connected(caller_auth.clone(), connection_id) .await .map_err(client_connected_error_to_response)?; @@ -737,6 +749,8 @@ where .authorize_sql(caller_identity, database.database_identity) .await?; + let sql_auth = + spacetimedb::auth::invocation::SqlCallAuth::authenticated(sql_auth, &caller_auth).map_err(log_and_500)?; host.exec_sql( sql_auth, database, diff --git a/crates/codegen/src/util.rs b/crates/codegen/src/util.rs index a13a7ff2aba..ac2c40a4d96 100644 --- a/crates/codegen/src/util.rs +++ b/crates/codegen/src/util.rs @@ -10,8 +10,8 @@ use convert_case::{Case, Casing}; use itertools::Itertools; use spacetimedb_lib::db::raw_def::v9::TableAccess; use spacetimedb_lib::sats::layout::PrimitiveType; +use spacetimedb_lib::sats::AlgebraicTypeRef; use spacetimedb_lib::version; -use spacetimedb_lib::{db::raw_def::v9::Lifecycle, sats::AlgebraicTypeRef}; use spacetimedb_primitives::ColList; use spacetimedb_schema::{def::ViewDef, type_for_generate::ProductTypeDef}; use spacetimedb_schema::{ @@ -98,31 +98,20 @@ pub(super) fn is_reducer_invokable(reducer: &ReducerDef) -> bool { reducer.lifecycle.is_none() } -/// Iterate over all the [`ReducerDef`]s defined by the module, in alphabetical order by name. -/// -/// Skipping the `init` reducer and internal [`FunctionVisibiity::Internal`] reducers because -/// they should not be directly invokable. -/// Sorting is not necessary for reducers because they are already stored in an IndexMap. +/// Non-lifecycle reducer entry points in declaration order. Default clients see +/// only public functions; IncludePrivate adds Private and Internal methods. pub(super) fn iter_reducers(module: &ModuleDef, visibility: CodegenVisibility) -> impl Iterator { module .reducers() - // `RawModuleDefV10` already marks all lifecycle reducers as private, but we keep - // this filter for backward compatibility with older versions where `init` - // reducers were not private. - .filter(|reducer| reducer.lifecycle != Some(Lifecycle::Init)) - // Prior to `RawModuleDefV10`, all reducers were public by default. Filtering out - // internal reducers here does not break SDKs built against older versions. + .filter(|reducer| reducer.lifecycle.is_none()) .filter(move |reducer| match visibility { CodegenVisibility::IncludePrivate => true, - CodegenVisibility::OnlyPublic => !reducer.visibility.is_private(), + CodegenVisibility::OnlyPublic => reducer.visibility.is_client_callable(), }) } -/// Iterate over all the [`ProcedureDef`]s defined by the module, in alphabetical order by name. -/// -/// Skipping internal [`FunctionVisibiity::Internal`] procedures because they should not be -/// directly invokable. -/// Sorting is necessary to have deterministic reproducible codegen. +/// Procedure entry points in alphabetical order. Default clients see only Public +/// functions; IncludePrivate also generates Private and Internal methods. pub(super) fn iter_procedures( module: &ModuleDef, visibility: CodegenVisibility, @@ -132,7 +121,7 @@ pub(super) fn iter_procedures( .sorted_by_key(|procedure| &procedure.name) .filter(move |procedure| match visibility { CodegenVisibility::IncludePrivate => true, - CodegenVisibility::OnlyPublic => !procedure.visibility.is_private(), + CodegenVisibility::OnlyPublic => procedure.visibility.is_client_callable(), }) } @@ -222,3 +211,64 @@ pub(super) fn iter_constraints(table: &TableDef) -> impl Iterator impl Iterator { module.types().sorted_by_key(|table| &table.accessor_name) } + +#[cfg(test)] +mod visibility_tests { + use super::*; + use spacetimedb_lib::db::raw_def::{ + v11::{FunctionVisibility, RawModuleDefV11Builder}, + v9::Lifecycle, + }; + use spacetimedb_lib::{AlgebraicType, ProductType}; + + #[test] + fn public_codegen_excludes_internal_private_and_every_lifecycle() { + let mut builder = RawModuleDefV11Builder::new(); + builder.add_reducer("ordinary", ProductType::unit()); + for (name, visibility) in [ + ("public_function", FunctionVisibility::ClientCallable), + ("private_function", FunctionVisibility::Private), + ("internal_function", FunctionVisibility::Internal), + ] { + builder.add_reducer_with_visibility(name, ProductType::unit(), Some(visibility)); + builder.add_procedure_with_visibility( + format!("{name}_procedure"), + ProductType::unit(), + AlgebraicType::unit(), + Some(visibility), + ); + } + for (name, lifecycle) in [ + ("init", Lifecycle::Init), + ("connect", Lifecycle::OnConnect), + ("disconnect", Lifecycle::OnDisconnect), + ] { + builder.add_lifecycle_reducer(lifecycle, name, ProductType::unit()); + } + let module: ModuleDef = builder.finish().try_into().unwrap(); + let names = |visibility| { + iter_reducers(&module, visibility) + .map(|r| &r.name[..]) + .collect::>() + }; + assert_eq!(names(CodegenVisibility::OnlyPublic), ["ordinary", "public_function"]); + assert_eq!( + names(CodegenVisibility::IncludePrivate), + ["ordinary", "public_function", "private_function", "internal_function"] + ); + let names = |visibility| { + iter_procedures(&module, visibility) + .map(|p| &p.name[..]) + .collect::>() + }; + assert_eq!(names(CodegenVisibility::OnlyPublic), ["public_function_procedure"]); + assert_eq!( + names(CodegenVisibility::IncludePrivate), + [ + "internal_function_procedure", + "private_function_procedure", + "public_function_procedure" + ] + ); + } +} diff --git a/crates/core/src/auth/hosted_tokens.rs b/crates/core/src/auth/hosted_tokens.rs new file mode 100644 index 00000000000..f1e6c356460 --- /dev/null +++ b/crates/core/src/auth/hosted_tokens.rs @@ -0,0 +1,320 @@ +//! Explicit platform trust for hosted database credentials. No OIDC discovery or fallback. + +use anyhow::{ensure, Context}; +use jsonwebtoken::DecodingKey; +pub use spacetimedb_auth::hosted::{ + has_reserved_hosted_token_kind, sign_hosted_token, HostedTokenBinding, HostedTokenClaims, VerifiedHostedAuth, HOSTED_TOKEN_KIND, HOSTED_TOKEN_TYPE, + MAX_HOSTED_TOKEN_LIFETIME, +}; +use spacetimedb_auth::hosted::{unverified_hosted_token_claims, verify_hosted_token}; +use spacetimedb_lib::Identity; +use std::collections::HashMap; +use std::time::SystemTime; + +/// Only configured platform signers can attest registered source databases. +pub struct HostedTokenValidator { + trusted_issuers: HashMap, DecodingKey>, +} + +impl HostedTokenValidator { + pub fn new(issuers: impl IntoIterator, DecodingKey)>) -> anyhow::Result { + let mut trusted_issuers = HashMap::new(); + for (issuer, key) in issuers { + ensure!( + !issuer.is_empty() && issuer.len() <= 128, + "invalid trusted hosted issuer" + ); + ensure!( + trusted_issuers.insert(issuer, key).is_none(), + "duplicate trusted hosted issuer" + ); + } + Ok(Self { trusted_issuers }) + } + + /// `resolve_binding` reads authoritative state, including this issuer's source + /// registration, open admission, current placement/incarnation and target grant. + /// Return None if any requirement is absent. Its inputs are untrusted routing hints; + /// the callback must never copy them into a fabricated binding or mutate state. + /// The returned proof still requires target-fence checks at every later admission. + pub fn validate_token( + &self, + token: &str, + target: Identity, + now: SystemTime, + resolve_binding: impl FnOnce(&str, Identity, Identity) -> Option, + ) -> anyhow::Result { + let hints = unverified_hosted_token_claims(token)?; + ensure!(hints.target_database == target, "hosted credential target mismatch"); + let key = self + .trusted_issuers + .get(&hints.issuer) + .context("untrusted hosted credential issuer")?; + let binding = resolve_binding(&hints.issuer, hints.source_database, target) + .context("hosted source registration, instance, or target grant is unavailable")?; + ensure!( + binding.target_database == target, + "authoritative hosted binding target mismatch" + ); + verify_hosted_token(token, key, &hints.issuer, &binding, now) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::{ + token_validation::{FullTokenValidator, TokenValidator, UnimplementedTokenValidator}, + JwtKeys, + }; + use jsonwebtoken::{Algorithm, Header}; + use serde_json::{json, Value}; + use spacetimedb_auth::hosted::{has_reserved_hosted_token_kind, MAX_HOSTED_TOKEN_BYTES}; + use std::time::{Duration, UNIX_EPOCH}; + + fn fixture() -> (JwtKeys, HostedTokenValidator, HostedTokenBinding, SystemTime) { + let keys = JwtKeys::generate().unwrap(); + let validator = HostedTokenValidator::new([("platform.test".into(), keys.public.clone())]).unwrap(); + let now = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let binding = HostedTokenBinding { + source_database: Identity::from_claims("source", "database"), + target_database: Identity::from_claims("target", "database"), + generation: 9_007_199_254_740_993, + grant_revision: 9_007_199_254_740_995, + lease_expires_at: now + Duration::from_secs(30), + }; + (keys, validator, binding, now) + } + + fn mint(keys: &JwtKeys, binding: &HostedTokenBinding, now: SystemTime) -> String { + sign_hosted_token( + &keys.private, + "platform.test", + binding, + now, + now + Duration::from_secs(20), + "token-private-id", + ) + .unwrap() + } + + fn change(token: &str, keys: &JwtKeys, mutate: impl FnOnce(&mut Value, &mut Header)) -> String { + let mut claims = serde_json::to_value(unverified_hosted_token_claims(token).unwrap()).unwrap(); + let mut header = Header::new(Algorithm::ES256); + header.typ = Some(HOSTED_TOKEN_TYPE.into()); + mutate(&mut claims, &mut header); + jsonwebtoken::encode(&header, &claims, &keys.private).unwrap() + } + + #[test] + fn hosted_sender_target_authority_and_claims_are_preserved() { + let (keys, validator, mut binding, now) = fixture(); + for self_call in [false, true] { + if self_call { + binding.target_database = binding.source_database; + } + let token = mint(&keys, &binding, now); + assert!(has_reserved_hosted_token_kind(&token).unwrap()); + let verified = validator + .validate_token(&token, binding.target_database, now, |issuer, source, target| { + assert_eq!(issuer, "platform.test"); + assert_eq!(source, binding.source_database); + assert_eq!(target, binding.target_database); + Some(binding) + }) + .unwrap(); + assert_eq!(verified.is_internal(), self_call); + assert_eq!(verified.generation(), binding.generation); + assert_eq!(verified.grant_revision(), binding.grant_revision); + assert!(verified.check_at(now + Duration::from_secs(20)).is_err()); + let ctx = verified.into_connection_auth().unwrap(); + assert_eq!(ctx.claims.identity, binding.source_database); + assert_ne!( + ctx.claims.identity, + Identity::from_claims(&ctx.claims.issuer, &ctx.claims.subject) + ); + assert!(ctx.hosted.is_some()); + let payload: Value = serde_json::from_str(&ctx.jwt_payload).unwrap(); + assert_eq!(payload["generation"].as_u64(), Some(binding.generation)); + assert_eq!(payload["grant_revision"].as_u64(), Some(binding.grant_revision)); + assert_eq!(payload["aud"], binding.target_database.to_hex().as_str()); + assert_eq!(payload["iss"], "platform.test"); + let debug = format!("{ctx:?}"); + assert!(!debug.contains("token-private-id")); + assert!(!debug.contains(&token)); + assert!(!debug.contains("jwt_payload")); + } + } + + #[test] + fn hosted_validation_rejects_wrong_authority_binding_and_wire_shape() { + let (keys, validator, binding, now) = fixture(); + let token = mint(&keys, &binding, now); + let other_keys = JwtKeys::generate().unwrap(); + assert!(validator + .validate_token( + &mint(&other_keys, &binding, now), + binding.target_database, + now, + |_, _, _| Some(binding) + ) + .is_err()); + assert!(validator + .validate_token(&token, binding.source_database, now, |_, _, _| Some(binding)) + .is_err()); + assert!(validator + .validate_token(&token, binding.target_database, now, |_, _, _| None) + .is_err()); + let invalid_fields = [ + ("kind", json!("spacetimedb_hosted_v2")), + ("iss", json!("unknown.test")), + ("source_database", json!(binding.target_database.to_hex().as_str())), + ("sub", json!("other")), + ("aud", json!(binding.source_database.to_hex().as_str())), + ("aud", json!([binding.target_database.to_hex().as_str()])), + ("generation", json!(binding.generation - 1)), + ("grant_revision", json!(binding.grant_revision - 1)), + ("iat", json!(1_700_000_001_u64)), + ("exp", json!(1_700_000_000_u64)), + ("exp", json!(1_700_000_031_u64)), + ("exp", json!(u64::MAX)), + ("jti", json!("")), + ("hex_identity", json!(binding.source_database.to_hex().as_str())), + ]; + for (field, value) in invalid_fields { + let changed = change(&token, &keys, |claims, _| claims[field] = value); + assert!( + validator + .validate_token(&changed, binding.target_database, now, |_, _, _| Some(binding)) + .is_err(), + "accepted changed {field}" + ); + } + let wrong_type = change(&token, &keys, |_, header| header.typ = Some("JWT".into())); + assert!(validator + .validate_token(&wrong_type, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let missing_exp = change(&token, &keys, |claims, _| { + claims.as_object_mut().unwrap().remove("exp"); + }); + assert!(validator + .validate_token(&missing_exp, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let overflowing_time = change(&token, &keys, |claims, _| { + claims["iat"] = json!(u64::MAX - 20); + claims["exp"] = json!(u64::MAX); + }); + assert!(validator + .validate_token(&overflowing_time, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + let short_lease = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(19), + ..binding + }; + assert!(validator + .validate_token(&token, binding.target_database, now, |_, _, _| Some(short_lease)) + .is_err()); + assert!(validator + .validate_token( + &"x".repeat(MAX_HOSTED_TOKEN_BYTES + 1), + binding.target_database, + now, + |_, _, _| Some(binding) + ) + .is_err()); + let claims = unverified_hosted_token_claims(&token).unwrap(); + let mut hs_header = Header::new(Algorithm::HS256); + hs_header.typ = Some(HOSTED_TOKEN_TYPE.into()); + let hs_token = jsonwebtoken::encode( + &hs_header, + &claims, + &jsonwebtoken::EncodingKey::from_secret(b"not-a-platform-key"), + ) + .unwrap(); + assert!(validator + .validate_token(&hs_token, binding.target_database, now, |_, _, _| Some(binding)) + .is_err()); + } + + #[test] + fn broker_signing_obeys_confirmed_lease_and_lifetime() { + let (keys, _, binding, now) = fixture(); + for expiry in [now, now + Duration::from_secs(31)] { + assert!(sign_hosted_token(&keys.private, "platform.test", &binding, now, expiry, "id").is_err()); + } + let short_lease = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(10), + ..binding + }; + assert!(sign_hosted_token( + &keys.private, + "platform.test", + &short_lease, + now, + now + Duration::from_secs(11), + "id" + ) + .is_err()); + } + + #[tokio::test] + async fn ordinary_validation_rejects_reserved_hosted_kinds_and_types() { + let (keys, _, binding, _) = fixture(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + lease_expires_at: now + Duration::from_secs(30), + ..binding + }; + let token = mint(&keys, &binding, now); + let ordinary = FullTokenValidator { + local_key: keys.public.clone(), + local_issuer: "platform.test".into(), + oidc_validator: UnimplementedTokenValidator, + }; + for reserved in [ + token.clone(), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_hosted_future"); + header.typ = Some("JWT".into()); + }), + change(&token, &keys, |claims, header| { + claims.as_object_mut().unwrap().remove("kind"); + header.typ = Some("spacetimedb-hosted-v2+jwt".into()); + }), + ] { + assert!(has_reserved_hosted_token_kind(&reserved).unwrap()); + assert!(keys.public.validate_token(&reserved).await.is_err()); + assert!(ordinary.validate_token(&reserved).await.is_err()); + } + } + + #[tokio::test] + async fn reserved_classification_preserves_ordinary_token_algorithms() { + let rsa = openssl::rsa::Rsa::generate(2048).unwrap(); + let rsa = openssl::pkey::PKey::from_rsa(rsa).unwrap(); + let ec = JwtKeys::generate().unwrap(); + let keys = [ + (Algorithm::ES256, ec.private, ec.public), + ( + Algorithm::RS256, + jsonwebtoken::EncodingKey::from_rsa_pem(&rsa.private_key_to_pem_pkcs8().unwrap()).unwrap(), + DecodingKey::from_rsa_pem(&rsa.public_key_to_pem().unwrap()).unwrap(), + ), + ( + Algorithm::HS256, + jsonwebtoken::EncodingKey::from_secret(b"ordinary-oidc-test-secret"), + DecodingKey::from_secret(b"ordinary-oidc-test-secret"), + ), + ]; + for (algorithm, private, public) in keys { + let claims = json!({ "iss": "ordinary.test", "sub": "a-user", "iat": 1_700_000_000_u64, "kind": "ordinary_application_kind" }); + let token = jsonwebtoken::encode(&Header::new(algorithm), &claims, &private).unwrap(); + assert!( + !has_reserved_hosted_token_kind(&token).unwrap(), + "misclassified {algorithm:?}" + ); + let validated = public.validate_token(&token).await.unwrap(); + assert_eq!(validated.identity, Identity::from_claims("ordinary.test", "a-user")); + } + } +} diff --git a/crates/core/src/auth/invocation.rs b/crates/core/src/auth/invocation.rs new file mode 100644 index 00000000000..3c5fdf6e82f --- /dev/null +++ b/crates/core/src/auth/invocation.rs @@ -0,0 +1,123 @@ +//! Authority carried from authenticated admission to module execution. +//! +//! Identity equality never establishes internal authority. A hosted proof can +//! only originate in signature verification against trusted platform state. + +use super::hosted_tokens::VerifiedHostedAuth; +use spacetimedb_auth::identity::ConnectionAuthCtx; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_lib::Identity; +use spacetimedb_schema::def::ModuleDef; +use std::time::SystemTime; + +#[derive(Clone, Debug)] +pub struct InvocationCaller { + pub(crate) identity: Identity, + pub(crate) hosted: Option>, +} + +impl From for InvocationCaller { + fn from(identity: Identity) -> Self { + Self { identity, hosted: None } + } +} + +impl From<&ConnectionAuthCtx> for InvocationCaller { + fn from(auth: &ConnectionAuthCtx) -> Self { + Self { + identity: auth.claims.identity, + hosted: auth.hosted.clone().map(std::sync::Arc::new), + } + } +} + +/// SQL permissions and the authenticated container restrictions are independent. +/// Internal status never impersonates the database owner or grants SQL rights. +#[derive(Clone)] +pub struct SqlCallAuth { + permissions: spacetimedb_lib::identity::AuthCtx, + pub(crate) hosted: Option>, +} + +impl From for SqlCallAuth { + fn from(permissions: spacetimedb_lib::identity::AuthCtx) -> Self { + Self { + permissions, + hosted: None, + } + } +} + +impl std::ops::Deref for SqlCallAuth { + type Target = spacetimedb_lib::identity::AuthCtx; + fn deref(&self) -> &Self::Target { + &self.permissions + } +} + +impl SqlCallAuth { + pub fn authenticated( + permissions: spacetimedb_lib::identity::AuthCtx, + auth: &ConnectionAuthCtx, + ) -> anyhow::Result { + anyhow::ensure!( + permissions.caller() == auth.claims.identity, + "SQL caller does not match authenticated sender" + ); + if let Some(proof) = &auth.hosted { + anyhow::ensure!( + proof.source_database() == auth.claims.identity, + "SQL hosted proof does not match authenticated sender" + ); + } + Ok(Self { + permissions, + hosted: auth.hosted.clone().map(std::sync::Arc::new), + }) + } +} + +impl InvocationCaller { + pub(crate) fn flags_for(&self, target: Identity, module: &ModuleDef) -> anyhow::Result { + let Some(proof) = &self.hosted else { return Ok(0) }; + anyhow::ensure!( + proof.source_database() == self.identity, + "hosted caller does not match its proof" + ); + anyhow::ensure!( + proof.target_database() == target, + "hosted credential targets another database" + ); + anyhow::ensure!( + module.supports_hosted_auth_v1(), + "module does not support hosted authentication" + ); + proof.check_at(SystemTime::now())?; + Ok(u32::from(proof.is_internal())) + } +} + +/// Must run while holding the transaction that admits the database operation. +/// A check before queueing does not serialize with generation revocation. +pub(crate) fn check_hosted_admission( + state: &S, + target: Identity, + proof: Option<&VerifiedHostedAuth>, +) -> anyhow::Result<()> { + let Some(proof) = proof else { return Ok(()) }; + anyhow::ensure!( + proof.target_database() == target, + "hosted credential targets another database" + ); + proof.check_at(SystemTime::now())?; + crate::db::deployment::check_container_fence( + state, + proof.source_database(), + proof.generation(), + proof.grant_revision(), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/auth/invocation/tests.rs b/crates/core/src/auth/invocation/tests.rs new file mode 100644 index 00000000000..9453da7d03c --- /dev/null +++ b/crates/core/src/auth/invocation/tests.rs @@ -0,0 +1,156 @@ +use super::*; +use crate::auth::hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}; +use crate::auth::JwtKeys; +use crate::db::deployment::install_container_fence; +use crate::db::relational_db::tests_utils::TestDB; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::StContainerFenceRow; +use spacetimedb_lib::db::auth::StAccess; +use spacetimedb_lib::db::raw_def::v11::RawModuleDefV11Builder; +use spacetimedb_lib::identity::AuthCtx; +use std::time::Duration; + +fn module(hosted_auth: bool) -> ModuleDef { + let mut builder = RawModuleDefV11Builder::new(); + if hosted_auth { + builder.add_capability("hosted_auth_v1"); + } + builder.finish().try_into().unwrap() +} + +/// Obtain every proof through the production signer and target-bound verifier. +fn authenticate(source: Identity, target: Identity, now: SystemTime) -> ConnectionAuthCtx { + let keys = JwtKeys::generate().unwrap(); + let binding = HostedTokenBinding { + source_database: source, + target_database: target, + generation: 3, + grant_revision: 7, + lease_expires_at: now + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "test.platform", + &binding, + now, + now + Duration::from_secs(20), + "invocation-test", + ) + .unwrap(); + HostedTokenValidator::new([("test.platform".into(), keys.public)]) + .unwrap() + .validate_token(&token, target, now, |issuer, requested_source, requested_target| { + (issuer == "test.platform" && requested_source == source && requested_target == target).then_some(binding) + }) + .unwrap() + .into_connection_auth() + .unwrap() +} + +fn fence(source: Identity, generation: u64, grant_revision: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: source.into(), + generation, + target_grant_revision: grant_revision, + target_set_hash: spacetimedb_lib::hash_bytes(b"configured targets"), + allowed, + } +} + +#[test] +fn internal_requires_verified_self_call_and_updated_bindings() { + let target = Identity::ONE; + let foreign = Identity::from_u256(2u64.into()); + let updated = module(true); + let old_bindings = module(false); + // This also covers an ordinary connection whose sender equals the database. + assert_eq!(InvocationCaller::from(target).flags_for(target, &updated).unwrap(), 0); + for (source, expected_flags) in [(target, 1), (foreign, 0)] { + let auth = authenticate(source, target, SystemTime::now()); + let caller = InvocationCaller::from(&auth); + assert_eq!(caller.flags_for(target, &updated).unwrap(), expected_flags); + assert!(caller.flags_for(target, &old_bindings).is_err()); + assert!(caller.flags_for(Identity::ZERO, &updated).is_err()); + } +} + +#[test] +fn authenticated_proof_cannot_be_paired_with_another_sender_or_sql_caller() { + let source = Identity::ONE; + let target = Identity::from_u256(2u64.into()); + let owner = Identity::from_u256(3u64.into()); + let mut auth = authenticate(source, target, SystemTime::now()); + assert!(SqlCallAuth::authenticated(AuthCtx::for_current(owner), &auth).is_err()); + // ConnectionAuthCtx has public fields for trusted host code. Defend the + // boundary against accidentally mixing separately authenticated contexts. + auth.claims.identity = owner; + assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); + assert!(SqlCallAuth::authenticated(AuthCtx::for_current(owner), &auth).is_err()); +} + +#[test] +fn internal_authentication_does_not_grant_owner_sql_permissions() { + let source = Identity::ONE; + let owner = Identity::from_u256(3u64.into()); + let auth = authenticate(source, source, SystemTime::now()); + assert_eq!( + InvocationCaller::from(&auth).flags_for(source, &module(true)).unwrap(), + 1 + ); + let sql = SqlCallAuth::authenticated(AuthCtx::new(owner, source), &auth).unwrap(); + assert_eq!(sql.caller(), source); + assert!(sql.has_read_access(StAccess::Public)); + assert!(!sql.has_read_access(StAccess::Private)); + assert!(!sql.has_write_access()); + assert!(!sql.bypass_rls()); +} + +#[test] +fn transaction_admission_rechecks_persisted_fences_after_initial_authentication() { + let db = TestDB::in_memory().unwrap(); + let target = db.database_identity(); + let source = Identity::ONE; + let auth = authenticate(source, target, SystemTime::now()); + let proof = auth.hosted.as_ref(); + let caller = InvocationCaller::from(&auth); + assert_eq!(caller.flags_for(target, &module(true)).unwrap(), 0); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + assert!(check_hosted_admission(tx, target, proof).is_err()); + install_container_fence(&db, tx, &fence(source, 3, 7, true))?; + check_hosted_admission(tx, target, proof)?; + assert!(check_hosted_admission(tx, Identity::from_u256(99u64.into()), proof).is_err()); + Ok(()) + }) + .unwrap(); + // Revocation commits after token verification and before the queued call. + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + install_container_fence(&db, tx, &fence(source, 4, 8, false))?; + Ok(()) + }) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + assert!(check_hosted_admission(tx, target, proof).is_err()); + check_hosted_admission(tx, target, None)?; + // Another generation does not reactivate a copied credential. + install_container_fence(&db, tx, &fence(source, 5, 9, true))?; + assert!(check_hosted_admission(tx, target, proof).is_err()); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn expired_verified_proof_is_rejected_at_both_call_and_transaction_admission() { + let db = TestDB::in_memory().unwrap(); + let target = db.database_identity(); + let source = Identity::ONE; + // Valid when received, expired before execution, without sleeps or forged proofs. + let auth = authenticate(source, target, SystemTime::now() - Duration::from_secs(60)); + assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + install_container_fence(&db, tx, &fence(source, 3, 7, true))?; + assert!(check_hosted_admission(tx, target, auth.hosted.as_ref()).is_err()); + Ok(()) + }) + .unwrap(); +} diff --git a/crates/core/src/auth/mod.rs b/crates/core/src/auth/mod.rs index e1e38a667c4..49f2686f062 100644 --- a/crates/core/src/auth/mod.rs +++ b/crates/core/src/auth/mod.rs @@ -7,6 +7,8 @@ use spacetimedb_paths::cli::{PrivKeyPath, PubKeyPath}; use crate::config::CertificateAuthority; pub use spacetimedb_auth::identity; +pub mod hosted_tokens; +pub mod invocation; pub mod token_validation; /// JWT verification and signing keys. diff --git a/crates/core/src/auth/token_validation.rs b/crates/core/src/auth/token_validation.rs index c38d732882d..77aca274baf 100644 --- a/crates/core/src/auth/token_validation.rs +++ b/crates/core/src/auth/token_validation.rs @@ -97,6 +97,7 @@ where T: TokenValidator + Send + Sync, { async fn validate_token(&self, token: &str) -> Result { + reject_reserved_hosted_credentials(token)?; let local_key_error = { let first_validator = BasicTokenValidator { public_key: self.local_key.clone(), @@ -145,6 +146,7 @@ lazy_static! { #[async_trait] impl TokenValidator for DecodingKey { async fn validate_token(&self, token: &str) -> Result { + reject_reserved_hosted_credentials(token)?; let mut validation = Validation::new(jsonwebtoken::Algorithm::ES256); validation.algorithms = vec![ jsonwebtoken::Algorithm::ES256, @@ -245,6 +247,7 @@ pub struct OidcTokenValidator; // Get the issuer out of a token without validating the signature. fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { + reject_reserved_hosted_credentials(token)?; let mut validation = Validation::new(jsonwebtoken::Algorithm::ES256); validation.set_required_spec_claims(&REQUIRED_CLAIMS); validation.validate_aud = false; @@ -254,6 +257,16 @@ fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { Ok(data.claims.issuer) } +fn reject_reserved_hosted_credentials(token: &str) -> Result<(), TokenValidationError> { + if spacetimedb_auth::hosted::has_reserved_hosted_token_kind(token)? { + return Err(anyhow::anyhow!( + "hosted credentials require dedicated target-bound validation and cannot be exchanged" + ) + .into()); + } + Ok(()) +} + #[async_trait] impl TokenValidator for OidcTokenValidator { async fn validate_token(&self, token: &str) -> Result { diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 8f19156d6bb..af45a3ea748 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -8,6 +8,7 @@ use std::task::{Context, Poll}; use std::time::{Instant, SystemTime}; use super::{message_handlers, ClientActorId, MessageHandleError, OutboundMessage}; +use crate::auth::{hosted_tokens::VerifiedHostedAuth, invocation::check_hosted_admission}; use crate::db::relational_db::RelationalDB; use crate::error::DBError; use crate::host::module_host::{ClientConnectedError, ProcedureResultTarget}; @@ -23,6 +24,7 @@ use log::warn; use prometheus::{Histogram, IntCounter, IntGauge}; use spacetimedb_auth::identity::{ConnectionAuthCtx, SpacetimeIdentityClaims}; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2}; +use spacetimedb_datastore::execution_context::Workload; use spacetimedb_durability::{DurableOffset, TxOffset}; use spacetimedb_lib::identity::{AuthCtx, RequestId}; use spacetimedb_lib::metrics::ExecutionMetrics; @@ -125,9 +127,29 @@ pub trait DurableOffsetSupply: Send { /// - `Ok(Some(DurableOffset))` otherwise /// fn durable_offset(&mut self) -> Result, NoSuchModule>; + + /// Recheck the authoritative target state, never a cached generation. + fn check_hosted_auth( + &mut self, + _proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + Box::pin(async { anyhow::bail!("hosted connection has no authoritative database state") }) + } } impl DurableOffsetSupply for watch::Receiver { + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + if self.has_changed().is_err() { + return Box::pin(async { Err(NoSuchModule.into()) }); + } + let module = self.borrow().clone(); + let mut db = module.relational_db().clone(); + db.check_hosted_auth(proof) + } + fn durable_offset(&mut self) -> Result, NoSuchModule> { let module = if self.has_changed().map_err(|_| NoSuchModule)? { self.borrow_and_update() @@ -140,6 +162,22 @@ impl DurableOffsetSupply for watch::Receiver { } impl DurableOffsetSupply for Arc { + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + let db = self.clone(); + let proof = proof.clone(); + Box::pin(async move { + tokio::task::spawn_blocking(move || { + db.with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, db.database_identity(), Some(&proof)) + }) + }) + .await? + }) + } + fn durable_offset(&mut self) -> Result, NoSuchModule> { Ok(self.durable_tx_offset()) } @@ -157,6 +195,7 @@ pub struct ClientConnectionReceiver { channel: MeteredReceiver, pending: Vec, offset_supply: Box, + hosted_sender: Option>, } impl ClientConnectionReceiver { @@ -172,6 +211,7 @@ impl ClientConnectionReceiver { channel, pending: Vec::new(), offset_supply: Box::new(offset_supply), + hosted_sender: None, } } @@ -214,6 +254,9 @@ impl ClientConnectionReceiver { /// These values are stored internally, so calling `recv_many` again will /// not lose data. pub async fn recv_many(&mut self, buf: &mut Vec, max: usize) -> usize { + if !self.hosted_connection_is_valid().await { + return 0; + } // If there are no pending updates and the input channel has been closed, // no more messages can be received from this receiver. if max == 0 || (self.pending.is_empty() && self.channel.recv_many(&mut self.pending, max).await == 0) { @@ -223,14 +266,14 @@ impl ClientConnectionReceiver { // If we don't have to wait for txns to be made durable, // drain the pending updates. if !self.confirmed_reads { - return self.drain_pending(buf, max); + return self.drain_pending(buf, max).await; } // If we do have to wait for txns to be made durable, // but the next client update doesn't have a tx offset, // there's no reason to wait - just send it. if !self.pending_update_has_offset() { - return self.drain_pending(buf, 1); + return self.drain_pending(buf, 1).await; } // Otherwise, grab the next offset that we should wait for. @@ -243,12 +286,12 @@ impl ClientConnectionReceiver { warn!("database went away while waiting for durable offset"); return 0; } - self.drain_pending(buf, n) + self.drain_pending(buf, n).await } // Database shut down or crashed. Err(NoSuchModule) => 0, // In-memory database. - Ok(None) => self.drain_pending(buf, max), + Ok(None) => self.drain_pending(buf, max).await, } } @@ -265,13 +308,40 @@ impl ClientConnectionReceiver { } /// Drain the pending [`ClientUpdate`]s, up to `max, into `buf`. - fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + async fn drain_pending(&mut self, buf: &mut Vec, max: usize) -> usize { + // A queued update may predate revocation, and a confirmed-read wait may + // outlast the credential. Check again immediately before delivery. + if !self.hosted_connection_is_valid().await { + return 0; + } let n = self.pending.len().min(max); buf.reserve(n); buf.extend(self.pending.drain(..n).map(|u| u.message)); n } + async fn hosted_connection_is_valid(&mut self) -> bool { + let Some(sender) = &self.hosted_sender else { return true }; + let valid = match sender.upgrade() { + Some(sender) => { + let valid = match &sender.auth.hosted { + Some(proof) if !sender.is_cancelled() => self.offset_supply.check_hosted_auth(proof).await.is_ok(), + _ => false, + }; + if !valid { + sender.cancel_hosted_connection(); + } + valid + } + None => false, + }; + if !valid { + self.pending.clear(); + self.close(); + } + valid + } + /// Does the next pending update have a tx offset? /// /// Assumes that [`Self::pending`] is not empty. @@ -354,6 +424,13 @@ pub enum ClientSendError { } impl ClientConnectionSender { + /// The fence installer awaits the returned task's completion before ack. + pub(crate) fn cancel_hosted_connection(&self) -> AbortHandle { + self.cancelled.store(true, Ordering::Release); + self.abort_handle.abort(); + self.abort_handle.clone() + } + pub fn dummy_with_channel( id: ClientActorId, config: ClientConfig, @@ -423,6 +500,16 @@ impl ClientConnectionSender { } fn send(&self, message: ClientUpdate) -> Result<(), ClientSendError> { + // Do not acquire a database transaction here: broadcasts can already + // hold one. Durable fencing is checked at admission and delivery. + if self + .auth + .hosted + .as_ref() + .is_some_and(|proof| proof.check_at(SystemTime::now()).is_err()) + { + self.cancel_hosted_connection(); + } if self.cancelled.load(Relaxed) { return Err(ClientSendError::Cancelled); } @@ -469,6 +556,66 @@ impl ClientConnectionSender { } } +/// Runs independently of the socket actor, so blocked writes and idle sockets +/// cannot keep credentials alive. Expiry uses a monotonic deadline captured once. +fn spawn_hosted_connection_watchdog( + sender: std::sync::Weak, + mut supply: impl DurableOffsetSupply + 'static, + subscriptions: Option, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let actor = sender.upgrade().map(|connection| connection.abort_handle.clone()); + async { + let Some(connection) = sender.upgrade() else { return }; + let Some(proof) = connection.auth.hosted.clone() else { + return; + }; + let deadline = + tokio::time::Instant::now() + proof.expires_at().duration_since(SystemTime::now()).unwrap_or_default(); + drop(connection); + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + if let Some(connection) = sender.upgrade() { connection.cancel_hosted_connection(); } + return; + } + _ = interval.tick() => {} + } + let Some(connection) = sender.upgrade() else { return }; + if connection.abort_handle.is_finished() || connection.is_cancelled() { + return; + } + let checked = tokio::select! { + biased; + _ = tokio::time::sleep_until(deadline) => { + connection.cancel_hosted_connection(); + return; + } + checked = supply.check_hosted_auth(&proof) => checked, + }; + if checked.is_err() { + connection.cancel_hosted_connection(); + return; + } + } + } + .await; + // Keep the registry entry until socket I/O has actually stopped. A + // concurrent target barrier must still find and await an aborted actor. + if let Some(actor) = actor { + while !actor.is_finished() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + if let Some(subscriptions) = subscriptions { + subscriptions.unregister_hosted_connection(&sender); + } + }) +} + #[derive(Clone)] #[non_exhaustive] pub struct ClientConnection { @@ -874,7 +1021,7 @@ impl ClientConnection { .abort_handle(); let metrics = ClientConnectionMetrics::new(database_identity, config.protocol); - let receiver = ClientConnectionReceiver::new( + let mut receiver = ClientConnectionReceiver::new( config.confirmed_reads, MeteredReceiver::with_gauge(sendrx, metrics.sendtx_queue_size.clone()), module_rx.clone(), @@ -889,6 +1036,18 @@ impl ClientConnection { cancelled: AtomicBool::new(false), metrics: Some(metrics), }); + if sender.auth.hosted.is_some() { + receiver.hosted_sender = Some(Arc::downgrade(&sender)); + if module.subscriptions().register_hosted_connection(&sender).is_err() { + sender.cancel_hosted_connection(); + } else { + spawn_hosted_connection_watchdog( + Arc::downgrade(&sender), + module_rx.clone(), + Some(module.subscriptions().clone()), + ); + } + } let this = Self { sender, replica_id, @@ -998,7 +1157,7 @@ impl ClientConnection { self.module() .call_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), caller, Some(request_id), @@ -1019,7 +1178,7 @@ impl ClientConnection { ) -> Result { self.module() .call_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(self.sender()), Some(request_id), @@ -1045,7 +1204,7 @@ impl ClientConnection { self.module() .enqueue_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), caller, Some(request_id), @@ -1066,7 +1225,7 @@ impl ClientConnection { ) -> Result<(), ReducerCallError> { self.module() .enqueue_reducer( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(self.sender()), Some(request_id), @@ -1086,7 +1245,7 @@ impl ClientConnection { ) -> Result<(), BroadcastError> { self.module() .enqueue_procedure( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(timer), procedure, @@ -1106,7 +1265,7 @@ impl ClientConnection { ) -> Result<(), BroadcastError> { self.module() .enqueue_procedure( - self.id.identity, + &self.sender.auth, Some(self.id.connection_id), Some(timer), procedure, @@ -1335,6 +1494,288 @@ mod tests { assert_matches!(futures::poll!(f), Poll::Pending); } + fn hosted_auth(db: &RelationalDB, lifetime: std::time::Duration) -> ConnectionAuthCtx { + use crate::auth::{ + hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, + JwtKeys, + }; + let keys = JwtKeys::generate().unwrap(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + source_database: db.database_identity(), + target_database: db.database_identity(), + generation: 1, + grant_revision: 1, + lease_expires_at: now + std::time::Duration::from_secs(30), + }; + let token = sign_hosted_token(&keys.private, "platform.test", &binding, now, now + lifetime, "test").unwrap(); + HostedTokenValidator::new([("platform.test".into(), keys.public)]) + .unwrap() + .validate_token(&token, db.database_identity(), now, |_, _, _| Some(binding)) + .unwrap() + .into_connection_auth() + .unwrap() + } + + fn set_fence(db: &RelationalDB, generation: u64, allowed: bool) { + use crate::db::deployment::install_container_fence; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + db, + tx, + &StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: spacetimedb_lib::hash_bytes(b"targets"), + allowed, + }, + ) + }) + .unwrap(); + } + + fn hosted_client( + db: &RelationalDB, + supply: impl DurableOffsetSupply + 'static, + confirmed_reads: bool, + lifetime: std::time::Duration, + ) -> ( + Arc, + ClientConnectionReceiver, + tokio::task::JoinHandle<()>, + ) { + let (mut sender, mut receiver) = ClientConnectionSender::dummy_with_channel( + ClientActorId::for_test(db.database_identity()), + ClientConfig { + confirmed_reads, + ..ClientConfig::for_test() + }, + supply, + ); + sender.auth = hosted_auth(db, lifetime); + let actor = tokio::spawn(std::future::pending()); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + receiver.hosted_sender = Some(Arc::downgrade(&sender)); + (sender, receiver, actor) + } + + struct HostedConfirmedSupply { + db: Arc, + durable: FakeDurableOffset, + durability_requested: Arc, + } + impl DurableOffsetSupply for HostedConfirmedSupply { + fn durable_offset(&mut self) -> Result, NoSuchModule> { + self.durability_requested.store(true, Ordering::Release); + self.durable.durable_offset() + } + fn check_hosted_auth( + &mut self, + proof: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + self.db.check_hosted_auth(proof) + } + } + + #[tokio::test] + async fn hosted_queued_delivery_rechecks_committed_fence_and_preserves_ordinary_identity() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, mut receiver, actor) = + hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + sender.send_message(None, empty_tx_update()).unwrap(); + set_fence(&db, 2, false); + assert_receiver_closed(receiver.recv()).await; + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + let (ordinary, mut ordinary_rx) = default_client(db.db.clone()); + ordinary.send_message(None, empty_tx_update()).unwrap(); + assert_received_update(ordinary_rx.recv()).await; + } + + #[tokio::test] + async fn hosted_confirmed_delivery_rechecks_fence_after_durability_wait() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let durable = FakeDurableOffset::new(); + let durability_requested = Arc::new(AtomicBool::new(false)); + let supply = HostedConfirmedSupply { + db: db.db.clone(), + durable: durable.clone(), + durability_requested: durability_requested.clone(), + }; + let (sender, mut receiver, actor) = hosted_client(&db, supply, true, std::time::Duration::from_secs(20)); + sender.send_message(Some(7), empty_tx_update()).unwrap(); + let mut receiving = Box::pin(receiver.recv()); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !durability_requested.load(Ordering::Acquire) { + assert_pending(&mut receiving).await; + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + set_fence(&db, 2, false); + durable.mark_durable_at(7); + assert_receiver_closed(receiving).await; + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_idle_connection_expires_without_outbound_traffic() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(2)); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), None); + tokio::time::timeout(std::time::Duration::from_secs(3), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_idle_connection_rechecks_durable_revocation() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), None); + set_fence(&db, 2, false); + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + + #[tokio::test] + async fn hosted_watchdog_ends_and_unregisters_after_socket_actor_finishes() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + subscriptions.register_hosted_connection(&sender).unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 1); + let watchdog = + spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), Some(subscriptions.clone())); + actor.abort(); + let _ = actor.await; + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 0); + // The registry releases the entry even while another owner retains sender. + assert!(!sender.is_cancelled()); + } + + #[tokio::test] + async fn hosted_registry_retains_cancelled_actor_until_delivery_cleanup_finishes() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (mut sender, _receiver) = default_client(db.db.clone()); + sender.auth = hosted_auth(&db, std::time::Duration::from_secs(20)); + let (release, blocked) = std::sync::mpsc::channel(); + let (started, ready) = oneshot::channel(); + // A started blocking task models cleanup that cannot complete merely + // because abort was requested. The barrier must retain its handle. + let actor = tokio::task::spawn_blocking(move || { + let _ = started.send(()); + let _ = blocked.recv(); + }); + ready.await.unwrap(); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + subscriptions.register_hosted_connection(&sender).unwrap(); + let watchdog = + spawn_hosted_connection_watchdog(Arc::downgrade(&sender), db.db.clone(), Some(subscriptions.clone())); + set_fence(&db, 2, false); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !sender.is_cancelled() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(subscriptions.hosted_connection_count(), 1); + let handles = db.with_read_only(Workload::ForTests, |tx| { + subscriptions.cancel_invalid_hosted_connections(tx) + }); + assert_eq!(handles.len(), 1); + assert!(!handles[0].is_finished()); + release.send(()).unwrap(); + actor.await.unwrap(); + watchdog.await.unwrap(); + assert!(handles[0].is_finished()); + assert_eq!(subscriptions.hosted_connection_count(), 0); + } + + #[tokio::test] + async fn hosted_target_barrier_cancels_connections_without_subscriptions_and_waits_for_actor() { + use crate::db::deployment::install_container_fence; + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + subscriptions.register_hosted_connection(&sender).unwrap(); + let handles = db + .with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + &db, + tx, + &StContainerFenceRow { + source_identity: db.database_identity().into(), + generation: 2, + target_grant_revision: 1, + target_set_hash: spacetimedb_lib::hash_bytes(b"targets"), + allowed: true, + }, + )?; + Ok::<_, anyhow::Error>(subscriptions.cancel_invalid_hosted_connections(tx)) + }) + .unwrap(); + assert_eq!(handles.len(), 1); + assert!(actor.await.unwrap_err().is_cancelled()); + assert!(handles[0].is_finished()); + assert!(subscriptions.register_hosted_connection(&sender).is_err()); + } + + #[tokio::test] + async fn hosted_subscription_rejected_under_transaction_before_query_compilation() { + use crate::subscription::module_subscription_actor::ModuleSubscriptions; + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.db.clone()); + let (sender, _receiver, actor) = hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + // There is deliberately no installed fence. The malformed SQL verifies + // authentication fails before query parsing or view materialization. + let result = subscriptions + .add_legacy_subscriber( + None, + sender.clone(), + AuthCtx::new(db.database_identity(), db.database_identity()), + ws_v1::Subscribe { + query_strings: ["invalid SQL".into()].into(), + request_id: 0, + }, + Instant::now(), + None, + ) + .await; + assert!(result.unwrap_err().to_string().contains("container")); + sender.cancel_hosted_connection(); + let _ = actor.await; + } + fn default_client( offset_supply: impl DurableOffsetSupply + 'static, ) -> (ClientConnectionSender, ClientConnectionReceiver) { diff --git a/crates/core/src/db/deployment.rs b/crates/core/src/db/deployment.rs new file mode 100644 index 00000000000..3c794194c93 --- /dev/null +++ b/crates/core/src/db/deployment.rs @@ -0,0 +1,347 @@ +//! Transactional deployment and hosted-client fences. +//! +//! Only authenticated host operations may call the mutation functions here. +//! Callers retain the same serializable transaction through module migration, +//! deployment recording, and commit. These functions do not contact control, +//! perform process IO, or turn an unverified client Identity into host authority. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + ConnectionIdViaU128, StConnectionAuthRow, StContainerFenceRow, StDeploymentOperationRow, StDeploymentRow, + StPublishFenceRow, ST_CONNECTION_AUTH_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, ST_DEPLOYMENT_OPERATION_ID, + ST_PUBLISH_FENCE_ID, +}; +use spacetimedb_lib::container::ContainerSpecLimits; +use spacetimedb_lib::deployment::{operation_expiry_ms, DeploymentSpec, DeploymentValidationError}; +use spacetimedb_lib::{bsatn, hash_bytes, ConnectionId, Hash, Identity, SpacetimeType, Timestamp, Uuid}; +use spacetimedb_primitives::{ColId, TableId}; +use spacetimedb_sats::AlgebraicValue; + +#[derive(Debug, thiserror::Error)] +pub enum DeploymentError { + #[error("the publication coordinator no longer owns the database fence")] + PublicationFenced, + #[error("the expected deployment revision does not match the database")] + RevisionConflict, + #[error("operation ID is already bound to a different publication")] + OperationConflict, + #[error("container generation or grant is not authorized by this database")] + ContainerFenced, + #[error("a conflicting or older container fence cannot replace current authority")] + FenceConflict, + #[error("deployment metadata is inconsistent")] + CorruptMetadata, + #[error(transparent)] + Validation(#[from] DeploymentValidationError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Prepared by the authorized coordinator after recording durable intent. +/// The publisher is its verified original caller, never a guest-selected claim. +#[derive(Clone, Debug)] +pub struct DeploymentCommit { + pub operation_id: Uuid, + pub publication_epoch: u64, + pub publisher: Identity, + pub expected_revision: Option, + pub prepared_manifest_hash: Hash, + pub deployment: DeploymentSpec, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishResult { + #[serde(with = "spacetimedb_lib::deployment::uuid_json")] + pub operation_id: Uuid, + pub previous_revision: Option, + pub revision: Hash, +} + +#[derive(Clone, Debug, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +enum CommitReceipt { + V1(CommitReceiptV1), +} + +#[derive(Clone, Debug, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct CommitReceiptV1 { + request_hash: Hash, + publisher: Identity, + result: PublishResult, +} + +#[derive(Clone, Debug)] +pub enum CommitAdmission { + /// Return this result without repeating module init/update or any effects. + AlreadyCommitted(PublishResult), + Ready, +} + +fn singleton( + state: &S, + table: TableId, +) -> Result>, DeploymentError> { + Ok(state.iter_by_col_eq(table, ColId(0), &AlgebraicValue::U8(0))?.next()) +} + +pub fn current_deployment(state: &S) -> Result, DeploymentError> { + let Some(row) = singleton(state, ST_DEPLOYMENT_ID)? else { + return Ok(None); + }; + let row = StDeploymentRow::try_from(row)?; + let spec = DeploymentSpec::decode(&row.payload)?; + if spec.revision()? != row.revision { + return Err(DeploymentError::CorruptMetadata); + } + Ok(Some((row.revision, spec))) +} + +/// Monotonic compare-and-set, serialized against user-database commits. +/// Advancing this epoch does not itself quiesce a container or authorize launch. +pub fn install_publication_fence( + tx: &mut MutTx, + publication_epoch: u64, + operation_id: Uuid, +) -> Result<(), DeploymentError> { + if publication_epoch == 0 { + return Err(DeploymentError::PublicationFenced); + } + let next = StPublishFenceRow { + key: 0, + publication_epoch, + operation_id: operation_id.as_u128(), + }; + if let Some(current) = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()? + { + if current == next { + return Ok(()); + } + if current.publication_epoch >= publication_epoch { + return Err(DeploymentError::PublicationFenced); + } + } + tx.clear_table(ST_PUBLISH_FENCE_ID)?; + tx.insert_via_serialize_bsatn(ST_PUBLISH_FENCE_ID, &next)?; + Ok(()) +} + +fn normalized_request( + request: &DeploymentCommit, + limits: &ContainerSpecLimits, +) -> Result<(DeploymentSpec, Hash, Hash), DeploymentError> { + let spec = request.deployment.clone().normalize(limits)?; + let revision = spec.revision()?; + let mut bytes = b"spacetimedb/deployment-operation\0".to_vec(); + bytes.extend_from_slice( + &bsatn::to_vec(&( + request.operation_id, + request.publication_epoch, + request.publisher, + request.expected_revision, + request.prepared_manifest_hash, + revision, + )) + .map_err(|_| DeploymentError::CorruptMetadata)?, + ); + Ok((spec, revision, hash_bytes(bytes))) +} + +/// Call before module execution, while holding the transaction later used for +/// migration. A cache or pre-enqueue check cannot replace this admission check. +pub fn check_deployment_commit( + tx: &MutTx, + request: &DeploymentCommit, + now: Timestamp, + limits: &ContainerSpecLimits, +) -> Result { + let now_ms = u64::try_from(now.to_micros_since_unix_epoch()).map_err(|_| DeploymentError::CorruptMetadata)? / 1000; + operation_expiry_ms(request.operation_id, now_ms)?; + let (_, revision, request_hash) = normalized_request(request, limits)?; + let operation_key = AlgebraicValue::U128(request.operation_id.as_u128().into()); + if let Some(row) = tx + .iter_by_col_eq(ST_DEPLOYMENT_OPERATION_ID, ColId(0), &operation_key)? + .next() + { + let row = StDeploymentOperationRow::try_from(row)?; + let CommitReceipt::V1(receipt) = + bsatn::from_slice(&row.commit_result).map_err(|_| DeploymentError::CorruptMetadata)?; + if receipt.request_hash != request_hash || receipt.publisher != request.publisher { + return Err(DeploymentError::OperationConflict); + } + if receipt.result.revision != revision + || row.committed_revision != revision + || row.previous_revision != receipt.result.previous_revision + || receipt.result.operation_id != request.operation_id + { + return Err(DeploymentError::CorruptMetadata); + } + return Ok(CommitAdmission::AlreadyCommitted(receipt.result)); + } + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + if !fence.is_some_and(|f| { + f.publication_epoch == request.publication_epoch && f.operation_id == request.operation_id.as_u128() + }) { + return Err(DeploymentError::PublicationFenced); + } + if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + Ok(CommitAdmission::Ready) +} + +/// Record after successful module initialization/migration in that same +/// transaction. An error must roll back the entire transaction, including the +/// module changes. The caller waits for durability before reporting acceptance. +pub fn record_deployment_commit( + tx: &mut MutTx, + request: &DeploymentCommit, + now: Timestamp, + limits: &ContainerSpecLimits, +) -> Result { + if let CommitAdmission::AlreadyCommitted(result) = check_deployment_commit(tx, request, now, limits)? { + return Ok(result); + } + let (spec, revision, request_hash) = normalized_request(request, limits)?; + let result = PublishResult { + operation_id: request.operation_id, + previous_revision: request.expected_revision, + revision, + }; + let receipt = CommitReceipt::V1(CommitReceiptV1 { + request_hash, + publisher: request.publisher, + result: result.clone(), + }); + let receipt = bsatn::to_vec(&receipt) + .map_err(|_| DeploymentError::CorruptMetadata)? + .into_boxed_slice(); + let expires_ms = operation_expiry_ms(request.operation_id, (now.to_micros_since_unix_epoch() / 1000) as u64)?; + let expires_us = i64::try_from(expires_ms * 1000).map_err(|_| DeploymentError::CorruptMetadata)?; + let row = StDeploymentRow { + key: 0, + revision, + last_operation_id: request.operation_id.as_u128(), + payload: spec.encode()?, + }; + tx.clear_table(ST_DEPLOYMENT_ID)?; + tx.insert_via_serialize_bsatn(ST_DEPLOYMENT_ID, &row)?; + tx.insert_via_serialize_bsatn( + ST_DEPLOYMENT_OPERATION_ID, + &StDeploymentOperationRow { + operation_id: request.operation_id.as_u128(), + previous_revision: request.expected_revision, + committed_revision: revision, + commit_result: receipt, + expires_at: Timestamp::from_micros_since_unix_epoch(expires_us).into(), + }, + )?; + Ok(result) +} + +/// Install the frozen generation/grant tuple. Changes at the same generation +/// cannot reopen a revoked grant, even if a stale coordinator changes only the +/// target-set hash. Control must allocate a new generation for every barrier. +pub fn install_container_fence( + db: &RelationalDB, + tx: &mut MutTx, + next: &StContainerFenceRow, +) -> Result<(), DeploymentError> { + if next.generation == 0 { + return Err(DeploymentError::FenceConflict); + } + let key: AlgebraicValue = next.source_identity.into(); + let previous = tx + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(|row| StContainerFenceRow::try_from(row).map(|value| (row.pointer(), value))) + .transpose()?; + if let Some((pointer, previous)) = previous { + if previous == *next { + return Ok(()); + } + if next.generation <= previous.generation || next.target_grant_revision < previous.target_grant_revision { + return Err(DeploymentError::FenceConflict); + } + db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + } + tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, next)?; + Ok(()) +} + +/// Called with verified hosted credentials inside every admitted transaction, +/// including each later transaction of a procedure. Signature, audience, expiry, +/// capability, and interface checks are additional receiving-host requirements. +pub fn check_container_fence( + state: &S, + source: Identity, + generation: u64, + target_grant_revision: u64, +) -> Result<(), DeploymentError> { + let key = AlgebraicValue::U256(source.to_u256().into()); + let fence = state + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(StContainerFenceRow::try_from) + .transpose()?; + if !fence + .is_some_and(|f| f.allowed && f.generation == generation && f.target_grant_revision == target_grant_revision) + { + return Err(DeploymentError::ContainerFenced); + } + Ok(()) +} + +/// Capture validated hosted connection authority in the transaction inserting st_client. +/// This is host-only metadata; ordinary connections leave no row and retain flags zero. +pub(crate) fn record_connection_auth( + tx: &mut MutTx, + connection_id: ConnectionId, + sender: Identity, + call_auth_flags: u32, +) -> Result<(), DeploymentError> { + tx.insert_via_serialize_bsatn( + ST_CONNECTION_AUTH_ID, + &StConnectionAuthRow { + connection_id: connection_id.into(), + sender_identity: sender.into(), + call_auth_flags, + }, + )?; + Ok(()) +} + +/// Recover captured authority for a host-dispatched disconnect event. It is not +/// a new container admission and does not require a still-valid credential/lease. +pub(crate) fn connection_auth_flags( + state: &S, + connection_id: ConnectionId, + sender: Identity, +) -> Result { + let key: AlgebraicValue = ConnectionIdViaU128::from(connection_id).into(); + let row = state + .iter_by_col_eq(ST_CONNECTION_AUTH_ID, ColId(0), &key)? + .next() + .map(StConnectionAuthRow::try_from) + .transpose()?; + let Some(row) = row else { return Ok(0) }; + if row.sender_identity.0 != sender { + return Err(DeploymentError::CorruptMetadata); + } + Ok(row.call_auth_flags) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/db/deployment/tests.rs b/crates/core/src/db/deployment/tests.rs new file mode 100644 index 00000000000..130a4368093 --- /dev/null +++ b/crates/core/src/db/deployment/tests.rs @@ -0,0 +1,385 @@ +use super::*; +use crate::db::relational_db::tests_utils::{begin_mut_tx, TestDB}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::system_tables::{ + StEnvRow, ST_CLIENT_ID, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, ST_ENV_ID, +}; +use spacetimedb_durability::Durability; +use spacetimedb_lib::deployment::{ + DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind, PUBLISH_RETRY_WINDOW_MS, +}; + +fn request(sequence: u64, previous: Option) -> DeploymentCommit { + DeploymentCommit { + operation_id: Uuid::from_u128(0x01991ec4000070008000000000000000 | u128::from(sequence)), + publication_epoch: sequence, + publisher: Identity::from_u256(55u64.into()), + expected_revision: previous, + prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(sequence.to_le_bytes()), + }), + container: None, + }), + } +} + +fn now() -> Timestamp { + Timestamp::from_micros_since_unix_epoch(((request(1, None).operation_id.as_u128() >> 80) as i64) * 1000) +} + +fn transact( + db: &RelationalDB, + f: impl FnOnce(&mut MutTx) -> Result, +) -> Result { + db.with_auto_commit(Workload::ForTests, f) +} + +#[test] +fn deployment_retry_returns_original_result_after_later_publish_without_mutation() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let accepted = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let second = request(2, Some(accepted.revision)); + let later = transact(&db, |tx| { + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!(check_deployment_commit(tx, &first, now(), &limits)?, CommitAdmission::AlreadyCommitted(ref r) if r == &accepted)); + assert_eq!(record_deployment_commit(tx, &first, now(), &limits)?, accepted); + assert_eq!(current_deployment(tx)?.unwrap().0, later.revision); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(2)); + Ok(()) + }).unwrap(); +} + +#[test] +fn deployment_fence_and_revision_conflicts_fail_before_module_execution() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let mut second = request(2, Some(hash_bytes(b"not current"))); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, second.publication_epoch, second.operation_id) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits), + Err(DeploymentError::RevisionConflict) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + second.expected_revision = None; + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn deployment_and_module_effects_roll_back_together() { + let db = TestDB::in_memory().unwrap(); + let request = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); + let failed: Result<(), DeploymentError> = transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + // A second persistent table stands in for migration effects in the same + // transaction. Integration must additionally execute Wasm/JS migrations. + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: "MIGRATED".into(), + value: "yes".into(), + }, + )?; + record_deployment_commit(tx, &request, now(), &limits)?; + Err(DeploymentError::Database(DBError::Other(anyhow::anyhow!( + "injected failure before commit" + )))) + }); + assert!(failed.is_err()); + transact(&db, |tx| { + assert!(current_deployment(tx)?.is_none()); + assert_eq!(tx.table_row_count(ST_ENV_ID), Some(0)); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(0)); + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn deployment_operation_cannot_be_reused_by_another_publisher_or_changed_request() { + let db = TestDB::in_memory().unwrap(); + let original = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, original.publication_epoch, original.operation_id)?; + record_deployment_commit(tx, &original, now(), &limits) + }) + .unwrap(); + transact(&db, |tx| { + let mut changed = original.clone(); + changed.publisher = Identity::from_u256(99u64.into()); + assert!(matches!( + check_deployment_commit(tx, &changed, now(), &limits), + Err(DeploymentError::OperationConflict) + )); + changed = original.clone(); + changed.deployment = DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(1), + container: None, + }); + assert!(matches!( + check_deployment_commit(tx, &changed, now(), &limits), + Err(DeploymentError::OperationConflict) + )); + // A retained row does not extend the advertised retry window. + let expired = Timestamp::from_micros_since_unix_epoch( + now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS as i64) * 1000, + ); + assert!(matches!( + check_deployment_commit(tx, &original, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn container_fence_revokes_copied_credentials_and_does_not_reopen_at_same_generation() { + let db = TestDB::in_memory().unwrap(); + let source = Identity::from_u256(777u64.into()); + let first = StContainerFenceRow { + source_identity: source.into(), + generation: 1, + target_grant_revision: 3, + target_set_hash: hash_bytes(b"targets1"), + allowed: true, + }; + transact(&db, |tx| { + assert!(matches!( + check_container_fence(tx, source, 1, 3), + Err(DeploymentError::ContainerFenced) + )); + install_container_fence(&db, tx, &first)?; + install_container_fence(&db, tx, &first)?; + check_container_fence(tx, source, 1, 3)?; + assert!(matches!( + check_container_fence(tx, source, 1, 4), + Err(DeploymentError::ContainerFenced) + )); + Ok(()) + }) + .unwrap(); + let revoked = StContainerFenceRow { + generation: 2, + target_grant_revision: 4, + target_set_hash: hash_bytes(b"targets2"), + allowed: false, + ..first.clone() + }; + transact(&db, |tx| { + install_container_fence(&db, tx, &revoked)?; + assert!(matches!( + check_container_fence(tx, source, 1, 3), + Err(DeploymentError::ContainerFenced) + )); + assert!(matches!( + check_container_fence(tx, source, 2, 4), + Err(DeploymentError::ContainerFenced) + )); + let reopen = StContainerFenceRow { + allowed: true, + ..revoked.clone() + }; + assert!(matches!( + install_container_fence(&db, tx, &reopen), + Err(DeploymentError::FenceConflict) + )); + assert!(matches!( + install_container_fence(&db, tx, &first), + Err(DeploymentError::FenceConflict) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn captured_connection_authority_survives_replay_without_inferring_sender_authority() { + let db = TestDB::durable().unwrap(); + let self_sender = db.database_identity(); + let foreign_sender = Identity::ONE; + let self_connection = ConnectionId::from_u128(41); + let foreign_connection = ConnectionId::from_u128(42); + let ordinary_connection = ConnectionId::from_u128(43); + transact(&db, |tx| { + for (connection, sender, flags) in [ + (self_connection, self_sender, 1), + (foreign_connection, foreign_sender, 0), + ] { + tx.insert_st_client( + sender, + connection, + r#"{"iss":"platform","sub":"previously-admitted","exp":1}"#, + )?; + record_connection_auth(tx, connection, sender, flags)?; + } + tx.insert_st_client(self_sender, ordinary_connection, "ordinary JWT")?; + Ok(()) + }) + .unwrap(); + // TestDB::reopen expects zero connected clients. Reopen the same committed + // log explicitly to exercise crash recovery with outstanding connections. + let (db, durability, runtime, directory) = db.into_parts(); + let runtime = runtime.unwrap(); + let directory = directory.unwrap(); + let durability = durability.unwrap(); + runtime.block_on(db.shutdown()); + drop(db); + runtime.block_on(durability.close()); + drop(durability); + let _runtime_guard = runtime.enter(); + let (db, durability) = TestDB::open_existing_durable( + &directory, + runtime.handle().clone(), + 0, + TestDB::DATABASE_IDENTITY, + TestDB::OWNER, + true, + ) + .unwrap(); + transact(&db, |tx| { + assert_eq!(connection_auth_flags(tx, self_connection, self_sender)?, 1); + assert_eq!(connection_auth_flags(tx, foreign_connection, foreign_sender)?, 0); + // Equal sender/database identities do not invent internal authority. + assert_eq!(connection_auth_flags(tx, ordinary_connection, self_sender)?, 0); + assert_eq!(tx.table_row_count(ST_CONNECTION_AUTH_ID), Some(2)); + assert!(matches!( + connection_auth_flags(tx, self_connection, foreign_sender), + Err(DeploymentError::CorruptMetadata) + )); + Ok(()) + }) + .unwrap(); + db.clear_all_clients().unwrap(); + transact(&db, |tx| { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(0)); + } + Ok(()) + }) + .unwrap(); + runtime.block_on(db.shutdown()); + drop(db); + runtime.block_on(durability.close()); +} + +#[test] +fn connection_auth_and_client_rows_share_connect_and_cleanup_transactions() { + let db = TestDB::in_memory().unwrap(); + let sender = db.database_identity(); + let connection = ConnectionId::from_u128(77); + let rejected: Result<(), DeploymentError> = transact(&db, |tx| { + tx.insert_st_client(sender, connection, "JWT")?; + record_connection_auth(tx, connection, sender, 1)?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(rejected.is_err()); + transact(&db, |tx| { + assert!(tx.st_client_row(sender, connection).is_none()); + assert_eq!(connection_auth_flags(tx, connection, sender)?, 0); + tx.insert_st_client(sender, connection, "JWT")?; + record_connection_auth(tx, connection, sender, 1)?; + Ok(()) + }) + .unwrap(); + // A failed callback transaction cannot partially delete its captured auth. + let failed_callback: Result<(), DeploymentError> = transact(&db, |tx| { + tx.delete_st_client(sender, connection, db.database_identity())?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(failed_callback.is_err()); + transact(&db, |tx| { + assert!(tx.st_client_row(sender, connection).is_some()); + assert_eq!(connection_auth_flags(tx, connection, sender)?, 1); + // Both successful callbacks and fallback cleanup use this deletion path. + tx.delete_st_client(sender, connection, db.database_identity())?; + Ok(()) + }) + .unwrap(); + transact(&db, |tx| { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(0)); + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn container_fence_installation_serializes_with_admitted_transactions() { + let db = TestDB::in_memory().unwrap(); + let source = Identity::from_u256(778u64.into()); + let first = StContainerFenceRow { + source_identity: source.into(), + generation: 1, + target_grant_revision: 0, + target_set_hash: hash_bytes(b"self"), + allowed: true, + }; + transact(&db, |tx| install_container_fence(&db, tx, &first)).unwrap(); + let admitted = begin_mut_tx(&db); + check_container_fence(&admitted, source, 1, 0).unwrap(); + // A fence cannot commit midway through an already admitted transaction. + assert!(db + .try_begin_mut_tx( + spacetimedb_datastore::traits::IsolationLevel::Serializable, + Workload::ForTests + ) + .is_none()); + let _ = db.rollback_mut_tx(admitted); + transact(&db, |tx| { + install_container_fence(&db, tx, &StContainerFenceRow { generation: 2, ..first }) + }) + .unwrap(); + transact(&db, |tx| { + assert!(matches!( + check_container_fence(tx, source, 1, 0), + Err(DeploymentError::ContainerFenced) + )); + check_container_fence(tx, source, 2, 0) + }) + .unwrap(); +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 45777fbd612..86686e6a133 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -10,6 +10,7 @@ use spacetimedb_datastore::execution_context::WorkloadType; use spacetimedb_datastore::{locking_tx_datastore::datastore::TxMetrics, traits::TxData}; mod durability; +pub mod deployment; pub mod persistence; pub mod relational_db; pub mod snapshot; diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index c8b81b90fea..e5e2ef57373 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -1489,6 +1489,7 @@ impl RelationalDB { self.with_auto_commit(Workload::Internal, |mut_tx| { self.clear_all_views(mut_tx)?; self.clear_table(mut_tx, ST_CONNECTION_CREDENTIALS_ID)?; + self.clear_table(mut_tx, spacetimedb_datastore::system_tables::ST_CONNECTION_AUTH_ID)?; self.clear_table(mut_tx, ST_CLIENT_ID)?; self.clear_table(mut_tx, ST_VIEW_SUB_ID)?; Ok(()) diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 4ba103979ec..5eaa9e2f02b 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -237,6 +237,8 @@ pub enum LogReplayError { #[derive(Error, Debug)] pub enum NodesError { + #[error("hosted invocation rejected: {0}")] + HostedInvocationRejected(String), #[error("Failed to decode row: {0}")] DecodeRow(#[source] DecodeError), #[error("Failed to decode value: {0}")] diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 2fec6f79ebf..774e23d5339 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -989,9 +989,10 @@ impl Host { // the host type in `st_module` was always set to wasm. // We now correctly use the host type from the database, but the // module may in fact be a JS module. - // So if launching it as a wasm module fails, try JS instead. - // If this succeeds, the module is definitely a JS module, so - // attempt to repair `st_module` in this case. + // Retry JS only for an existing stored module whose database + // declaration explicitly identifies it as JS. A new publication + // or declared Wasm module must preserve its Wasm validation error. + // If the legacy retry succeeds, repair `st_module`. // // TODO: This code should eventually be removed once all // databases have been repaired. @@ -1015,6 +1016,9 @@ impl Host { match launch_wasm_result { Ok(program_and_module_host) => program_and_module_host, Err(e) => { + if program_needs_init || database.host_type != HostType::Js { + return Err(e); + } warn!("failed to launch wasm module, trying js: {e:#}"); program.kind = ModuleKind::JS; @@ -1041,7 +1045,9 @@ impl Host { .with_auto_commit(Workload::Internal, |tx| relational_db.update_program(tx, program)); } - res? + res.map_err(|js_error| { + e.context(format!("legacy JS host-type repair also failed: {js_error:#}")) + })? } } } @@ -1246,7 +1252,10 @@ impl Host { old_module.module_def.raw_module_def_version(), module_def.raw_module_def_version() ), - (RawModuleDefVersion::V9OrEarlier, RawModuleDefVersion::V10) + ( + RawModuleDefVersion::V9OrEarlier, + RawModuleDefVersion::V10 | RawModuleDefVersion::V11 + ) ); let res = match ponder_migrate(&old_module.module_def, &module_def) { diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 6b1c5054cde..25614821750 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1,4 +1,6 @@ use super::scheduler::{get_schedule_from_row, ScheduleError, Scheduler}; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::check_hosted_admission; use crate::database_logger::{BacktraceFrame, BacktraceProvider, LogLevel, ModuleBacktrace, Record}; use crate::db::relational_db::{MutTx, RelationalDB}; use crate::error::{DBError, DatastoreError, IndexError, NodesError}; @@ -18,6 +20,7 @@ use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, IndexScanPointOrRange, MutTxId}; +use spacetimedb_datastore::system_tables::{is_module_restricted_index, is_module_restricted_table}; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{http as st_http, ConnectionId, Identity, Timestamp}; use spacetimedb_primitives::{ColId, ColList, IndexId, TableId}; @@ -49,6 +52,10 @@ pub struct InstanceEnv { pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. pub func_name: Option, + /// Set by trusted host dispatch for this invocation, independently of JWTs + /// and connection IDs. Cleared before each new function call. + call_auth_flags: u32, + hosted_auth: Option>, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. @@ -235,6 +242,8 @@ impl InstanceEnv { // run a function func_type: FuncCallType::Reducer, func_name: None, + call_auth_flags: 0, + hosted_auth: None, in_anon_tx: false, procedure_last_tx_offset: None, } @@ -251,6 +260,20 @@ impl InstanceEnv { self.start_instant = Instant::now(); self.func_type = func_type; self.func_name = Some(name); + self.call_auth_flags = 0; + self.hosted_auth = None; + } + + pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { + self.call_auth_flags = flags; + } + + pub(crate) fn set_hosted_auth(&mut self, auth: Option>) { + self.hosted_auth = auth; + } + + pub(crate) fn get_call_auth_flags(&self) -> u32 { + self.call_auth_flags } /// Returns the name of the most recent reducer to be run in this environment, @@ -282,8 +305,15 @@ impl InstanceEnv { } pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { - let tx = &mut *self.get_tx()?; - Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?) + if let Ok(tx) = self.get_tx() { + return Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?); + } + // Procedures may inspect authentication before opening their first + // transaction. Use a short read transaction without manufacturing an + // internal caller or dropping the real connection's JWT. + Ok(self.relational_db().with_read_only(Workload::Internal, |tx| { + tx.get_jwt_payload(connection_id).map_err(DBError::from) + })?) } #[tracing::instrument(level = "trace", skip_all)] @@ -355,9 +385,20 @@ impl InstanceEnv { count } + /// Engine-owned deployment/authentication records and environment values + /// are reachable only through their dedicated host interfaces. + fn require_module_table(table_id: TableId) -> Result<(), NodesError> { + if is_module_restricted_table(table_id) { + Err(NodesError::TableNotFound) + } else { + Ok(()) + } + } + pub fn insert(&self, table_id: TableId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, insert_flags) = stdb .insert(tx, table_id, buffer) @@ -436,6 +477,7 @@ impl InstanceEnv { pub fn update(&self, table_id: TableId, index_id: IndexId, buffer: &mut [u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let (row_len, row_ptr, update_flags) = stdb .update(tx, table_id, index_id, buffer) @@ -479,6 +521,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, _, iter) = stdb.index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = iter.map(|row_ref| row_ref.pointer()).collect::>(); @@ -499,6 +542,7 @@ impl InstanceEnv { // Find all rows in the table to delete. let (table_id, iter) = stdb.index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Re. `SmallVec`, `delete_by_field` only cares about 1 element, so optimize for that. let rows_to_delete = match iter { IndexScanPointOrRange::Point(_, iter) => iter.map(|row_ref| row_ref.pointer()).collect(), @@ -540,6 +584,7 @@ impl InstanceEnv { pub fn datastore_delete_all_by_eq_bsatn(&self, table_id: TableId, relation: &[u8]) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Track the number of bytes coming from the caller tx.metrics.bytes_scanned += relation.len(); @@ -563,6 +608,7 @@ impl InstanceEnv { pub fn clear(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; let rows_deleted = stdb.clear_table(tx, table_id).map_err(NodesError::from)?; @@ -584,6 +630,7 @@ impl InstanceEnv { // Query the table id from the name. stdb.table_id_from_name_mut(tx, table_name)? + .filter(|id| !is_module_restricted_table(*id)) .ok_or(NodesError::TableNotFound) } @@ -598,6 +645,7 @@ impl InstanceEnv { // Query the index id from the name. stdb.index_id_from_name_mut(tx, index_name)? + .filter(|id| !is_module_restricted_index(*id)) .ok_or(NodesError::IndexNotFound) } @@ -609,6 +657,7 @@ impl InstanceEnv { pub fn datastore_table_row_count(&self, table_id: TableId) -> Result { let stdb = self.relational_db(); let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Query the row count for id. stdb.table_row_count_mut(tx, table_id) @@ -625,6 +674,7 @@ impl InstanceEnv { table_id: TableId, ) -> Result>, NodesError> { let tx = &mut *self.get_tx()?; + Self::require_module_table(table_id)?; // Open the iterator. let iter = self.relational_db().iter_mut(tx, table_id)?; @@ -652,6 +702,7 @@ impl InstanceEnv { // Open index iterator let (table_id, point, iter) = self.relational_db().index_scan_point(tx, index_id, point)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (chunks, rows_scanned, bytes_scanned) = ChunkedWriter::collect_iter(pool, iter); @@ -682,6 +733,7 @@ impl InstanceEnv { let (table_id, iter) = self.relational_db() .index_scan_range(tx, index_id, prefix, prefix_elems, rstart, rend)?; + Self::require_module_table(table_id)?; // Scan the index and serialize rows to BSATN. let (point, (chunks, rows_scanned, bytes_scanned)) = match iter { @@ -741,6 +793,10 @@ impl InstanceEnv { let tx = self .relational_db() .begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + if let Err(err) = check_hosted_admission(&tx, *self.database_identity(), self.hosted_auth.as_deref()) { + let _ = tx.rollback(); + return Err(NodesError::HostedInvocationRejected(err.to_string())); + } self.tx.set_raw(tx); self.in_anon_tx = true; @@ -1407,6 +1463,91 @@ mod test { product!(id, str) } + #[test] + fn module_cannot_access_hosted_system_records_by_guessed_ids() -> Result<()> { + use spacetimedb_datastore::system_tables::{ + ST_CONNECTION_AUTH_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, + ST_PUBLISH_FENCE_ID, + }; + let db = relational_db()?; + let (env, _runtime) = instance_env(db.clone())?; + let mut slot = env.tx.clone(); + let protected = [ + (ST_ENV_ID, "st_env", to_vec("TOKEN")?), + (ST_DEPLOYMENT_ID, "st_deployment", to_vec(&0u8)?), + (ST_PUBLISH_FENCE_ID, "st_publish_fence", to_vec(&0u8)?), + (ST_DEPLOYMENT_OPERATION_ID, "st_deployment_operation", to_vec(&0u128)?), + (ST_CONNECTION_AUTH_ID, "st_connection_auth", to_vec(&0u128)?), + ( + ST_CONTAINER_FENCE_ID, + "st_container_fence", + to_vec(&spacetimedb_sats::u256::ZERO)?, + ), + ]; + let tx = begin_mut_tx(&db); + let (tx, result) = slot.set(tx, || -> Result<()> { + for (table, name, point) in &protected { + // Host lookup remains available, independently of module lookup. + let (index, index_name) = { + let tx = env.get_tx()?; + let schema = db.schema_for_table_mut(&tx, *table)?; + let index = &schema.indexes[0]; + (index.index_id, index.index_name.to_string()) + }; + assert!(matches!(env.table_id_from_name(name), Err(NodesError::TableNotFound))); + assert!(matches!( + env.index_id_from_name(&index_name), + Err(NodesError::IndexNotFound) + )); + assert!(matches!(env.insert(*table, &mut []), Err(NodesError::TableNotFound))); + assert!(matches!( + env.update(*table, index, &mut []), + Err(NodesError::TableNotFound) + )); + assert!(matches!(env.clear(*table), Err(NodesError::TableNotFound))); + assert!(matches!( + env.datastore_table_row_count(*table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_table_scan_bsatn_chunks(&mut ChunkPool::default(), *table), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_all_by_eq_bsatn(*table, &[]), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_index_scan_point_bsatn_chunks(&mut ChunkPool::default(), index, point), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_point_bsatn(index, point), + Err(NodesError::TableNotFound) + )); + let bound = to_vec(&Bound::::Unbounded)?; + assert!(matches!( + env.datastore_index_scan_range_bsatn_chunks( + &mut ChunkPool::default(), + index, + &[], + 0.into(), + &bound, + &bound + ), + Err(NodesError::TableNotFound) + )); + assert!(matches!( + env.datastore_delete_by_index_scan_range_bsatn(index, &[], 0.into(), &bound, &bound), + Err(NodesError::TableNotFound) + )); + } + Ok(()) + }); + let _ = db.rollback_mut_tx(tx); + result + } + /// Generate a BSATN encoded row for use in [create_table_with_index] fn bsatn_row(i: usize) -> Result> { Ok(to_vec(&product_row(i))?) diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index daa506a8cf5..c25c47bb99b 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -187,6 +187,7 @@ pub enum AbiCall { Identity, JwtLength, GetJwt, + GetCallAuthFlags, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 1a8cf3257f9..9232be528c0 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -2,6 +2,8 @@ use super::{ ArgsTuple, FunctionArgs, InvalidProcedureArguments, InvalidReducerArguments, ReducerCallResult, ReducerId, ReducerOutcome, Scheduler, }; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::{check_hosted_admission, InvocationCaller, SqlCallAuth}; use crate::client::messages::{OneOffQueryResponseMessage, ProcedureResultMessage, SerializableMessage}; use crate::client::{ClientActorId, ClientConnectionSender, WsVersion}; use crate::database_logger::{DatabaseLogger, LogLevel, Record}; @@ -690,6 +692,13 @@ pub fn call_identity_connected( stdb.report_mut_tx_metrics(reducer_name, metrics, None); }); + let caller = InvocationCaller::from(&caller_auth); + let flags = caller + .flags_for(module.database_identity, &module.module_def) + .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; + check_hosted_admission(&*mut_tx, module.database_identity, caller.hosted.as_deref()) + .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; + mut_tx .insert_st_client( caller_auth.claims.identity, @@ -699,13 +708,23 @@ pub fn call_identity_connected( .map_err(DBError::from) .map_err(Box::new)?; + if caller.hosted.is_some() { + crate::db::deployment::record_connection_auth( + &mut mut_tx, + caller_connection_id, + caller_auth.claims.identity, + flags, + ) + .map_err(|err| ClientConnectedError::DBError(Box::new(DBError::Other(err.into()))))?; + } + if let Some((reducer_id, reducer_def)) = reducer_lookup { // The module defined a lifecycle reducer to handle new connections. // Call this reducer. // If the call fails (as in, something unexpectedly goes wrong with guest execution), // abort the connection: we can't really recover. let tx = Some(ScopeGuard::into_inner(mut_tx)); - let params = ModuleHost::call_reducer_params( + let mut params = ModuleHost::call_reducer_params( module, caller_auth.claims.identity, Some(caller_connection_id), @@ -717,6 +736,8 @@ pub fn call_identity_connected( FunctionArgs::Nullary, ) .map_err(ReducerCallError::from)?; + params.call_auth_flags = flags; + params.hosted_auth = caller.hosted; let (reducer_outcome, trapped) = call_reducer(tx, params); *trapped_slot = trapped; @@ -761,6 +782,9 @@ pub struct CallReducerParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + /// Verified invocation authority. Bit 0 is internal; never client-decoded. + pub(crate) call_auth_flags: u32, + pub(crate) hosted_auth: Option>, pub client: Option>, pub request_id: Option, pub timer: Option, @@ -781,6 +805,8 @@ impl CallReducerParams { timestamp, caller_identity, caller_connection_id: ConnectionId::ZERO, + call_auth_flags: 1, + hosted_auth: None, client: None, request_id: None, timer: None, @@ -986,7 +1012,7 @@ impl ViewCommandErrorTarget { pub(in crate::host) struct SqlCommand { pub(in crate::host) db: Arc, pub(in crate::host) sql_text: String, - pub(in crate::host) auth: AuthCtx, + pub(in crate::host) auth: SqlCallAuth, pub(in crate::host) subs: Option, } @@ -1093,6 +1119,8 @@ pub struct CallProcedureParams { pub timestamp: Timestamp, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + pub(crate) call_auth_flags: u32, + pub(crate) hosted_auth: Option>, pub timer: Option, pub procedure_id: ProcedureId, pub args: ArgsTuple, @@ -1111,6 +1139,8 @@ impl CallProcedureParams { timestamp, caller_identity, caller_connection_id: ConnectionId::ZERO, + call_auth_flags: 1, + hosted_auth: None, timer: None, procedure_id, args, @@ -2017,7 +2047,7 @@ impl ModuleHost { } /// Invokes the `client_disconnected` reducer, if present, - /// then deletes the client’s rows from `st_client` and `st_connection_credentials`. + /// then deletes the client's rows from `st_client`, `st_connection_credentials`, and `st_connection_auth`. /// If the reducer fails, the rows are still deleted. /// Calling this on an already-disconnected client is a no-op. pub fn call_identity_disconnected_inner( @@ -2081,22 +2111,35 @@ impl ModuleHost { // The module defined a lifecycle reducer to handle disconnects. Call it. // If it succeeds, `WasmModuleInstance::call_reducer_with_tx` has already ensured // that `st_client` is updated appropriately. + let flags = crate::db::deployment::connection_auth_flags(&mut_tx, caller_connection_id, caller_identity) + .map_err(|err| { + InvalidReducerArguments(InvalidFunctionArguments { + err: err.into(), + function_name: reducer_name.clone().into(), + }) + }); let tx = Some(mut_tx); - let result = Self::call_reducer_params( - info, - caller_identity, - Some(caller_connection_id), - None, - None, - None, - reducer_id, - reducer_def, - FunctionArgs::Nullary, - ) - .map(|params| { - let (res, trapped) = call_reducer(tx, params); - *trapped_slot = trapped; - res + let result = flags.and_then(|flags| { + Self::call_reducer_params( + info, + caller_identity, + Some(caller_connection_id), + None, + None, + None, + reducer_id, + reducer_def, + FunctionArgs::Nullary, + ) + .map(|mut params| { + // This host event retains the connection's captured authority, + // including after credential expiry, revocation, or host recovery. + // It must not carry a live hosted proof that could block cleanup. + params.call_auth_flags = flags; + let (res, trapped) = call_reducer(tx, params); + *trapped_slot = trapped; + res + }) }); // If it failed, we still need to update `st_client`: the client's not coming back. @@ -2188,6 +2231,8 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + call_auth_flags: 0, + hosted_auth: None, client, request_id, timer, @@ -2198,7 +2243,7 @@ impl ModuleHost { fn reducer_call_params<'a>( &'a self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2206,6 +2251,10 @@ impl ModuleHost { reducer_name: &str, args: FunctionArgs, ) -> Result<(&'a ReducerDef, CallReducerParams), ReducerCallError> { + let flags = caller + .flags_for(self.info.database_identity, &self.info.module_def) + .map_err(|_| ReducerCallError::NoSuchReducer)?; + let caller_identity = caller.identity; let (reducer_id, reducer_def) = self .info .module_def @@ -2215,24 +2264,27 @@ impl ModuleHost { return Err(ReducerCallError::LifecycleReducer(lifecycle)); } - if reducer_def.visibility.is_private() && !self.is_database_owner(caller_identity) { + if !reducer_def + .visibility + .allows_invocation(flags & 1 != 0, self.is_database_owner(caller_identity)) + { return Err(ReducerCallError::NoSuchReducer); } - Ok(( + let mut params = Self::call_reducer_params( + &self.info, + caller_identity, + caller_connection_id, + client, + request_id, + timer, + reducer_id, reducer_def, - Self::call_reducer_params( - &self.info, - caller_identity, - caller_connection_id, - client, - request_id, - timer, - reducer_id, - reducer_def, - args, - )?, - )) + args, + )?; + params.call_auth_flags = flags; + params.hosted_auth = caller.hosted; + Ok((reducer_def, params)) } async fn call_reducer_with_params( @@ -2260,7 +2312,7 @@ impl ModuleHost { async fn with_reducer_call( &self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2271,7 +2323,7 @@ impl ModuleHost { ) -> Result { let res = async { let (reducer_def, params) = self.reducer_call_params( - caller_identity, + caller, caller_connection_id, client, request_id, @@ -2296,7 +2348,7 @@ impl ModuleHost { pub async fn call_reducer( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2305,7 +2357,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result { self.with_reducer_call( - caller_identity, + caller.into(), caller_connection_id, client, request_id, @@ -2319,7 +2371,7 @@ impl ModuleHost { pub async fn enqueue_reducer( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, client: Option>, request_id: Option, @@ -2328,7 +2380,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result<(), ReducerCallError> { self.with_reducer_call( - caller_identity, + caller.into(), caller_connection_id, client, request_id, @@ -2471,10 +2523,15 @@ impl ModuleHost { &self, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result { + InvocationCaller { + identity: auth.caller(), + hosted: auth.hosted.clone(), + } + .flags_for(self.info.database_identity, &self.info.module_def)?; let cmd = SqlCommand { db, sql_text, @@ -2490,15 +2547,14 @@ impl ModuleHost { pub async fn call_procedure( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> CallProcedureReturn { let res = async { - let call = - self.prepare_procedure_call(caller_identity, caller_connection_id, timer, procedure_name, args)?; + let call = self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args)?; self.call_procedure_with_params(&call.name, call.params) .await .map_err(Into::into) @@ -2514,7 +2570,7 @@ impl ModuleHost { pub(crate) async fn enqueue_procedure( &self, - caller_identity: Identity, + caller: impl Into + Send, caller_connection_id: Option, timer: Option, procedure_name: &str, @@ -2522,7 +2578,7 @@ impl ModuleHost { target: ProcedureResultTarget, ) -> Result<(), BroadcastError> { let PreparedProcedureCall { name, params } = - match self.prepare_procedure_call(caller_identity, caller_connection_id, timer, procedure_name, args) { + match self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args) { Ok(value) => value, Err(err) => { return self.send_procedure_error(procedure_name, timer, target, err); @@ -2697,14 +2753,14 @@ impl ModuleHost { fn prepare_procedure_call( &self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> Result { let (procedure_def, params) = - self.procedure_call_params(caller_identity, caller_connection_id, timer, procedure_name, args)?; + self.procedure_call_params(caller, caller_connection_id, timer, procedure_name, args)?; Ok(PreparedProcedureCall { name: procedure_def.name.to_string(), params, @@ -2713,19 +2769,26 @@ impl ModuleHost { fn procedure_call_params<'a>( &'a self, - caller_identity: Identity, + caller: InvocationCaller, caller_connection_id: Option, timer: Option, procedure_name: &str, args: FunctionArgs, ) -> Result<(&'a ProcedureDef, CallProcedureParams), ProcedureCallError> { + let flags = caller + .flags_for(self.info.database_identity, &self.info.module_def) + .map_err(|_| ProcedureCallError::NoSuchProcedure)?; + let caller_identity = caller.identity; let (procedure_id, procedure_def) = self .info .module_def .procedure_full(procedure_name) .ok_or(ProcedureCallError::NoSuchProcedure)?; - if procedure_def.visibility.is_private() && !self.is_database_owner(caller_identity) { + if !procedure_def + .visibility + .allows_invocation(flags & 1 != 0, self.is_database_owner(caller_identity)) + { return Err(ProcedureCallError::NoSuchProcedure); } @@ -2740,6 +2803,8 @@ impl ModuleHost { timestamp: Timestamp::now(), caller_identity, caller_connection_id, + call_auth_flags: flags, + hosted_auth: caller.hosted, timer, procedure_id, args, @@ -3294,8 +3359,10 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { - ws_v1::OneOffTable { table_name, rows } + let result = check_hosted_admission(&*tx, db.database_identity(), client.auth.hosted.as_ref()).and_then(|()| { + Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { + ws_v1::OneOffTable { table_name, rows } + }) }); let total_host_execution_duration = timer.elapsed().into(); @@ -3373,10 +3440,11 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = + let result = check_hosted_admission(&*tx, db.database_identity(), client.auth.hosted.as_ref()).and_then(|()| { Self::execute_one_off_query::(&db, &tx, &auth, &query, &rlb_pool, |table, rows| { ws_v2::SingleTableRows { table, rows } - }); + }) + }); let (message, metrics) = match result { Ok((rows, metrics)) => { diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 433fe5e2af8..9c6237d9826 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -899,7 +899,7 @@ enum JsProcedureWorkerRequest { }, } -static_assert_size!(CallReducerParams, 192); +static_assert_size!(CallReducerParams, 208); fn send_worker_reply(ctx: &str, reply_tx: JsReplyTx, value: T) { if reply_tx.send(Ok(value)).is_err() { @@ -1984,6 +1984,9 @@ where // Start the timer. // We'd like this tightly around `call`. env.start_funcall(op.name().clone(), op.timestamp(), op.call_type()); + env.instance_env.set_call_auth_flags(op.call_auth_flags()); + env.instance_env + .set_hosted_auth(op.hosted_auth().map(|proof| std::sync::Arc::new(proof.clone()))); // Wrap the call in `TryCatch`. // @@ -2113,6 +2116,8 @@ mod test { name: &ReducerName::for_test("foobar"), caller_identity: &Identity::ONE, caller_connection_id: &ConnectionId::ZERO, + call_auth_flags: 0, + hosted_auth: None, timestamp: Timestamp::from_micros_since_unix_epoch(24), args: &ArgsTuple::nullary(), }; diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index 3d93e0b2679..c495b7b0c64 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -44,6 +44,8 @@ pub fn call_call_procedure( name: _, caller_identity: sender, caller_connection_id: connection_id, + call_auth_flags: _, + hosted_auth: _, timestamp, arg_bytes: procedure_args, } = op; diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index a09e7cbba0c..467e7f26562 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -62,6 +62,7 @@ fn resolve_sys_module_inner<'scope>( (1, 3) => Ok(v1::sys_v1_3(scope)), (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), + (2, 2) => Ok(v2::sys_v2_2(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v1.rs b/crates/core/src/host/v8/syscall/v1.rs index f3aea9f6c52..ab852eda401 100644 --- a/crates/core/src/host/v8/syscall/v1.rs +++ b/crates/core/src/host/v8/syscall/v1.rs @@ -495,6 +495,8 @@ pub(super) fn call_call_reducer( name: _, caller_identity: sender, caller_connection_id: conn_id, + call_auth_flags: _, + hosted_auth: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index f49d2260549..e34c9fa9cfe 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -169,6 +169,19 @@ pub(super) fn sys_v2_1<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } +/// Invocation authentication is independent of transaction or connection state. +pub(super) fn sys_v2_2<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!( + scope, + "spacetime:sys@2.2", + (with_sys_result, AbiCall::GetCallAuthFlags, get_call_auth_flags), + ) +} + +fn get_call_auth_flags(scope: &mut PinScope<'_, '_>, _args: FunctionCallbackArguments<'_>) -> SysCallResult { + Ok(get_env(scope)?.instance_env.get_call_auth_flags()) +} + /// Registers a function in `module` /// where the function has `name` and does `body`. fn register_module_fun( @@ -449,6 +462,8 @@ pub(super) fn call_call_reducer<'scope>( name: _, caller_identity: sender, caller_connection_id: conn_id, + call_auth_flags: _, + hosted_auth: _, timestamp, args: reducer_args, } = op; diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 1e7fd18b5e1..17af2c7bb5f 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -442,6 +442,7 @@ macro_rules! abi_funcs { "spacetime_10.4"::datastore_delete_by_index_scan_point_bsatn, "spacetime_10.5"::datastore_clear, + "spacetime_10.6"::get_call_auth_flags, } $link_async! { diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 2fb5eab0492..7087ba93aec 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -1,5 +1,7 @@ use super::instrumentation::CallTimes; use super::*; +use crate::auth::hosted_tokens::VerifiedHostedAuth; +use crate::auth::invocation::check_hosted_admission; use crate::client::ClientActorId; use crate::database_logger; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; @@ -761,11 +763,29 @@ impl InstanceCommon { timestamp, caller_identity, caller_connection_id, + call_auth_flags, + hosted_auth, timer, procedure_id, args, } = params; + let admission = inst + .replica_ctx() + .relational_db() + .with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, self.info.database_identity, hosted_auth.as_deref()) + }); + if let Err(err) = admission { + return ( + CallProcedureReturn { + result: Err(ProcedureCallError::InternalError(err.to_string())), + tx_offset: None, + }, + false, + ); + } + // We've already validated by this point that the procedure exists, // so it's fine to use the panicking `procedure_by_id`. let procedure_def = self.info.module_def.procedure_by_id(procedure_id); @@ -780,6 +800,8 @@ impl InstanceCommon { name: procedure_name.clone(), caller_identity, caller_connection_id, + call_auth_flags, + hosted_auth, timestamp, arg_bytes: args.get_bsatn().clone(), }; @@ -957,6 +979,8 @@ impl InstanceCommon { timestamp, caller_identity, caller_connection_id, + call_auth_flags, + hosted_auth, client, request_id, reducer_id, @@ -980,12 +1004,41 @@ impl InstanceCommon { name: reducer_name, caller_identity: &caller_identity, caller_connection_id: &caller_connection_id, + call_auth_flags, + hosted_auth, timestamp, args: &args, }; let workload = Workload::Reducer(ReducerContext::from(op.clone())); let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable, workload)); + if let Err(err) = check_hosted_admission(&tx, info.database_identity, op.hosted_auth.as_deref()) { + let event = ModuleEvent { + timestamp, + caller_identity, + caller_connection_id: caller_connection_id_opt, + function_call: ModuleFunctionCall { + reducer: Some(reducer_name.clone()), + reducer_id, + args, + }, + status: EventStatus::FailedInternal(err.to_string()), + reducer_return_value: None, + energy_quanta_used: crate::energy::EnergyQuanta::ZERO, + host_execution_duration: Default::default(), + request_id, + timer, + }; + let event = commit_and_broadcast_event(&info.subscriptions, client, event, tx).event; + return ( + ReducerCallResult { + outcome: ReducerOutcome::from(&event.status), + energy_used: crate::energy::EnergyQuanta::ZERO, + execution_duration: Default::default(), + }, + false, + ); + } let mut tx_slot = inst.tx_slot(); let vm_metrics = self.vm_metrics.get_for_reducer_id(reducer_id); @@ -1772,6 +1825,12 @@ pub trait InstanceOp { fn name(&self) -> &Identifier; fn timestamp(&self) -> Timestamp; fn call_type(&self) -> FuncCallType; + fn call_auth_flags(&self) -> u32 { + 0 + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + None + } } /// Describes a view call in a cheaply shareable way. @@ -1842,12 +1901,20 @@ pub struct ReducerOp<'a> { pub name: &'a ReducerName, pub caller_identity: &'a Identity, pub caller_connection_id: &'a ConnectionId, + pub call_auth_flags: u32, + pub hosted_auth: Option>, pub timestamp: Timestamp, /// The arguments passed to the reducer. pub args: &'a ArgsTuple, } impl InstanceOp for ReducerOp<'_> { + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + self.hosted_auth.as_deref() + } fn name(&self) -> &Identifier { self.name.as_identifier() } @@ -1866,6 +1933,8 @@ impl From> for execution_context::ReducerContext { name, caller_identity, caller_connection_id, + call_auth_flags: _, + hosted_auth: _, timestamp, args, }: ReducerOp<'_>, @@ -1887,11 +1956,19 @@ pub struct ProcedureOp { pub name: Identifier, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, + pub call_auth_flags: u32, + pub hosted_auth: Option>, pub timestamp: Timestamp, pub arg_bytes: Bytes, } impl InstanceOp for ProcedureOp { + fn call_auth_flags(&self) -> u32 { + self.call_auth_flags + } + fn hosted_auth(&self) -> Option<&VerifiedHostedAuth> { + self.hosted_auth.as_deref() + } fn name(&self) -> &Identifier { &self.name } diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 23113f1ab6c..82daff1bb20 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -1544,6 +1544,23 @@ impl WasmInstanceEnv { }) } + /// Returns host-verified invocation flags. Bit 0 is internal authority. + /// This does not read tables and is available outside transactions. + pub fn get_call_auth_flags(caller: Caller<'_, Self>) -> u32 { + caller.data().instance_env.get_call_auth_flags() + } + + pub(crate) fn set_hosted_auth( + &mut self, + auth: Option>, + ) { + self.instance_env.set_hosted_auth(auth); + } + + pub(crate) fn set_call_auth_flags(&mut self, flags: u32) { + self.instance_env.set_call_auth_flags(flags); + } + /// Writes the identity of the module into `out = out_ptr[..32]`. /// /// # Traps diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index c89131fa08b..bb11b15e191 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -55,7 +55,7 @@ impl WasmtimeModule { WasmtimeModule { module } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 5); + pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 6); pub(super) fn link_imports(linker: &mut Linker) -> anyhow::Result<()> { link_imports(linker, AsyncImportMode::SyncStub) @@ -635,6 +635,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { store .data_mut() .start_funcall(reducer_name, args_bytes, op.timestamp, op.call_type()); + store.data_mut().set_call_auth_flags(op.call_auth_flags); + store.data_mut().set_hosted_auth(op.hosted_auth.clone()); let call_result = call_sync_typed_func( &self.call_reducer, @@ -758,6 +760,8 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { store .data_mut() .start_funcall(op.name.clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); + store.data_mut().set_call_auth_flags(op.call_auth_flags); + store.data_mut().set_hosted_auth(op.hosted_auth.clone()); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 2ecf80ad11a..175de2fd9a0 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1,3 +1,4 @@ +use crate::auth::invocation::{check_hosted_admission, SqlCallAuth}; use std::sync::Arc; use std::time::Duration; @@ -18,6 +19,7 @@ use anyhow::anyhow; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_expr::statement::Statement; +#[cfg(test)] use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_lib::Timestamp; @@ -52,11 +54,15 @@ pub struct SqlResult { pub async fn run( db: Arc, sql_text: String, - auth: AuthCtx, + auth: impl Into, subs: Option, module: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result { + let auth = auth.into(); + if auth.hosted.is_some() && module.is_none() { + return Err(anyhow!("hosted SQL requires a module with hosted authentication capability").into()); + } match module { Some(module) => module.call_sql(db, sql_text, auth, subs, head).await, None => run_inner::(None, db, sql_text, auth, subs, head).map(|x| x.0), @@ -70,7 +76,7 @@ pub(crate) fn run_with_instance( instance: &mut RefInstance, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result<(SqlResult, bool), DBError> { @@ -81,14 +87,23 @@ fn run_inner( instance: Option<&mut RefInstance>, db: Arc, sql_text: String, - auth: AuthCtx, + auth: SqlCallAuth, subs: Option, head: &mut Vec<(RawIdentifier, AlgebraicType)>, ) -> Result<(SqlResult, bool), DBError> { // We parse the sql statement in a mutable transaction. // If it turns out to be a query, we downgrade the tx. let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { - compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth) + check_hosted_admission(tx, db.database_identity(), auth.hosted.as_deref())?; + let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; + if let Statement::DML(dml) = &stmt { + if spacetimedb_datastore::system_tables::is_host_managed_deployment_table(dml.table_id()) { + return Err(anyhow!( + "Deployment and container authorization metadata may only be changed by the host" + )); + } + } + Ok(stmt) })?; let mut metrics = ExecutionMetrics::default(); @@ -387,6 +402,25 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn owner_sql_cannot_rewrite_container_authority_or_deployment() -> ResultTest<()> { + let db = TestDB::in_memory()?; + for table in [ + "st_deployment", + "st_publish_fence", + "st_deployment_operation", + "st_container_fence", + "st_connection_auth", + ] { + // This uses the owner test context. Permission to mutate application + // tables does not grant permission to replace host operational state. + let error = run_for_testing(&db, &format!("DELETE FROM {table}")).unwrap_err(); + assert!(error.to_string().contains("may only be changed by the host"), "{error}"); + assert!(run_for_testing(&db, &format!("SELECT * FROM {table}"))?.is_empty()); + } + Ok(()) + } + #[test] fn test_limit() -> ResultTest<()> { let (db, _) = create_data(5)?; diff --git a/crates/core/src/subscription/module_subscription_actor.rs b/crates/core/src/subscription/module_subscription_actor.rs index d748fdd09ab..1603a04c77b 100644 --- a/crates/core/src/subscription/module_subscription_actor.rs +++ b/crates/core/src/subscription/module_subscription_actor.rs @@ -7,6 +7,7 @@ use super::module_subscription_manager::{ use super::query::compile_query_with_hashes; use super::tx::DeltaTx; use super::{collect_table_update, TableUpdateType}; +use crate::auth::invocation::check_hosted_admission; use crate::client::messages::{ ProcedureResultMessage, SerializableMessage, SubscriptionData, SubscriptionError, SubscriptionMessage, SubscriptionResult, SubscriptionRows, SubscriptionUpdateMessage, TransactionUpdateMessage, @@ -32,6 +33,7 @@ use spacetimedb_data_structures::map::{HashCollectionExt as _, HashSet}; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::{Workload, WorkloadType}; use spacetimedb_datastore::locking_tx_datastore::datastore::TxMetrics; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, TxId}; use spacetimedb_datastore::traits::{IsolationLevel, TxData}; use spacetimedb_durability::TxOffset; @@ -66,6 +68,7 @@ pub struct ModuleSubscriptions { stats: Arc, metrics: Arc, module_def_version: Arc, + hosted_connections: Arc>>>, } #[derive(Debug, Clone)] @@ -305,6 +308,7 @@ impl ModuleSubscriptions { Self { relational_db, subscriptions, + hosted_connections: Arc::new(RwLock::new(Vec::new())), broadcast_queue, stats, metrics, @@ -315,6 +319,52 @@ impl ModuleSubscriptions { } } + /// Register every hosted socket, even if it has no subscriptions. Taking the + /// database transaction before the registry lock serializes with fencing. + pub(crate) fn register_hosted_connection(&self, sender: &Arc) -> anyhow::Result<()> { + self.relational_db.with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, self.relational_db.database_identity(), sender.auth.hosted.as_ref())?; + let mut connections = self.hosted_connections.write(); + connections.retain(|connection| connection.strong_count() != 0); + connections.push(Arc::downgrade(sender)); + Ok(()) + }) + } + + pub(crate) fn unregister_hosted_connection(&self, sender: &std::sync::Weak) { + self.hosted_connections + .write() + .retain(|connection| connection.strong_count() != 0 && !connection.ptr_eq(sender)); + } + + #[cfg(test)] + pub(crate) fn hosted_connection_count(&self) -> usize { + self.hosted_connections.read().len() + } + + /// Call with the transaction containing the new target fence. After its + /// durable commit, await every returned actor's completion before acking the + /// fence barrier. This also cancels socket batches already dequeued for I/O. + pub fn cancel_invalid_hosted_connections(&self, tx: &S) -> Vec { + let mut cancelled = Vec::new(); + self.hosted_connections.write().retain(|connection| { + let Some(connection) = connection.upgrade() else { + return false; + }; + if check_hosted_admission( + tx, + self.relational_db.database_identity(), + connection.auth.hosted.as_ref(), + ) + .is_err() + { + cancelled.push(connection.cancel_hosted_connection()); + } + true + }); + cancelled + } + pub fn set_module_def_version(&self, version: RawModuleDefVersion) { self.module_def_version .store(Self::encode_module_def_version(version), Ordering::Release); @@ -328,13 +378,16 @@ impl ModuleSubscriptions { match version { RawModuleDefVersion::V9OrEarlier => 0, RawModuleDefVersion::V10 => 1, + RawModuleDefVersion::V11 => 2, } } fn decode_module_def_version(version: u8) -> RawModuleDefVersion { match version { 1 => RawModuleDefVersion::V10, - _ => RawModuleDefVersion::V9OrEarlier, + 2 => RawModuleDefVersion::V11, + 0 => RawModuleDefVersion::V9OrEarlier, + _ => unreachable!("invalid stored module definition version"), } } @@ -635,6 +688,11 @@ impl ModuleSubscriptions { let hash_with_param = QueryHash::from_string(&sql, auth.caller(), true); let (mut_tx, _) = self.begin_mut_tx(Workload::Subscribe); + check_hosted_admission( + &*mut_tx, + self.relational_db.database_identity(), + sender.auth.hosted.as_ref(), + )?; let existing_query = { let guard = self.subscriptions.read(); @@ -737,6 +795,12 @@ impl ModuleSubscriptions { ) }; + let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission( + &*mut_tx, + self.relational_db.database_identity(), + sender.auth.hosted.as_ref(), + )?; let mut subscriptions = self.subscriptions.write(); let queries = return_on_err!( @@ -753,7 +817,8 @@ impl ModuleSubscriptions { return Ok(None); }; - let (mut tx, tx_offset) = self.unsubscribe_views(query, auth.caller())?; + let mut_tx = ScopeGuard::::into_inner(mut_tx); + let (mut tx, tx_offset) = self.unsubscribe_views_and_downgrade_tx(mut_tx, query, auth.caller())?; let (table_rows, metrics) = return_on_err_with_sql!( self.evaluate_initial_subscription(sender.clone(), query.clone(), &tx, &auth, TableUpdateType::Unsubscribe), @@ -816,6 +881,11 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission( + &*mut_tx, + self.relational_db.database_identity(), + sender.auth.hosted.as_ref(), + )?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -933,6 +1003,11 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); + check_hosted_admission( + &*mut_tx, + self.relational_db.database_identity(), + sender.auth.hosted.as_ref(), + )?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -1013,7 +1088,7 @@ impl ModuleSubscriptions { #[allow(clippy::type_complexity)] fn compile_queries( &self, - sender: Identity, + sender: &ClientConnectionSender, auth: AuthCtx, queries: &[Box], num_queries: usize, @@ -1029,13 +1104,18 @@ impl ModuleSubscriptions { subscribe_to_all_tables = true; continue; } - let hash = QueryHash::from_string(sql, sender, false); - let hash_with_param = QueryHash::from_string(sql, sender, true); + let hash = QueryHash::from_string(sql, sender.id.identity, false); + let hash_with_param = QueryHash::from_string(sql, sender.id.identity, true); query_hashes.push((sql, hash, hash_with_param)); } // We always get the db lock before the subscription lock to avoid deadlocks. let (mut_tx, _tx_offset) = self.begin_mut_tx(Workload::Subscribe); + check_hosted_admission( + &*mut_tx, + self.relational_db.database_identity(), + sender.auth.hosted.as_ref(), + )?; let compile_timer = metrics.compilation_time.start_timer(); @@ -1276,13 +1356,7 @@ impl ModuleSubscriptions { subscription_metrics.num_queries_subscribed.inc_by(num_queries as _); let (queries, auth, mut_tx, _compile_timer) = return_on_err!( - self.compile_queries( - sender.id.identity, - auth, - &request.query_strings, - num_queries, - subscription_metrics - ), + self.compile_queries(&sender, auth, &request.query_strings, num_queries, subscription_metrics), send_err_msg, (None, false) ); @@ -1367,13 +1441,7 @@ impl ModuleSubscriptions { subscription_metrics.num_queries_subscribed.inc_by(num_queries as _); let (queries, auth, mut_tx, compile_timer) = return_on_err!( - self.compile_queries( - sender.id.identity, - auth, - &request.query_strings, - num_queries, - subscription_metrics - ), + self.compile_queries(&sender, auth, &request.query_strings, num_queries, subscription_metrics), send_err_msg, (None, false) ); @@ -1522,7 +1590,7 @@ impl ModuleSubscriptions { subscription_metrics.num_queries_subscribed.inc_by(num_queries as _); let (queries, auth, mut_tx, compile_timer) = self.compile_queries( - sender.id.identity, + &sender, auth, &subscription.query_strings, num_queries, diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index 03c3392942f..3a053a2d020 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -1980,7 +1980,7 @@ impl SendWorker { } // Send all the other updates. - let hide_reducer_info_for_non_callers = matches!(module_def_version, RawModuleDefVersion::V10); + let hide_reducer_info_for_non_callers = !matches!(module_def_version, RawModuleDefVersion::V9OrEarlier); for (id, update) in client_id_updates.drain() { let database_update = SubscriptionUpdateMessage::from_event_and_update(&event, update); diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index c479be085d9..b31deca8c68 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -309,6 +309,9 @@ impl CommittedState { self.create_table(ST_TABLE_ACCESSOR_ID, schemas[ST_TABLE_ACCESSOR_IDX].clone()); self.create_table(ST_INDEX_ACCESSOR_ID, schemas[ST_INDEX_ACCESSOR_IDX].clone()); self.create_table(ST_COLUMN_ACCESSOR_ID, schemas[ST_COLUMN_ACCESSOR_IDX].clone()); + for schema in crate::system_tables::deployment_system_schemas() { + self.create_table(schema.table_id, schema.into()); + } // Insert the sequences into `st_sequences` let (st_sequences, blob_store, pool) = diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index e9d67103b16..bbd4b6a63d9 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1006,6 +1006,10 @@ pub(crate) mod tests { ST_VIEW_ARG_NAME, ST_VIEW_COLUMN_ID, ST_VIEW_COLUMN_NAME, ST_VIEW_ID, ST_VIEW_NAME, ST_VIEW_PARAM_ID, ST_VIEW_PARAM_NAME, ST_VIEW_SUB_ID, ST_VIEW_SUB_NAME, }; + use crate::system_tables::{ + ST_CONNECTION_AUTH_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, + ST_PUBLISH_FENCE_ID, + }; use crate::traits::{IsolationLevel, MutTx}; use crate::Result; use core::{fmt, mem}; @@ -1472,6 +1476,12 @@ pub(crate) mod tests { TableRow { id: ST_TABLE_ACCESSOR_ID.into(), name: ST_TABLE_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_INDEX_ACCESSOR_ID.into(), name: ST_INDEX_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, TableRow { id: ST_COLUMN_ACCESSOR_ID.into(), name: ST_COLUMN_ACCESSOR_NAME, ty: StTableType::System, access: StAccess::Public, primary_key: None }, + TableRow { id: ST_ENV_ID.into(), name: "st_env", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_DEPLOYMENT_ID.into(), name: "st_deployment", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_PUBLISH_FENCE_ID.into(), name: "st_publish_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_DEPLOYMENT_OPERATION_ID.into(), name: "st_deployment_operation", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONTAINER_FENCE_ID.into(), name: "st_container_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONNECTION_AUTH_ID.into(), name: "st_connection_auth", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] @@ -1569,6 +1579,28 @@ pub(crate) mod tests { ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 0, name: "table_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 1, name: "col_name", ty: AlgebraicType::String }, ColRow { table: ST_COLUMN_ACCESSOR_ID.into(), pos: 2, name: "accessor_name", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 0, name: "key", ty: AlgebraicType::String }, + ColRow { table: ST_ENV_ID.into(), pos: 1, name: "value", ty: AlgebraicType::String }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 0, name: "key", ty: AlgebraicType::U8 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 1, name: "revision", ty: AlgebraicType::U256 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 2, name: "last_operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_ID.into(), pos: 3, name: "payload", ty: AlgebraicType::bytes() }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 0, name: "key", ty: AlgebraicType::U8 }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 1, name: "publication_epoch", ty: AlgebraicType::U64 }, + ColRow { table: ST_PUBLISH_FENCE_ID.into(), pos: 2, name: "operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 0, name: "operation_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 1, name: "previous_revision", ty: AlgebraicType::option(AlgebraicType::U256) }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 2, name: "committed_revision", ty: AlgebraicType::U256 }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 3, name: "commit_result", ty: AlgebraicType::bytes() }, + ColRow { table: ST_DEPLOYMENT_OPERATION_ID.into(), pos: 4, name: "expires_at", ty: AlgebraicType::I64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 0, name: "source_identity", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 1, name: "generation", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 2, name: "target_grant_revision", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 3, name: "target_set_hash", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONTAINER_FENCE_ID.into(), pos: 4, name: "allowed", ty: AlgebraicType::Bool }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 0, name: "connection_id", ty: AlgebraicType::U128 }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 1, name: "sender_identity", ty: AlgebraicType::U256 }, + ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 2, name: "call_auth_flags", ty: AlgebraicType::U32 }, ])); #[rustfmt::skip] assert_eq!(query.scan_st_indexes()?, map_array([ @@ -1601,6 +1633,12 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col(0), name: "st_env_key_idx_btree", }, + IndexRow { id: 31, table: ST_DEPLOYMENT_ID.into(), col: col(0), name: "st_deployment_key_idx_btree", }, + IndexRow { id: 32, table: ST_PUBLISH_FENCE_ID.into(), col: col(0), name: "st_publish_fence_key_idx_btree", }, + IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col(0), name: "st_deployment_operation_operation_id_idx_btree", }, + IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col(0), name: "st_container_fence_source_identity_idx_btree", }, + IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col(0), name: "st_connection_auth_connection_id_idx_btree", }, ])); let start = ST_RESERVED_SEQUENCE_RANGE as i128 + 1; #[rustfmt::skip] @@ -1646,6 +1684,12 @@ pub(crate) mod tests { ConstraintRow { constraint_id: 23, table_id: ST_INDEX_ACCESSOR_ID.into(), unique_columns: col(1), constraint_name: "st_index_accessor_accessor_name_key", }, ConstraintRow { constraint_id: 24, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 1], constraint_name: "st_column_accessor_table_name_col_name_key", }, ConstraintRow { constraint_id: 25, table_id: ST_COLUMN_ACCESSOR_ID.into(), unique_columns: col_list![0, 2], constraint_name: "st_column_accessor_table_name_accessor_name_key", }, + ConstraintRow { constraint_id: 26, table_id: ST_ENV_ID.into(), unique_columns: col(0), constraint_name: "st_env_key_key", }, + ConstraintRow { constraint_id: 27, table_id: ST_DEPLOYMENT_ID.into(), unique_columns: col(0), constraint_name: "st_deployment_key_key", }, + ConstraintRow { constraint_id: 28, table_id: ST_PUBLISH_FENCE_ID.into(), unique_columns: col(0), constraint_name: "st_publish_fence_key_key", }, + ConstraintRow { constraint_id: 29, table_id: ST_DEPLOYMENT_OPERATION_ID.into(), unique_columns: col(0), constraint_name: "st_deployment_operation_operation_id_key", }, + ConstraintRow { constraint_id: 30, table_id: ST_CONTAINER_FENCE_ID.into(), unique_columns: col(0), constraint_name: "st_container_fence_source_identity_key", }, + ConstraintRow { constraint_id: 31, table_id: ST_CONNECTION_AUTH_ID.into(), unique_columns: col(0), constraint_name: "st_connection_auth_connection_id_key", }, ])); // Verify we get back the tables correctly with the proper ids... @@ -2079,6 +2123,12 @@ pub(crate) mod tests { IndexRow { id: 27, table: ST_INDEX_ACCESSOR_ID.into(), col: col(1), name: "st_index_accessor_accessor_name_idx_btree", }, IndexRow { id: 28, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 1], name: "st_column_accessor_table_name_col_name_idx_btree", }, IndexRow { id: 29, table: ST_COLUMN_ACCESSOR_ID.into(), col: col_list![0, 2], name: "st_column_accessor_table_name_accessor_name_idx_btree", }, + IndexRow { id: 30, table: ST_ENV_ID.into(), col: col(0), name: "st_env_key_idx_btree", }, + IndexRow { id: 31, table: ST_DEPLOYMENT_ID.into(), col: col(0), name: "st_deployment_key_idx_btree", }, + IndexRow { id: 32, table: ST_PUBLISH_FENCE_ID.into(), col: col(0), name: "st_publish_fence_key_idx_btree", }, + IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col(0), name: "st_deployment_operation_operation_id_idx_btree", }, + IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col(0), name: "st_container_fence_source_identity_idx_btree", }, + IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col(0), name: "st_connection_auth_connection_id_idx_btree", }, IndexRow { id: seq_start, table: FIRST_NON_SYSTEM_ID, col: col(0), name: "Foo_id_idx_btree", }, IndexRow { id: seq_start + 1, table: FIRST_NON_SYSTEM_ID, col: col(1), name: "Foo_name_idx_btree", }, IndexRow { id: seq_start + 2, table: FIRST_NON_SYSTEM_ID, col: col(2), name: "Foo_age_idx_btree", }, diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 9008bc6da4e..b946d3667ac 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -11,9 +11,10 @@ use super::{ use crate::{ error::ViewError, system_tables::{ - system_tables, ConnectionIdViaU128, IdentityViaU256, StConnectionCredentialsFields, StConnectionCredentialsRow, - StViewColumnFields, StViewFields, StViewParamFields, StViewParamRow, StViewSubFields, StViewSubRow, - ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, ST_VIEW_ID, ST_VIEW_PARAM_ID, ST_VIEW_SUB_ID, + system_tables, ConnectionIdViaU128, IdentityViaU256, StConnectionAuthFields, StConnectionCredentialsFields, + StConnectionCredentialsRow, StViewColumnFields, StViewFields, StViewParamFields, StViewParamRow, + StViewSubFields, StViewSubRow, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, ST_VIEW_COLUMN_ID, + ST_VIEW_ID, ST_VIEW_PARAM_ID, ST_VIEW_SUB_ID, }, }; use crate::{ @@ -2676,7 +2677,13 @@ impl MutTxId { ); } } - self.delete_st_client_credentials(database_identity, connection_id) + self.delete_st_client_credentials(database_identity, connection_id)?; + self.delete_col_eq( + ST_CONNECTION_AUTH_ID, + StConnectionAuthFields::ConnectionId.col_id(), + &ConnectionIdViaU128::from(connection_id).into(), + )?; + Ok(()) } /// Look up a client row by identity and connection ID in the `st_clients` system table. diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index e75cc76b365..0f30b180294 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -175,7 +175,7 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result false, + ST_CONNECTION_CREDENTIALS_ID | ST_CONNECTION_AUTH_ID => false, // We don't define any system views, so none of the view-related tables can be system meta-descriptors. ST_VIEW_ID | ST_VIEW_PARAM_ID | ST_VIEW_COLUMN_ID | ST_VIEW_SUB_ID | ST_VIEW_ARG_ID => false, ST_EVENT_TABLE_ID => { @@ -205,7 +205,9 @@ pub enum SystemTable { st_table_accessor, } -pub fn system_tables() -> [TableSchema; 20] { +pub fn system_tables() -> [TableSchema; 26] { + let [env, deployment, publish_fence, deployment_operation, container_fence, connection_auth] = + deployment_system_schemas(); [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. st_table_schema(), @@ -228,6 +230,12 @@ pub fn system_tables() -> [TableSchema; 20] { st_table_accessor_schema(), st_index_accessor_schema(), st_column_accessor_schema(), + env, + deployment, + publish_fence, + deployment_operation, + container_fence, + connection_auth, ] } @@ -311,6 +319,9 @@ macro_rules! st_fields_enum { } } +mod deployment; +pub use deployment::*; + // WARNING: For a stable schema, don't change the field names and discriminants. st_fields_enum!(enum StTableFields { "table_id", TableId = 0, @@ -668,6 +679,8 @@ fn system_module_def() -> ModuleDef { .with_unique_constraint(st_column_accessor_table_alias_cols) .with_index_no_accessor_name(btree(st_column_accessor_table_alias_cols)); + deployment::register_tables(&mut builder); + let result = builder .finish() .try_into() @@ -693,6 +706,7 @@ fn system_module_def() -> ModuleDef { validate_system_table::(&result, ST_TABLE_ACCESSOR_NAME); validate_system_table::(&result, ST_INDEX_ACCESSOR_NAME); validate_system_table::(&result, ST_COLUMN_ACCESSOR_NAME); + deployment::validate_tables(&result); result } @@ -741,6 +755,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_key", ConstraintId(23)); m.insert("st_column_accessor_table_name_col_name_key", ConstraintId(24)); m.insert("st_column_accessor_table_name_accessor_name_key", ConstraintId(25)); + m.extend(deployment::CONSTRAINTS); m }; } @@ -779,6 +794,7 @@ lazy_static::lazy_static! { m.insert("st_index_accessor_accessor_name_idx_btree", IndexId(27)); m.insert("st_column_accessor_table_name_col_name_idx_btree", IndexId(28)); m.insert("st_column_accessor_table_name_accessor_name_idx_btree", IndexId(29)); + m.extend(deployment::INDEXES); m }; } @@ -968,7 +984,7 @@ pub(crate) fn system_table_schema(table_id: TableId) -> Option { ST_TABLE_ACCESSOR_ID => Some(st_table_accessor_schema()), ST_INDEX_ACCESSOR_ID => Some(st_index_accessor_schema()), ST_COLUMN_ACCESSOR_ID => Some(st_column_accessor_schema()), - _ => None, + table => deployment::system_schema(table), } } diff --git a/crates/datastore/src/system_tables/deployment.rs b/crates/datastore/src/system_tables/deployment.rs new file mode 100644 index 00000000000..a10c64ab0c3 --- /dev/null +++ b/crates/datastore/src/system_tables/deployment.rs @@ -0,0 +1,236 @@ +//! Stable system schemas for environment and container deployment state. +//! +//! The deployment payload and operation result have versioned binary encodings, +//! so adding a protocol version does not change the system table row layout. +//! Publication and target fences are operational metadata: application restore +//! must preserve/reconcile their current authority before admitting execution. + +use super::*; + +pub const ST_ENV_ID: TableId = TableId(21); +pub const ST_DEPLOYMENT_ID: TableId = TableId(22); +pub const ST_PUBLISH_FENCE_ID: TableId = TableId(23); +pub const ST_DEPLOYMENT_OPERATION_ID: TableId = TableId(24); +pub const ST_CONTAINER_FENCE_ID: TableId = TableId(25); +pub const ST_CONNECTION_AUTH_ID: TableId = TableId(26); + +pub const ST_ENV_NAME: &str = "st_env"; +pub const ST_DEPLOYMENT_NAME: &str = "st_deployment"; +pub const ST_PUBLISH_FENCE_NAME: &str = "st_publish_fence"; +pub const ST_DEPLOYMENT_OPERATION_NAME: &str = "st_deployment_operation"; +pub const ST_CONTAINER_FENCE_NAME: &str = "st_container_fence"; +pub const ST_CONNECTION_AUTH_NAME: &str = "st_connection_auth"; + +st_fields_enum!(enum StEnvFields { + "key", Key = 0, + "value", Value = 1, +}); +st_fields_enum!(enum StDeploymentFields { + "key", Key = 0, + "revision", Revision = 1, + "last_operation_id", LastOperationId = 2, + "payload", Payload = 3, +}); +st_fields_enum!(enum StPublishFenceFields { + "key", Key = 0, + "publication_epoch", PublicationEpoch = 1, + "operation_id", OperationId = 2, +}); +st_fields_enum!(enum StDeploymentOperationFields { + "operation_id", OperationId = 0, + "previous_revision", PreviousRevision = 1, + "committed_revision", CommittedRevision = 2, + "commit_result", CommitResult = 3, + "expires_at", ExpiresAt = 4, +}); +st_fields_enum!(enum StContainerFenceFields { + "source_identity", SourceIdentity = 0, + "generation", Generation = 1, + "target_grant_revision", TargetGrantRevision = 2, + "target_set_hash", TargetSetHash = 3, + "allowed", Allowed = 4, +}); +st_fields_enum!(enum StConnectionAuthFields { + "connection_id", ConnectionId = 0, + "sender_identity", SenderIdentity = 1, + "call_auth_flags", CallAuthFlags = 2, +}); + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StEnvRow { + pub key: String, + pub value: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDeploymentRow { + pub key: u8, + pub revision: Hash, + pub last_operation_id: u128, + pub payload: Box<[u8]>, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StPublishFenceRow { + pub key: u8, + pub publication_epoch: u64, + pub operation_id: u128, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StDeploymentOperationRow { + pub operation_id: u128, + pub previous_revision: Option, + pub committed_revision: Hash, + pub commit_result: Box<[u8]>, + pub expires_at: TimestampViaI64, +} + +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StContainerFenceRow { + pub source_identity: IdentityViaU256, + pub generation: u64, + pub target_grant_revision: u64, + pub target_set_hash: Hash, + pub allowed: bool, +} + +/// Captured host authentication for lifecycle cleanup, including crash recovery. +/// Only hosted connections need a row; absent rows retain ordinary flags zero. +/// JWT claims and sender equality never reconstruct these flags. +#[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StConnectionAuthRow { + pub connection_id: ConnectionIdViaU128, + pub sender_identity: IdentityViaU256, + pub call_auth_flags: u32, +} + +macro_rules! row_conversions { + ($($row:ty),+ $(,)?) => {$ ( + impl TryFrom> for $row { + type Error = DatastoreError; + fn try_from(row: RowRef<'_>) -> Result { + read_via_bsatn(row) + } + } + impl From<$row> for ProductValue { + fn from(row: $row) -> Self { to_product_value(&row) } + } + )+}; +} + +row_conversions!( + StEnvRow, + StDeploymentRow, + StPublishFenceRow, + StDeploymentOperationRow, + StContainerFenceRow, + StConnectionAuthRow +); + +pub(super) fn register_tables(builder: &mut RawModuleDefV9Builder) { + fn register(builder: &mut RawModuleDefV9Builder, name: &'static str) { + let ty = builder.add_type::(); + builder + .build_table(name, *ty.as_ref().expect("system row must be a product")) + .with_type(TableType::System) + .with_access(v9::TableAccess::Private) + .with_primary_key(ColId(0)) + .with_unique_constraint(ColId(0)) + .with_index_no_accessor_name(btree(ColId(0))); + } + register::(builder, ST_ENV_NAME); + register::(builder, ST_DEPLOYMENT_NAME); + register::(builder, ST_PUBLISH_FENCE_NAME); + register::(builder, ST_DEPLOYMENT_OPERATION_NAME); + register::(builder, ST_CONTAINER_FENCE_NAME); + register::(builder, ST_CONNECTION_AUTH_NAME); +} + +pub(super) fn validate_tables(def: &ModuleDef) { + validate_system_table::(def, ST_ENV_NAME); + validate_system_table::(def, ST_DEPLOYMENT_NAME); + validate_system_table::(def, ST_PUBLISH_FENCE_NAME); + validate_system_table::(def, ST_DEPLOYMENT_OPERATION_NAME); + validate_system_table::(def, ST_CONTAINER_FENCE_NAME); + validate_system_table::(def, ST_CONNECTION_AUTH_NAME); +} + +pub(crate) fn deployment_system_schemas() -> [TableSchema; 6] { + [ + st_schema(ST_ENV_NAME, ST_ENV_ID), + st_schema(ST_DEPLOYMENT_NAME, ST_DEPLOYMENT_ID), + st_schema(ST_PUBLISH_FENCE_NAME, ST_PUBLISH_FENCE_ID), + st_schema(ST_DEPLOYMENT_OPERATION_NAME, ST_DEPLOYMENT_OPERATION_ID), + st_schema(ST_CONTAINER_FENCE_NAME, ST_CONTAINER_FENCE_ID), + st_schema(ST_CONNECTION_AUTH_NAME, ST_CONNECTION_AUTH_ID), + ] +} + +pub(super) fn system_schema(table: TableId) -> Option { + let name = match table { + ST_ENV_ID => ST_ENV_NAME, + ST_DEPLOYMENT_ID => ST_DEPLOYMENT_NAME, + ST_PUBLISH_FENCE_ID => ST_PUBLISH_FENCE_NAME, + ST_DEPLOYMENT_OPERATION_ID => ST_DEPLOYMENT_OPERATION_NAME, + ST_CONTAINER_FENCE_ID => ST_CONTAINER_FENCE_NAME, + ST_CONNECTION_AUTH_ID => ST_CONNECTION_AUTH_NAME, + _ => return None, + }; + Some(st_schema(name, table)) +} + +/// These tables are read through dedicated host operations by module code. +/// In particular, resolving a numeric table or index ID must not bypass this. +pub fn is_module_restricted_table(table: TableId) -> bool { + matches!( + table, + ST_ENV_ID + | ST_DEPLOYMENT_ID + | ST_PUBLISH_FENCE_ID + | ST_DEPLOYMENT_OPERATION_ID + | ST_CONTAINER_FENCE_ID + | ST_CONNECTION_AUTH_ID + ) +} + +pub fn is_module_restricted_index(index: IndexId) -> bool { + INDEXES.iter().any(|(_, restricted)| *restricted == index) +} + +/// Environment management has its own validated SQL path. Deployment and +/// authorization metadata may only be changed by authenticated host operations. +pub fn is_host_managed_deployment_table(table: TableId) -> bool { + matches!( + table, + ST_DEPLOYMENT_ID + | ST_PUBLISH_FENCE_ID + | ST_DEPLOYMENT_OPERATION_ID + | ST_CONTAINER_FENCE_ID + | ST_CONNECTION_AUTH_ID + ) +} + +pub(super) const CONSTRAINTS: [(&str, ConstraintId); 6] = [ + ("st_env_key_key", ConstraintId(26)), + ("st_deployment_key_key", ConstraintId(27)), + ("st_publish_fence_key_key", ConstraintId(28)), + ("st_deployment_operation_operation_id_key", ConstraintId(29)), + ("st_container_fence_source_identity_key", ConstraintId(30)), + ("st_connection_auth_connection_id_key", ConstraintId(31)), +]; + +pub(super) const INDEXES: [(&str, IndexId); 6] = [ + ("st_env_key_idx_btree", IndexId(30)), + ("st_deployment_key_idx_btree", IndexId(31)), + ("st_publish_fence_key_idx_btree", IndexId(32)), + ("st_deployment_operation_operation_id_idx_btree", IndexId(33)), + ("st_container_fence_source_identity_idx_btree", IndexId(34)), + ("st_connection_auth_connection_id_idx_btree", IndexId(35)), +]; diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs new file mode 100644 index 00000000000..885d64abca0 --- /dev/null +++ b/crates/lib/src/container.rs @@ -0,0 +1,469 @@ +//! Shared, versioned container deployment data. +//! +//! A decoded declaration is untrusted. Publication must normalize and validate +//! the complete effective deployment before hashing or admitting it. Runtime +//! capabilities and database authorization are additional admission checks. + +use crate::{bsatn, hash_bytes, Hash, SpacetimeType}; +use std::{collections::BTreeSet, fmt, str::FromStr}; + +/// Version of the normalized deployment encoding, independent of module ABI. +pub const CONTAINER_SPEC_VERSION: u32 = 1; +pub const MAX_ARGV_ENTRIES: usize = 256; +pub const MAX_ENV_KEYS: usize = 256; +pub const MAX_PORTS: usize = 16; +pub const MAX_EXEC_STRING_BYTES: usize = 32 * 1024; +/// Includes NUL terminators, 64-bit pointer arrays, and reserved startup space. +pub const MAX_EXEC_BYTES: usize = 128 * 1024; +pub const EXEC_RESERVED_BYTES: usize = 4096; +pub const DEFAULT_STOP_GRACE_MS: u32 = 30_000; +pub const MAX_STOP_GRACE_MS: u32 = 120_000; + +/// The digest of an OCI object. This is never a SpacetimeDB BLAKE3 object key. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, SpacetimeType)] +#[sats(crate = crate)] +pub enum OciDigest { + Sha256(Hash), +} + +impl OciDigest { + pub const fn sha256(bytes: [u8; 32]) -> Self { + Self::Sha256(Hash::from_byte_array(bytes)) + } + + pub fn as_bytes(&self) -> &[u8; 32] { + match self { + Self::Sha256(hash) => &hash.data, + } + } +} + +impl fmt::Display for OciDigest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Sha256(bytes) => write!(f, "sha256:{}", bytes.to_hex()), + } + } +} + +impl FromStr for OciDigest { + type Err = ContainerValidationError; + + fn from_str(value: &str) -> Result { + let hex = value + .strip_prefix("sha256:") + .ok_or_else(|| invalid("image_manifest", "only sha256 OCI digests are supported"))?; + if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(invalid( + "image_manifest", + "expected 64 lowercase hexadecimal digest characters", + )); + } + let mut bytes = [0; 32]; + hex::decode_to_slice(hex, &mut bytes).map_err(|_| invalid("image_manifest", "invalid SHA-256 digest"))?; + Ok(Self::sha256(bytes)) + } +} + +#[cfg(feature = "serde")] +impl serde::Serialize for OciDigest { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(self) + } +} + +#[cfg(feature = "serde")] +impl<'de> serde::Deserialize<'de> for OciDigest { + fn deserialize>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ImagePlatform { + pub os: String, + pub architecture: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum ContainerMode { + Service, + Job, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum RestartPolicy { + Never, + OnFailure, + Always, +} + +impl RestartPolicy { + /// Only process termination drives this policy. Readiness is independent. + pub fn restarts_after(self, exit_code: Option) -> bool { + match self { + Self::Never => false, + Self::OnFailure => exit_code != Some(0), + Self::Always => true, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerResources { + pub cpu_millicores: u64, + pub memory_bytes: u64, + pub scratch_bytes: u64, + /// Linux tasks, including threads and commands started through exec. + pub pids_max: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum PortProtocol { + Http, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum PortExposure { + Public, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ReadinessProbe { + Tcp(TcpProbe), + Http(HttpProbe), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct TcpProbe { + pub timeout_ms: u32, + pub interval_ms: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct HttpProbe { + pub path: String, + pub timeout_ms: u32, + pub interval_ms: u32, +} + +impl Default for ReadinessProbe { + fn default() -> Self { + Self::Tcp(TcpProbe { + timeout_ms: 1000, + interval_ms: 5000, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerPort { + pub name: String, + pub port: u16, + pub protocol: PortProtocol, + /// Required in the public input, with no implicit exposure default. + pub exposure: PortExposure, + #[cfg_attr(feature = "serde", serde(default))] + pub readiness_probe: ReadinessProbe, +} + +/// A placeholder declaration is never admitted until the mount protocol ships. +/// Keeping explicit declarations lets Stage 1 return an unsupported-feature error. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerMount { + pub database: String, + pub source: String, + pub target: String, + pub read_only: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ContainerSpec { + pub image_manifest: OciDigest, + pub image_platform: ImagePlatform, + /// Effective complete argv after applying OCI Entrypoint/Cmd or override. + pub argv: Vec, + pub user: String, + pub working_directory: String, + pub mode: ContainerMode, + pub restart: RestartPolicy, + pub env_keys: Vec, + pub resources: ContainerResources, + pub ports: Vec, + pub mounts: Vec, + pub stop_grace_ms: u32, +} + +/// Explicit component removal is distinct from omission or an empty replacement. +#[derive(Clone, Debug, Default, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ContainerAction { + #[default] + Keep, + Set(ContainerSpec), + Remove, +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("invalid container {field}: {reason}")] +pub struct ContainerValidationError { + pub field: &'static str, + pub reason: &'static str, +} + +fn invalid(field: &'static str, reason: &'static str) -> ContainerValidationError { + ContainerValidationError { field, reason } +} + +/// Limits are operational configuration; normalized specs are checked again at +/// admission against the selected node's enforceable capacities and capabilities. +#[derive(Clone, Copy, Debug)] +pub struct ContainerSpecLimits { + pub resources: ContainerResources, +} + +impl Default for ContainerSpecLimits { + fn default() -> Self { + Self { + resources: ContainerResources { + cpu_millicores: 64_000, + memory_bytes: 128 * 1024 * 1024 * 1024, + scratch_bytes: 1024 * 1024 * 1024 * 1024, + pids_max: 4096, + }, + } + } +} + +impl ContainerSpec { + /// Sort semantically unordered declarations so equivalent specs hash alike. + /// Duplicates are rejected, never silently deduplicated. + pub fn normalize(mut self, limits: &ContainerSpecLimits) -> Result { + self.validate(limits)?; + self.env_keys.sort_unstable(); + self.ports.sort_unstable_by(|a, b| a.name.cmp(&b.name)); + Ok(self) + } + + pub fn validate(&self, limits: &ContainerSpecLimits) -> Result<(), ContainerValidationError> { + if self.image_platform.os != "linux" || !matches!(self.image_platform.architecture.as_str(), "amd64" | "arm64") + { + return Err(invalid("image_platform", "expected linux/amd64 or linux/arm64")); + } + if !self.mounts.is_empty() { + return Err(invalid("mounts", "SpacetimeFS mounts are not supported by Stage 1")); + } + if self.mode == ContainerMode::Job && self.restart == RestartPolicy::Always { + return Err(invalid("restart", "jobs cannot use the always restart policy")); + } + if self.argv.is_empty() || self.argv.len() > MAX_ARGV_ENTRIES || self.argv[0].is_empty() { + return Err(invalid( + "argv", + "a nonempty command with at most 256 arguments is required", + )); + } + validate_exec_size(&self.argv, &[])?; + if self.user.len() > 255 || self.user.bytes().any(|b| b == 0 || b.is_ascii_control()) { + return Err(invalid( + "user", + "user must fit 255 bytes and contain no control characters", + )); + } + if !self.working_directory.starts_with('/') + || self.working_directory.len() > 4096 + || self.working_directory.contains('\0') + { + return Err(invalid( + "working_directory", + "expected an absolute Linux path of at most 4096 bytes", + )); + } + if self.stop_grace_ms > MAX_STOP_GRACE_MS { + return Err(invalid("stop_grace_ms", "stop grace exceeds the supported deadline")); + } + let r = self.resources; + let max = limits.resources; + if r.cpu_millicores == 0 + || r.cpu_millicores > max.cpu_millicores + || r.memory_bytes == 0 + || r.memory_bytes > max.memory_bytes + || r.scratch_bytes == 0 + || r.scratch_bytes > max.scratch_bytes + || r.pids_max == 0 + || r.pids_max > max.pids_max + { + return Err(invalid( + "resources", + "resource reservations must be positive and within server limits", + )); + } + if self.env_keys.len() > MAX_ENV_KEYS { + return Err(invalid("env_keys", "too many environment keys")); + } + let mut keys = BTreeSet::new(); + for key in &self.env_keys { + validate_env_key(key)?; + if !keys.insert(key) { + return Err(invalid("env_keys", "duplicate environment key")); + } + } + if self.ports.len() > MAX_PORTS { + return Err(invalid("ports", "too many declared ports")); + } + let (mut names, mut numbers) = (BTreeSet::new(), BTreeSet::new()); + for port in &self.ports { + let bytes = port.name.as_bytes(); + if bytes.is_empty() + || bytes.len() > 32 + || !bytes[0].is_ascii_lowercase() + || !bytes + .iter() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || *b == b'-') + { + return Err(invalid("ports.name", "expected [a-z][a-z0-9-]{0,31}")); + } + if port.port == 0 || !names.insert(&port.name) || !numbers.insert(port.port) { + return Err(invalid("ports", "port names and nonzero port numbers must be unique")); + } + let (timeout, interval) = match &port.readiness_probe { + ReadinessProbe::Tcp(TcpProbe { + timeout_ms, + interval_ms, + }) => (*timeout_ms, *interval_ms), + ReadinessProbe::Http(HttpProbe { + path, + timeout_ms, + interval_ms, + }) => { + if !path.starts_with('/') + || path.starts_with("//") + || path.len() > 2048 + || path + .bytes() + .any(|b| b.is_ascii_control() || b == b' ' || b == b'\\' || b == b'#') + { + return Err(invalid( + "readiness_probe.path", + "expected a bounded origin-relative HTTP path", + )); + } + (*timeout_ms, *interval_ms) + } + }; + if timeout == 0 || timeout > 30_000 || interval == 0 || interval > 300_000 || timeout > interval { + return Err(invalid("readiness_probe", "invalid probe timeout or interval")); + } + } + Ok(()) + } + + /// Domain-separated, versioned BSATN encoding, after normalization. + /// This hashes the container spec only; the full deployment also includes + /// the module selection and uses its own revision domain. + pub fn canonical_hash(&self, limits: &ContainerSpecLimits) -> Result { + let normalized = self.clone().normalize(limits)?; + let encoded = bsatn::to_vec(&(CONTAINER_SPEC_VERSION, normalized)) + .expect("encoding an in-memory container specification cannot fail"); + let mut bytes = b"spacetimedb/container-spec\0".to_vec(); + bytes.extend(encoded); + Ok(hash_bytes(&bytes)) + } +} + +pub fn validate_env_key(key: &str) -> Result<(), ContainerValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > 256 + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(invalid("env_keys", "invalid POSIX environment variable name")); + } + if key.starts_with("SPACETIMEDB_") { + return Err(invalid( + "env_keys", + "SPACETIMEDB_ variables are reserved for the platform", + )); + } + Ok(()) +} + +/// Call after merging image environment, the database snapshot, and platform +/// variables. Error messages deliberately contain neither argv nor env values. +pub fn validate_exec_size(argv: &[String], env: &[String]) -> Result<(), ContainerValidationError> { + let count = argv + .len() + .checked_add(env.len()) + .and_then(|n| n.checked_add(2)) + .ok_or_else(|| invalid("exec", "too many arguments or environment entries"))?; + let mut total = count + .checked_mul(8) + .and_then(|n| n.checked_add(EXEC_RESERVED_BYTES)) + .ok_or_else(|| invalid("exec", "argument and environment size overflow"))?; + for value in argv.iter().chain(env) { + if value.contains('\0') || value.len() >= MAX_EXEC_STRING_BYTES { + return Err(invalid( + "exec", + "argument or environment entry contains NUL or exceeds the per-entry limit", + )); + } + total = total + .checked_add(value.len() + 1) + .ok_or_else(|| invalid("exec", "argument and environment size overflow"))?; + } + if total > MAX_EXEC_BYTES { + return Err(invalid( + "exec", + "combined argument and environment size exceeds the startup limit", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/container/tests.rs b/crates/lib/src/container/tests.rs new file mode 100644 index 00000000000..dd7c208f43c --- /dev/null +++ b/crates/lib/src/container/tests.rs @@ -0,0 +1,246 @@ +use super::*; + +fn spec() -> ContainerSpec { + ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec!["API_KEY".into(), "DATABASE_URL".into()], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![ContainerPort { + name: "http".into(), + port: 8080, + protocol: PortProtocol::Http, + exposure: PortExposure::Public, + readiness_probe: ReadinessProbe::default(), + }], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } +} + +#[test] +fn oci_digest_is_strict_and_cannot_be_used_for_path_traversal() { + let digest = format!("sha256:{}", "ab".repeat(32)); + assert_eq!(digest.parse::().unwrap().to_string(), digest); + for value in [ + "sha256:../../host", + "sha256:abc", + "sha512:abc", + "blake3:abc", + "SHA256:abc", + "sha256:", + ] { + assert!(value.parse::().is_err(), "{value}"); + } + assert!(format!("sha256:{}", "AB".repeat(32)).parse::().is_err()); + assert!(format!("sha256:{}\n", "ab".repeat(32)).parse::().is_err()); +} + +#[test] +fn canonical_hash_ignores_declaration_order_but_preserves_launch_semantics() { + let limits = ContainerSpecLimits::default(); + let mut original = spec(); + original.ports.push(ContainerPort { + name: "admin".into(), + port: 8081, + ..original.ports[0].clone() + }); + let hash = original.canonical_hash(&limits).unwrap(); + let mut reordered = original.clone(); + reordered.env_keys.reverse(); + reordered.ports.reverse(); + assert_eq!(reordered.canonical_hash(&limits).unwrap(), hash); + let mut changed = reordered; + changed.argv.push("--read-only".into()); + assert_ne!(changed.canonical_hash(&limits).unwrap(), hash); + assert_ne!( + ContainerSpec { + resources: ContainerResources { + pids_max: 63, + ..original.resources + }, + ..original.clone() + } + .canonical_hash(&limits) + .unwrap(), + hash + ); + let bytes = bsatn::to_vec(&original).unwrap(); + assert_eq!(bsatn::from_slice::(&bytes).unwrap(), original); +} + +#[test] +fn duplicate_declarations_are_not_silently_deduplicated() { + let limits = ContainerSpecLimits::default(); + let mut value = spec(); + value.env_keys.push(value.env_keys[0].clone()); + assert_eq!(value.normalize(&limits).unwrap_err().field, "env_keys"); + let mut value = spec(); + value.ports.push(ContainerPort { + name: "another".into(), + ..value.ports[0].clone() + }); + assert_eq!(value.normalize(&limits).unwrap_err().field, "ports"); + let mut value = spec(); + value.ports.push(ContainerPort { + port: 9090, + ..value.ports[0].clone() + }); + assert_eq!(value.normalize(&limits).unwrap_err().field, "ports"); +} + +#[test] +fn readiness_cannot_change_authority_or_inject_a_request() { + for path in [ + "https://internal/", + "//internal/", + "/\\internal/", + "/health\r\nHost: internal", + "/health#fragment", + ] { + let mut value = spec(); + value.ports[0].readiness_probe = ReadinessProbe::Http(HttpProbe { + path: path.into(), + timeout_ms: 1000, + interval_ms: 5000, + }); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "readiness_probe.path" + ); + } + let mut value = spec(); + value.ports[0].readiness_probe = ReadinessProbe::Http(HttpProbe { + path: "/health?full=1".into(), + timeout_ms: 1000, + interval_ms: 5000, + }); + value.validate(&ContainerSpecLimits::default()).unwrap(); +} + +#[test] +fn stage_one_rejects_mounts_and_unsupported_platforms() { + let mut value = spec(); + value.mounts.push(ContainerMount { + database: "self".into(), + source: "/".into(), + target: "/spacetime".into(), + read_only: false, + }); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "mounts" + ); + let mut value = spec(); + value.image_platform.os = "windows".into(); + assert_eq!( + value.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "image_platform" + ); +} + +#[test] +fn startup_size_counts_environment_and_pointers_without_leaking_values() { + let secret = "THIS_VALUE_MUST_NOT_APPEAR_IN_ERRORS"; + let env = vec![format!("KEY={secret}\0")]; + let error = validate_exec_size(&spec().argv, &env).unwrap_err().to_string(); + assert!(!error.contains(secret)); + assert!(!error.contains("KEY=")); + let bounded_strings = vec!["x".repeat(MAX_EXEC_STRING_BYTES - 1); 5]; + assert!(validate_exec_size(&spec().argv, &bounded_strings).is_err()); + let many_empty_entries = vec![String::new(); MAX_EXEC_BYTES / 8]; + assert!(validate_exec_size(&[], &many_empty_entries).is_err()); + assert!(validate_exec_size(&spec().argv, &["KEY=value".into()]).is_ok()); +} + +#[test] +fn tenant_environment_cannot_override_platform_discovery_or_credentials() { + for key in [ + "SPACETIMEDB_DATABASE_IDENTITY", + "SPACETIMEDB_SERVER_URI", + "SPACETIMEDB_CREDENTIAL_BROKER", + "SPACETIMEDB_TOKEN", + "A=B", + "0BAD", + "BAD\0KEY", + ] { + assert!(validate_env_key(key).is_err(), "{key:?}"); + } + for key in ["API_KEY", "_CUSTOM", "path", "A1"] { + validate_env_key(key).unwrap(); + } +} + +#[test] +fn zero_or_overflowing_resource_requests_are_not_admitted() { + let limits = ContainerSpecLimits::default(); + for cpu in [0, u64::MAX, limits.resources.cpu_millicores + 1] { + let mut value = spec(); + value.resources.cpu_millicores = cpu; + assert_eq!(value.validate(&limits).unwrap_err().field, "resources"); + } + let mut value = spec(); + value.resources.scratch_bytes = 0; + assert!(value.validate(&limits).is_err()); +} + +#[test] +fn successful_jobs_and_on_failure_services_remain_terminal() { + assert!(!RestartPolicy::Never.restarts_after(None)); + assert!(!RestartPolicy::Never.restarts_after(Some(1))); + assert!(!RestartPolicy::OnFailure.restarts_after(Some(0))); + assert!(RestartPolicy::OnFailure.restarts_after(None)); + assert!(RestartPolicy::OnFailure.restarts_after(Some(1))); + let mut job = spec(); + job.mode = ContainerMode::Job; + job.restart = RestartPolicy::Always; + assert_eq!( + job.validate(&ContainerSpecLimits::default()).unwrap_err().field, + "restart" + ); +} + +#[cfg(feature = "serde")] +#[test] +fn json_requires_explicit_port_exposure_and_rejects_privileged_fields() { + let original = spec(); + let json = serde_json::to_value(&original).unwrap(); + assert_eq!(serde_json::from_value::(json.clone()).unwrap(), original); + let mut missing = json.clone(); + missing["ports"][0].as_object_mut().unwrap().remove("exposure"); + assert!(serde_json::from_value::(missing).is_err()); + let mut injected = json; + injected + .as_object_mut() + .unwrap() + .insert("privileged".into(), true.into()); + assert!(serde_json::from_value::(injected).is_err()); + let keep: ContainerAction = serde_json::from_str(r#"{"action":"keep"}"#).unwrap(); + assert_eq!(keep, ContainerAction::Keep); + let remove: ContainerAction = serde_json::from_str(r#"{"action":"remove"}"#).unwrap(); + assert_eq!(remove, ContainerAction::Remove); + assert!(serde_json::from_str::(r#"{"action":"remove","value":{}}"#).is_err()); +} + +#[test] +fn concrete_container_actions_preserve_wire_tags() { + let container = spec(); + assert_eq!(bsatn::to_vec(&ContainerAction::Keep).unwrap(), [0]); + assert_eq!(bsatn::to_vec(&ContainerAction::Remove).unwrap(), [2]); + let mut expected = vec![1]; + expected.extend(bsatn::to_vec(&container).unwrap()); + assert_eq!(bsatn::to_vec(&ContainerAction::Set(container)).unwrap(), expected); +} diff --git a/crates/lib/src/db/raw_def.rs b/crates/lib/src/db/raw_def.rs index a29161403a5..c67f22dacc8 100644 --- a/crates/lib/src/db/raw_def.rs +++ b/crates/lib/src/db/raw_def.rs @@ -16,3 +16,5 @@ pub use v8::*; pub mod v9; pub mod v10; + +pub mod v11; diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 47a4281e86a..8e5a9ecb1e6 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -671,7 +671,7 @@ impl RawModuleDefV10Builder { } /// Get mutable access to the typespace section, creating it if missing. - fn typespace_mut(&mut self) -> &mut Typespace { + pub(super) fn typespace_mut(&mut self) -> &mut Typespace { let idx = self .module .sections @@ -785,7 +785,7 @@ impl RawModuleDefV10Builder { } /// Get mutable access to the types section, creating it if missing. - fn types_mut(&mut self) -> &mut Vec { + pub(super) fn types_mut(&mut self) -> &mut Vec { let idx = self .module .sections diff --git a/crates/lib/src/db/raw_def/v11.rs b/crates/lib/src/db/raw_def/v11.rs new file mode 100644 index 00000000000..ac6cc593291 --- /dev/null +++ b/crates/lib/src/db/raw_def/v11.rs @@ -0,0 +1,269 @@ +//! Version 11 module definitions: explicit function visibility with contextual defaults. +//! +//! Non-function sections retain their V10 wire shapes. V11 is a distinct top-level +//! variant, so hosts unaware of Internal visibility reject the entire definition. + +use super::v10; +use super::v10::*; +use super::v9::Lifecycle; +use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::typespace::TypespaceBuilder; +use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductType, SpacetimeType, Typespace}; +use std::{ + any::TypeId, + collections::BTreeMap, + ops::{Deref, DerefMut}, +}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, SpacetimeType)] +#[sats(crate = crate)] +pub enum FunctionVisibilityV11 { + Private, + ClientCallable, + Internal, +} + +pub use FunctionVisibilityV11 as FunctionVisibility; + +#[derive(Default, Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawModuleDefV11 { + pub sections: Vec, +} + +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +#[non_exhaustive] +pub enum RawModuleDefV11Section { + Typespace(Typespace), + Types(Vec), + Tables(Vec), + Reducers(Vec), + Procedures(Vec), + Views(Vec), + Schedules(Vec), + LifeCycleReducers(Vec), + RowLevelSecurity(Vec), + CaseConversionPolicy(CaseConversionPolicy), + ExplicitNames(ExplicitNames), + HttpHandlers(Vec), + HttpRoutes(Vec), + /// Module bindings capabilities, independent of function visibility. + Capabilities(Vec), +} + +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawReducerDefV11 { + pub source_name: RawIdentifier, + pub params: ProductType, + /// None selects the context default; Some preserves the author's selection. + pub declared_visibility: Option, + pub ok_return_type: AlgebraicType, + pub err_return_type: AlgebraicType, +} + +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawProcedureDefV11 { + pub source_name: RawIdentifier, + pub params: ProductType, + /// None selects the context default; Some preserves the author's selection. + pub declared_visibility: Option, + pub return_type: AlgebraicType, +} + +/// Shares the unchanged V10 table/type builders while emitting only V11. +#[derive(Default)] +pub struct RawModuleDefV11Builder { + inner: RawModuleDefV10Builder, + type_map: BTreeMap, + declared_visibility: BTreeMap, + capabilities: Vec, +} + +impl Deref for RawModuleDefV11Builder { + type Target = RawModuleDefV10Builder; + fn deref(&self) -> &Self::Target { + &self.inner + } +} +impl DerefMut for RawModuleDefV11Builder { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} +impl RawModuleDefV11Builder { + pub fn new() -> Self { + Self::default() + } + pub fn add_type(&mut self) -> AlgebraicType { + TypespaceBuilder::add_type::(self) + } + + pub fn add_reducer_with_visibility( + &mut self, + name: impl Into, + params: ProductType, + visibility: Option, + ) { + let name = name.into(); + self.inner.add_reducer(name.clone(), params); + if let Some(visibility) = visibility { + self.declared_visibility.insert(name, visibility); + } + } + + pub fn add_lifecycle_reducer_with_visibility( + &mut self, + lifecycle: Lifecycle, + name: impl Into, + params: ProductType, + visibility: Option, + ) { + let name = name.into(); + self.inner.add_lifecycle_reducer(lifecycle, name.clone(), params); + if let Some(visibility) = visibility { + self.declared_visibility.insert(name, visibility); + } + } + + pub fn add_procedure_with_visibility( + &mut self, + name: impl Into, + params: ProductType, + return_type: AlgebraicType, + visibility: Option, + ) { + let name = name.into(); + self.inner.add_procedure(name.clone(), params, return_type); + if let Some(visibility) = visibility { + self.declared_visibility.insert(name, visibility); + } + } + + pub fn add_capability(&mut self, capability: impl Into) { + self.capabilities.push(capability.into()); + } + + pub fn finish(self) -> RawModuleDefV11 { + let declared = self.declared_visibility; + let mut sections: Vec<_> = self + .inner + .finish() + .sections + .into_iter() + .map(|section| match section { + RawModuleDefV10Section::Typespace(value) => RawModuleDefV11Section::Typespace(value), + RawModuleDefV10Section::Types(value) => RawModuleDefV11Section::Types(value), + RawModuleDefV10Section::Tables(value) => RawModuleDefV11Section::Tables(value), + RawModuleDefV10Section::Reducers(rows) => RawModuleDefV11Section::Reducers( + rows.into_iter() + .map(|row| RawReducerDefV11 { + declared_visibility: declared.get(&row.source_name).copied(), + source_name: row.source_name, + params: row.params, + ok_return_type: row.ok_return_type, + err_return_type: row.err_return_type, + }) + .collect(), + ), + RawModuleDefV10Section::Procedures(rows) => RawModuleDefV11Section::Procedures( + rows.into_iter() + .map(|row| RawProcedureDefV11 { + declared_visibility: declared.get(&row.source_name).copied(), + source_name: row.source_name, + params: row.params, + return_type: row.return_type, + }) + .collect(), + ), + RawModuleDefV10Section::Views(value) => RawModuleDefV11Section::Views(value), + RawModuleDefV10Section::Schedules(value) => RawModuleDefV11Section::Schedules(value), + RawModuleDefV10Section::LifeCycleReducers(value) => RawModuleDefV11Section::LifeCycleReducers(value), + RawModuleDefV10Section::RowLevelSecurity(value) => RawModuleDefV11Section::RowLevelSecurity(value), + RawModuleDefV10Section::CaseConversionPolicy(value) => { + RawModuleDefV11Section::CaseConversionPolicy(value) + } + RawModuleDefV10Section::ExplicitNames(value) => RawModuleDefV11Section::ExplicitNames(value), + RawModuleDefV10Section::HttpHandlers(value) => RawModuleDefV11Section::HttpHandlers(value), + RawModuleDefV10Section::HttpRoutes(value) => RawModuleDefV11Section::HttpRoutes(value), + }) + .collect(); + if !self.capabilities.is_empty() { + sections.push(RawModuleDefV11Section::Capabilities(self.capabilities)); + } + RawModuleDefV11 { sections } + } +} + +impl TypespaceBuilder for RawModuleDefV11Builder { + fn add( + &mut self, + typeid: TypeId, + source_name: Option<&'static str>, + make_ty: impl FnOnce(&mut Self) -> AlgebraicType, + ) -> AlgebraicType { + if let Some(reference) = self.type_map.get(&typeid) { + return AlgebraicType::Ref(*reference); + } + let reference = self.inner.typespace_mut().add(AlgebraicType::unit()); + self.type_map.insert(typeid, reference); + if let Some(name) = source_name { + self.inner.types_mut().push(RawTypeDefV10 { + source_name: v10::sats_name_to_scoped_name_v10(name), + ty: reference, + custom_ordering: true, + }); + } + let ty = make_ty(self); + self.inner.typespace_mut()[reference] = ty; + AlgebraicType::Ref(reference) + } +} + +impl RawModuleDefV11 { + pub fn reducers(&self) -> impl Iterator { + self.sections + .iter() + .filter_map(|section| match section { + RawModuleDefV11Section::Reducers(rows) => Some(rows), + _ => None, + }) + .flatten() + } + pub fn tables(&self) -> impl Iterator { + self.sections + .iter() + .filter_map(|section| match section { + RawModuleDefV11Section::Tables(rows) => Some(rows), + _ => None, + }) + .flatten() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{RawModuleDef, SpacetimeType}; + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacyRawModuleDef { + V8BackCompat(crate::RawModuleDefV8), + V9(super::super::v9::RawModuleDefV9), + V10(RawModuleDefV10), + } + + #[test] + fn legacy_decoder_rejects_v11_instead_of_ignoring_visibility() { + let bytes = crate::bsatn::to_vec(&RawModuleDef::V11(RawModuleDefV11::default())).unwrap(); + assert_eq!(bytes[0], 3); + assert!(crate::bsatn::from_slice::(&bytes).is_err()); + } +} diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs new file mode 100644 index 00000000000..fecc63055e9 --- /dev/null +++ b/crates/lib/src/deployment.rs @@ -0,0 +1,234 @@ +//! Normalized deployment protocol shared by publication coordinators and hosts. +//! +//! Module artifacts and OCI objects are uploaded separately. A deployment is +//! immutable configuration, never a place for runtime credentials or env values. + +use crate::container::{ContainerAction, ContainerSpec, ContainerSpecLimits, ContainerValidationError}; +use crate::{bsatn, hash_bytes, Hash, SpacetimeType, Uuid}; + +pub const PUBLISH_PROTOCOL_VERSION: u32 = 1; +pub const SYSTEM_EMPTY_MODULE_VERSION: u32 = 1; +pub const MAX_DEPLOYMENT_BYTES: usize = 256 * 1024; +pub const PUBLISH_RETRY_WINDOW_MS: u64 = 7 * 24 * 60 * 60 * 1000; +pub const MAX_OPERATION_CLOCK_SKEW_MS: u64 = 5 * 60 * 1000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum UserModuleKind { + Wasm, + Js, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct UserModule { + pub kind: UserModuleKind, + /// The existing module program hash, not an OCI object digest. + pub program_hash: Hash, +} + +/// Explicit module replacement/removal, with its own exported schema name. +#[derive(Clone, Debug, Default, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ModuleAction { + #[default] + Keep, + Set(UserModule), + Remove, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "kind", content = "value", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum ModuleComponent { + SystemEmpty(u32), + User(UserModule), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct DeploymentSpecV1 { + pub module: ModuleComponent, + pub container: Option, +} + +/// Persist the discriminant along with the payload. Unknown encodings fail to +/// decode; a restore must never fall back to an empty or older deployment. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde( + tag = "version", + content = "deployment", + rename_all = "snake_case", + deny_unknown_fields + ) +)] +pub enum DeploymentSpec { + V1(DeploymentSpecV1), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct PublishEnvelope { + pub version: u32, + #[cfg_attr(feature = "serde", serde(with = "uuid_json"))] + pub operation_id: Uuid, + /// Exact compare-and-set precondition. None means no deployment record has + /// been installed yet, rather than permission to overwrite any revision. + pub expected_revision: Option, + #[cfg_attr(feature = "serde", serde(default))] + pub module_action: ModuleAction, + #[cfg_attr(feature = "serde", serde(default))] + pub container_action: ContainerAction, +} + +#[derive(Debug, thiserror::Error)] +pub enum DeploymentValidationError { + #[error("unsupported publish protocol version")] + UnsupportedVersion, + #[error("unsupported platform empty-module version")] + UnsupportedEmptyModule, + #[error("operation_id must be a version 7 UUID")] + InvalidOperationId, + #[error("publication operation has expired")] + ExpiredOperation, + #[error("publication operation timestamp is too far in the future")] + FutureOperation, + #[error("deployment exceeds the protocol size limit")] + TooLarge, + #[error("deployment encoding is invalid or unsupported")] + InvalidEncoding, + #[error(transparent)] + Container(#[from] ContainerValidationError), +} + +impl DeploymentSpec { + pub fn current(&self) -> &DeploymentSpecV1 { + match self { + Self::V1(spec) => spec, + } + } + + pub fn normalize(self, limits: &ContainerSpecLimits) -> Result { + let Self::V1(mut spec) = self; + if let ModuleComponent::SystemEmpty(version) = spec.module { + if version != SYSTEM_EMPTY_MODULE_VERSION { + return Err(DeploymentValidationError::UnsupportedEmptyModule); + } + } + spec.container = spec.container.map(|spec| spec.normalize(limits)).transpose()?; + let spec = Self::V1(spec); + spec.encode()?; + Ok(spec) + } + + pub fn encode(&self) -> Result, DeploymentValidationError> { + let bytes = bsatn::to_vec(self).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + Ok(bytes.into()) + } + + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + bsatn::from_slice(bytes).map_err(|_| DeploymentValidationError::InvalidEncoding) + } + + /// Call on the result of normalize. Unlike a request fingerprint, this is + /// independent of operation ID, publisher, and mutable execution status. + pub fn revision(&self) -> Result { + let mut bytes = b"spacetimedb/deployment\0".to_vec(); + bytes.extend_from_slice(&self.encode()?); + Ok(hash_bytes(bytes)) + } +} + +impl PublishEnvelope { + pub fn resolve( + &self, + previous: Option<&DeploymentSpec>, + limits: &ContainerSpecLimits, + ) -> Result { + if self.version != PUBLISH_PROTOCOL_VERSION { + return Err(DeploymentValidationError::UnsupportedVersion); + } + use spacetimedb_sats::uuid::Version; + if !matches!(self.operation_id.get_version(), Some(Version::V7)) { + return Err(DeploymentValidationError::InvalidOperationId); + } + let prior = previous.map(DeploymentSpec::current); + let module = match &self.module_action { + ModuleAction::Keep => prior + .map(|p| p.module.clone()) + .unwrap_or(ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION)), + ModuleAction::Set(module) => ModuleComponent::User(module.clone()), + ModuleAction::Remove => ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + }; + let container = match &self.container_action { + ContainerAction::Keep => prior.and_then(|p| p.container.clone()), + ContainerAction::Set(container) => Some(container.clone()), + ContainerAction::Remove => None, + }; + DeploymentSpec::V1(DeploymentSpecV1 { module, container }).normalize(limits) + } + + pub fn requires_container_permission(&self) -> bool { + matches!(self.container_action, ContainerAction::Set(_)) + } +} + +/// UUIDv7 embeds its creation millisecond. This makes an expired retry +/// distinguishable from a new request even after its ledger row is collected. +/// A fresh UUID with a changed timestamp is a different operation. +pub fn operation_expiry_ms(id: Uuid, now_ms: u64) -> Result { + if id.get_version() != Some(spacetimedb_sats::uuid::Version::V7) { + return Err(DeploymentValidationError::InvalidOperationId); + } + let created_ms = (id.as_u128() >> 80) as u64; + if created_ms > now_ms.saturating_add(MAX_OPERATION_CLOCK_SKEW_MS) { + return Err(DeploymentValidationError::FutureOperation); + } + let expires_ms = created_ms + PUBLISH_RETRY_WINDOW_MS; + if now_ms >= expires_ms { + return Err(DeploymentValidationError::ExpiredOperation); + } + Ok(expires_ms) +} + +#[cfg(feature = "serde")] +pub mod uuid_json { + use super::Uuid; + pub fn serialize(id: &Uuid, serializer: S) -> Result { + serializer.collect_str(id) + } + pub fn deserialize<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let value = ::deserialize(deserializer)?; + Uuid::parse_str(&value).map_err(serde::de::Error::custom) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/lib/src/deployment/tests.rs b/crates/lib/src/deployment/tests.rs new file mode 100644 index 00000000000..d8f37993615 --- /dev/null +++ b/crates/lib/src/deployment/tests.rs @@ -0,0 +1,128 @@ +use super::*; + +fn operation_id() -> Uuid { + Uuid::parse_str("01991ec4-0000-7000-8000-000000000001").unwrap() +} + +fn request(module_action: ModuleAction) -> PublishEnvelope { + PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: operation_id(), + expected_revision: None, + module_action, + container_action: ContainerAction::Keep, + } +} + +#[test] +fn component_actions_preserve_or_explicitly_remove_the_module() { + let limits = ContainerSpecLimits::default(); + let user = UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(b"valid module artifact"), + }; + let set = request(ModuleAction::Set(user.clone())).resolve(None, &limits).unwrap(); + assert_eq!(set.current().module, ModuleComponent::User(user)); + let keep = request(ModuleAction::Keep).resolve(Some(&set), &limits).unwrap(); + assert_eq!(set, keep); + let remove = request(ModuleAction::Remove).resolve(Some(&set), &limits).unwrap(); + assert_eq!( + remove.current().module, + ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION) + ); + assert_ne!(remove.revision().unwrap(), keep.revision().unwrap()); +} + +#[test] +fn concrete_module_actions_preserve_wire_tags_and_export_distinct_names() { + let module = UserModule { + kind: UserModuleKind::Js, + program_hash: hash_bytes(b"module"), + }; + assert_eq!(bsatn::to_vec(&ModuleAction::Keep).unwrap(), [0]); + assert_eq!(bsatn::to_vec(&ModuleAction::Remove).unwrap(), [2]); + let mut expected = vec![1]; + expected.extend(bsatn::to_vec(&module).unwrap()); + assert_eq!(bsatn::to_vec(&ModuleAction::Set(module)).unwrap(), expected); + + use crate::db::raw_def::v11::{RawModuleDefV11Builder, RawModuleDefV11Section}; + let mut builder = RawModuleDefV11Builder::new(); + builder.add_type::(); + let raw = builder.finish(); + let names: Vec<_> = raw + .sections + .iter() + .filter_map(|section| match section { + RawModuleDefV11Section::Types(types) => Some(types), + _ => None, + }) + .flatten() + .map(|ty| &*ty.source_name.source_name) + .collect(); + let distinct: std::collections::BTreeSet<_> = names.iter().copied().collect(); + assert_eq!( + names.len(), + distinct.len(), + "publish envelope exports duplicate type names" + ); + assert!(distinct.contains("ModuleAction")); + assert!(distinct.contains("ContainerAction")); +} + +#[test] +fn unknown_deployments_do_not_fall_back_to_an_empty_module() { + assert!(DeploymentSpec::decode(&[255]).is_err()); + let mut envelope = request(ModuleAction::Keep); + envelope.version += 1; + assert!(matches!( + envelope.resolve(None, &ContainerSpecLimits::default()), + Err(DeploymentValidationError::UnsupportedVersion) + )); + let spec = DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(9000), + container: None, + }); + assert!(matches!( + spec.normalize(&ContainerSpecLimits::default()), + Err(DeploymentValidationError::UnsupportedEmptyModule) + )); +} + +#[test] +fn operation_age_is_enforced_even_when_no_ledger_row_remains() { + let id = operation_id(); + let created_ms = (id.as_u128() >> 80) as u64; + assert_eq!( + operation_expiry_ms(id, created_ms).unwrap(), + created_ms + PUBLISH_RETRY_WINDOW_MS + ); + assert!(matches!( + operation_expiry_ms(id, created_ms + PUBLISH_RETRY_WINDOW_MS), + Err(DeploymentValidationError::ExpiredOperation) + )); + assert!(matches!( + operation_expiry_ms(id, created_ms - MAX_OPERATION_CLOCK_SKEW_MS - 1), + Err(DeploymentValidationError::FutureOperation) + )); + assert!(matches!( + operation_expiry_ms(Uuid::NIL, created_ms), + Err(DeploymentValidationError::InvalidOperationId) + )); +} + +#[cfg(feature = "serde")] +#[test] +fn uuid_json_is_lossless_and_omission_means_keep() { + let envelope = request(ModuleAction::Keep); + let mut json = serde_json::to_value(&envelope).unwrap(); + assert_eq!(json["operation_id"], operation_id().to_string()); + json.as_object_mut().unwrap().remove("module_action"); + json.as_object_mut().unwrap().remove("container_action"); + let parsed: PublishEnvelope = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(parsed, envelope); + assert!(!parsed.requires_container_permission()); + json["publisher_identity"] = "untrusted owner override".into(); + assert!(serde_json::from_value::(json).is_err()); + let binary = bsatn::to_vec(&envelope).unwrap(); + assert_eq!(bsatn::from_slice::(&binary).unwrap(), envelope); +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 2e8b9c08336..547b78d0e26 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -11,6 +11,8 @@ use std::any::TypeId; use std::collections::{btree_map, BTreeMap}; pub mod connection_id; +pub mod container; +pub mod deployment; pub mod db; mod direct_index_key; pub mod error; @@ -166,6 +168,7 @@ pub enum RawModuleDef { V8BackCompat(RawModuleDefV8), V9(db::raw_def::v9::RawModuleDefV9), V10(db::raw_def::v10::RawModuleDefV10), + V11(db::raw_def::v11::RawModuleDefV11), // TODO(jgilles): It would be nice to have a custom error message if this fails with an unknown variant, // but I'm not sure if that can be done via the Deserialize trait. } diff --git a/crates/oci/Cargo.toml b/crates/oci/Cargo.toml new file mode 100644 index 00000000000..f32bc83ab59 --- /dev/null +++ b/crates/oci/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "spacetimedb-oci" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license-file = "../../LICENSE.txt" +description = "Bounded OCI image validation for SpacetimeDB container publishing" + +[dependencies] +spacetimedb-lib = { workspace = true, features = ["serde"] } +anyhow.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2 = "0.10" +flate2.workspace = true +tar.workspace = true +zstd = "0.13" + +[lints] +workspace = true diff --git a/crates/oci/src/layers.rs b/crates/oci/src/layers.rs new file mode 100644 index 00000000000..4e0d0a0f537 --- /dev/null +++ b/crates/oci/src/layers.rs @@ -0,0 +1,350 @@ +//! Stream-verify layer expansion without extracting anything onto the host. +//! Resource admission uses measured bytes and bounded metadata, never compressed +//! sizes alone. The runtime must also enforce a dedicated finite cache filesystem. + +use crate::{Descriptor, OciDigest}; +use anyhow::{bail, ensure, Context, Result}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeMap, + io::{self, Read}, +}; + +#[derive(Clone, Copy, Debug)] +pub struct LayerLimits { + pub max_uncompressed_bytes: u64, + pub max_regular_file_bytes: u64, + pub max_entries: u64, + pub max_metadata_bytes: usize, + pub max_path_bytes: usize, + pub zstd_window_log_max: u32, +} +impl Default for LayerLimits { + fn default() -> Self { + Self { + max_uncompressed_bytes: 128 * 1024 * 1024 * 1024, + max_regular_file_bytes: 64 * 1024 * 1024 * 1024, + max_entries: 1_000_000, + max_metadata_bytes: 64 * 1024, + max_path_bytes: 4096, + zstd_window_log_max: 27, + } + } +} + +#[derive(Clone, Copy, Debug, Serialize)] +pub struct VerifiedLayerSize { + pub uncompressed_tar_bytes: u64, + pub entries: u64, + pub regular_file_bytes: u64, +} + +#[derive(Clone, Copy, Debug, Serialize)] +pub struct VerifiedImageSize { + pub compressed_bytes: u64, + pub uncompressed_tar_bytes: u64, + pub entries: u64, + pub cache_reservation_bytes: u64, +} +impl VerifiedImageSize { + /// Conservative cache accounting, supplemented by the backend's hard + /// filesystem bound. Includes temporary compressed/unpacked copies and + /// per-entry/per-layer filesystem metadata. The trusted artifact service + /// supplies these verified measurements, never a publisher JSON field. + pub fn from_layers(compressed_bytes: u64, layers: &[VerifiedLayerSize]) -> Result { + let mut tar = 0u64; + let mut entries = 0u64; + for layer in layers { + tar = tar + .checked_add(layer.uncompressed_tar_bytes) + .context("expanded image size overflow")?; + entries = entries + .checked_add(layer.entries) + .context("image entry count overflow")?; + } + let reservation = compressed_bytes + .checked_mul(2) + .and_then(|n| tar.checked_mul(3).and_then(|v| n.checked_add(v))) + .and_then(|n| entries.checked_mul(64 * 1024).and_then(|v| n.checked_add(v))) + .and_then(|n| { + (layers.len() as u64) + .checked_mul(16 * 1024 * 1024) + .and_then(|v| n.checked_add(v)) + }) + .context("image cache reservation overflow")?; + Ok(Self { + compressed_bytes, + uncompressed_tar_bytes: tar, + entries, + cache_reservation_bytes: reservation, + }) + } +} + +struct HashBounded { + inner: R, + hash: Sha256, + count: u64, + limit: u64, +} +impl Read for HashBounded { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + return Ok(0); + } + if self.count == self.limit { + let mut extra = [0u8; 1]; + if self.inner.read(&mut extra)? == 0 { + return Ok(0); + } + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "layer exceeds configured byte bound", + )); + } + let maximum = buf.len().min((self.limit - self.count).min(usize::MAX as u64) as usize); + let count = self.inner.read(&mut buf[..maximum])?; + self.count += count as u64; + self.hash.update(&buf[..count]); + Ok(count) + } +} + +pub fn verify_layer( + reader: impl Read, + descriptor: &Descriptor, + diff_id: OciDigest, + limits: LayerLimits, +) -> Result { + ensure!( + descriptor.size > 0 && descriptor.size <= crate::MAX_IMAGE_BYTES, + "invalid compressed layer size" + ); + ensure!( + descriptor.urls.is_empty() && descriptor.data.is_none(), + "external layer sources are unsupported" + ); + let mut compressed = HashBounded { + inner: reader, + hash: Sha256::new(), + count: 0, + limit: descriptor.size, + }; + let decoder: Box = match descriptor.media_type.as_str() { + "application/vnd.oci.image.layer.v1.tar" | "application/vnd.docker.image.rootfs.diff.tar" => { + Box::new(&mut compressed) + } + "application/vnd.oci.image.layer.v1.tar+gzip" | "application/vnd.docker.image.rootfs.diff.tar.gzip" => { + Box::new(flate2::read::MultiGzDecoder::new(&mut compressed)) + } + "application/vnd.oci.image.layer.v1.tar+zstd" => { + let mut decoder = zstd::stream::read::Decoder::new(&mut compressed)?; + decoder.window_log_max(limits.zstd_window_log_max)?; + Box::new(decoder) + } + _ => bail!("unsupported or foreign layer media type"), + }; + let mut expanded = HashBounded { + inner: decoder, + hash: Sha256::new(), + count: 0, + limit: limits.max_uncompressed_bytes, + }; + let (entries, regular_file_bytes) = scan_tar(&mut expanded, limits)?; + let uncompressed_tar_bytes = expanded.count; + ensure!( + OciDigest::sha256(expanded.hash.clone().finalize().into()) == diff_id, + "layer uncompressed SHA-256 does not match rootfs diff ID" + ); + drop(expanded); + ensure!( + compressed.count == descriptor.size + && OciDigest::sha256(compressed.hash.finalize().into()) == descriptor.digest, + "compressed layer SHA-256 or length mismatch" + ); + Ok(VerifiedLayerSize { + uncompressed_tar_bytes, + entries, + regular_file_bytes, + }) +} + +fn scan_tar(reader: &mut impl Read, limits: LayerLimits) -> Result<(u64, u64)> { + let mut entries = 0u64; + let mut regular_bytes = 0u64; + let mut pax = BTreeMap::new(); + let mut global = BTreeMap::new(); + let mut long_path = None; + let mut long_link = None; + loop { + let mut block = [0u8; 512]; + let first = reader.read(&mut block[..1])?; + if first == 0 { + break; + } + reader.read_exact(&mut block[1..]).context("truncated TAR header")?; + if block.iter().all(|&b| b == 0) { + continue; + } + entries = entries.checked_add(1).context("TAR entry count overflow")?; + ensure!(entries <= limits.max_entries, "too many layer entries"); + let header = tar::Header::from_byte_slice(&block); + let checksum = block[..148] + .iter() + .chain(&block[156..]) + .map(|&b| u32::from(b)) + .sum::() + + 8 * 32; + ensure!(header.cksum()? == checksum, "invalid TAR header checksum"); + let kind = header.entry_type().as_byte(); + let header_size = header.entry_size()?; + if matches!(kind, b'x' | b'g' | b'L' | b'K') { + ensure!( + header_size <= limits.max_metadata_bytes as u64, + "TAR metadata entry is too large" + ); + let mut bytes = vec![0; header_size as usize]; + reader.read_exact(&mut bytes)?; + skip_padding(reader, header_size)?; + match kind { + b'x' => { + ensure!(pax.is_empty(), "duplicate local PAX metadata"); + pax = parse_pax(&bytes)?; + } + b'g' => { + let update = parse_pax(&bytes)?; + global.extend(update); + ensure!( + global.len() <= 128 + && global.iter().map(|(k, v)| k.len() + v.len()).sum::() + <= limits.max_metadata_bytes, + "global PAX metadata exceeds bound" + ); + } + b'L' => { + ensure!(long_path.is_none(), "duplicate GNU long path"); + long_path = Some(trim_nul(bytes)); + } + b'K' => { + ensure!(long_link.is_none(), "duplicate GNU long link"); + long_link = Some(trim_nul(bytes)); + } + _ => unreachable!(), + } + continue; + } + let property = |name: &str| pax.get(name).or_else(|| global.get(name)); + let size = property("size") + .map(|v| v.parse::()) + .transpose()? + .unwrap_or(header_size); + ensure!(size <= limits.max_regular_file_bytes, "layer file exceeds size bound"); + let raw_path = header.path_bytes(); + let path = property("path") + .map(String::as_bytes) + .or(long_path.as_deref()) + .unwrap_or(&raw_path); + validate_path(path, limits.max_path_bytes, kind == b'5')?; + match kind { + 0 | b'0' | b'7' => { + regular_bytes = regular_bytes.checked_add(size).context("layer file size overflow")?; + } + b'5' => ensure!(size == 0, "directory entry has data"), + b'1' | b'2' => { + ensure!(size == 0, "link entry has data"); + let raw_link = header.link_name_bytes(); + let link = property("linkpath") + .map(String::as_bytes) + .or(long_link.as_deref()) + .or(raw_link.as_deref()) + .context("link target is missing")?; + ensure!( + !link.is_empty() && link.len() <= limits.max_path_bytes && !link.contains(&0), + "invalid link target" + ); + if kind == b'1' { + validate_path(link, limits.max_path_bytes, false)?; + } + // Absolute symbolic links are normal inside a Linux image. + // Extraction remains the confined runtime unpacker's job. + } + _ => bail!("unsupported sparse, special-device, or unknown TAR entry type"), + } + skip_exact(reader, size)?; + skip_padding(reader, size)?; + pax.clear(); + long_path = None; + long_link = None; + } + ensure!( + pax.is_empty() && long_path.is_none() && long_link.is_none(), + "orphaned TAR extension metadata" + ); + Ok((entries, regular_bytes)) +} + +fn trim_nul(mut bytes: Vec) -> Vec { + while bytes.last() == Some(&0) { + bytes.pop(); + } + bytes +} +fn skip_exact(reader: &mut impl Read, mut bytes: u64) -> Result<()> { + let mut buffer = [0u8; 64 * 1024]; + while bytes > 0 { + let count = bytes.min(buffer.len() as u64) as usize; + reader.read_exact(&mut buffer[..count])?; + bytes -= count as u64; + } + Ok(()) +} +fn skip_padding(reader: &mut impl Read, size: u64) -> Result<()> { + skip_exact(reader, (512 - size % 512) % 512) +} +fn validate_path(path: &[u8], max: usize, root_directory: bool) -> Result<()> { + ensure!( + !path.is_empty() && path.len() <= max && !path.contains(&0) && !path.starts_with(b"/"), + "invalid layer entry path" + ); + ensure!( + !path.split(|&b| b == b'/').any(|p| p == b".."), + "layer entry path escapes image root" + ); + ensure!( + root_directory || path.split(|&b| b == b'/').any(|p| !p.is_empty() && p != b"."), + "invalid root file entry" + ); + Ok(()) +} +fn parse_pax(mut bytes: &[u8]) -> Result> { + let mut result = BTreeMap::new(); + while !bytes.is_empty() { + let space = bytes.iter().position(|&b| b == b' ').context("invalid PAX record")?; + ensure!(space > 0 && space <= 10, "invalid PAX length"); + let length = std::str::from_utf8(&bytes[..space])?.parse::()?; + ensure!( + length > space + 2 && length <= bytes.len() && bytes[length - 1] == b'\n', + "invalid PAX record length" + ); + let record = std::str::from_utf8(&bytes[space + 1..length - 1])?; + let (key, value) = record.split_once('=').context("invalid PAX property")?; + ensure!( + key.len() <= 256 + && (matches!( + key, + "path" | "linkpath" | "size" | "mtime" | "atime" | "ctime" | "uid" | "gid" | "uname" | "gname" + ) || key.starts_with("SCHILY.xattr.")), + "unsupported sparse or unknown PAX property" + ); + ensure!( + result.insert(key.to_owned(), value.to_owned()).is_none() && result.len() <= 128, + "duplicate or excessive PAX properties" + ); + bytes = &bytes[length..]; + } + Ok(result) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oci/src/layers/tests.rs b/crates/oci/src/layers/tests.rs new file mode 100644 index 00000000000..044ec7bb43e --- /dev/null +++ b/crates/oci/src/layers/tests.rs @@ -0,0 +1,130 @@ +use super::*; +use std::io::Write; + +fn archive() -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_ustar(); + header.set_path("usr/bin/main").unwrap(); + header.set_size(4); + header.set_mode(0o755); + header.set_cksum(); + builder.append(&header, &b"data"[..]).unwrap(); + builder.into_inner().unwrap() +} +fn descriptor(bytes: &[u8], media: &str) -> Descriptor { + Descriptor { + digest: crate::sha256(bytes), + size: bytes.len() as u64, + media_type: media.into(), + platform: None, + urls: vec![], + data: None, + artifact_type: None, + } +} +fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} +#[test] +fn verifies_plain_gzip_and_zstd_against_actual_expanded_bytes() { + let tar = archive(); + let diff = crate::sha256(&tar); + for (bytes, media) in [ + (tar.clone(), "application/vnd.oci.image.layer.v1.tar"), + (gzip(&tar), "application/vnd.oci.image.layer.v1.tar+gzip"), + ( + zstd::stream::encode_all(&tar[..], 1).unwrap(), + "application/vnd.oci.image.layer.v1.tar+zstd", + ), + ] { + let result = verify_layer(&bytes[..], &descriptor(&bytes, media), diff, LayerLimits::default()).unwrap(); + assert_eq!(result.uncompressed_tar_bytes, tar.len() as u64); + assert_eq!(result.entries, 1); + assert_eq!(result.regular_file_bytes, 4); + assert!( + VerifiedImageSize::from_layers(bytes.len() as u64, &[result]) + .unwrap() + .cache_reservation_bytes + > result.uncompressed_tar_bytes + ); + } +} +#[test] +fn decompression_bomb_is_stopped_before_declared_diff_id_can_be_trusted() { + let expanded = vec![0u8; 1024 * 1024]; + let compressed = gzip(&expanded); + let result = verify_layer( + &compressed[..], + &descriptor(&compressed, "application/vnd.oci.image.layer.v1.tar+gzip"), + crate::sha256(&expanded), + LayerLimits { + max_uncompressed_bytes: 4096, + ..LayerLimits::default() + }, + ); + assert!(result.unwrap_err().to_string().contains("byte bound")); +} +#[test] +fn catches_digest_mismatch_truncation_and_entry_limit() { + let bytes = archive(); + let descriptor = descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar"); + assert!(verify_layer(&bytes[..], &descriptor, crate::sha256(b"wrong"), LayerLimits::default()).is_err()); + assert!(verify_layer( + &bytes[..100], + &descriptor, + crate::sha256(&bytes), + LayerLimits::default() + ) + .is_err()); + assert!(verify_layer( + &bytes[..], + &descriptor, + crate::sha256(&bytes), + LayerLimits { + max_entries: 0, + ..LayerLimits::default() + } + ) + .is_err()); +} +#[test] +fn extension_size_is_bounded_before_allocating_and_sparse_metadata_is_rejected() { + let mut header = tar::Header::new_gnu(); + header.set_path("pax").unwrap(); + header.set_entry_type(tar::EntryType::XHeader); + header.set_size(1 << 30); + header.set_cksum(); + let bytes = header.as_bytes().to_vec(); + assert!(verify_layer( + &bytes[..], + &descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar"), + crate::sha256(&bytes), + LayerLimits::default() + ) + .unwrap_err() + .to_string() + .contains("metadata entry")); + assert!(parse_pax(b"25 GNU.sparse.size=123456\n").is_err()); + assert!(validate_path(b"safe/../../host", 4096, false).is_err()); + assert!(validate_path(b"/absolute", 4096, false).is_err()); +} +#[test] +fn counts_concatenated_gzip_members_and_checks_pax_sizes() { + let tar = archive(); + let mut bytes = gzip(&tar); + bytes.extend(gzip(&tar)); + let mut both = tar.clone(); + both.extend(&tar); + let result = verify_layer( + &bytes[..], + &descriptor(&bytes, "application/vnd.oci.image.layer.v1.tar+gzip"), + crate::sha256(&both), + LayerLimits::default(), + ) + .unwrap(); + assert_eq!(result.entries, 2); + assert_eq!(parse_pax(b"10 size=4\n").unwrap()["size"], "4"); + assert!(parse_pax(b"99 size=4\n").is_err()); +} diff --git a/crates/oci/src/lib.rs b/crates/oci/src/lib.rs new file mode 100644 index 00000000000..c6e9d04c29d --- /dev/null +++ b/crates/oci/src/lib.rs @@ -0,0 +1,377 @@ +//! Validate immutable OCI objects before accepting a container deployment. +//! +//! Registry references and index annotations are discovery inputs. Only verified +//! object bytes and an exact selected platform establish the published image. + +pub mod layers; + +use anyhow::{bail, ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::container::{ImagePlatform, OciDigest, MAX_ARGV_ENTRIES, MAX_ENV_KEYS, MAX_EXEC_STRING_BYTES}; +use std::collections::BTreeMap; + +pub const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; +pub const OCI_INDEX: &str = "application/vnd.oci.image.index.v1+json"; +pub const OCI_CONFIG: &str = "application/vnd.oci.image.config.v1+json"; +pub const DOCKER_MANIFEST: &str = "application/vnd.docker.distribution.manifest.v2+json"; +pub const DOCKER_INDEX: &str = "application/vnd.docker.distribution.manifest.list.v2+json"; +pub const DOCKER_CONFIG: &str = "application/vnd.docker.container.image.v1+json"; +pub const MAX_MANIFEST_BYTES: usize = 4 * 1024 * 1024; +pub const MAX_CONFIG_BYTES: usize = 1024 * 1024; +pub const MAX_LAYERS: usize = 256; +pub const MAX_INDEX_ENTRIES: usize = 256; +pub const MAX_IMAGE_BYTES: u64 = 64 * 1024 * 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Descriptor { + pub media_type: String, + pub digest: OciDigest, + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub urls: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artifact_type: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Platform { + pub os: String, + pub architecture: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub variant: Option, + #[serde(default, rename = "os.version", skip_serializing_if = "Option::is_none")] + pub os_version: Option, + #[serde(default, rename = "os.features", skip_serializing_if = "Vec::is_empty")] + pub os_features: Vec, +} + +impl Platform { + fn matches(&self, requested: &ImagePlatform) -> bool { + self.os == requested.os + && self.architecture == requested.architecture + && self.os_version.as_deref().is_none_or(str::is_empty) + && self.os_features.is_empty() + && match (self.architecture.as_str(), self.variant.as_deref()) { + (_, None | Some("")) | ("arm64", Some("v8")) => true, + _ => false, + } + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Manifest { + pub schema_version: u32, + pub media_type: String, + pub config: Descriptor, + pub layers: Vec, + #[serde(default)] + artifact_type: Option, + #[serde(default)] + subject: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ImageIndex { + pub schema_version: u32, + pub media_type: String, + pub manifests: Vec, +} + +pub fn sha256(bytes: &[u8]) -> OciDigest { + OciDigest::sha256(Sha256::digest(bytes).into()) +} + +pub fn verify_object(descriptor: &Descriptor, bytes: &[u8]) -> Result<()> { + ensure!( + descriptor.size == bytes.len() as u64, + "OCI object length differs from its descriptor" + ); + ensure!( + sha256(bytes) == descriptor.digest, + "OCI object SHA-256 differs from its descriptor" + ); + Ok(()) +} + +fn validate_descriptor(descriptor: &Descriptor, max_bytes: u64) -> Result<()> { + ensure!( + descriptor.size > 0 && descriptor.size <= max_bytes, + "OCI descriptor size exceeds admission bounds" + ); + ensure!( + descriptor.urls.is_empty(), + "external OCI descriptor URLs are not supported" + ); + ensure!(descriptor.data.is_none(), "inline OCI descriptor data is not supported"); + ensure!( + descriptor.artifact_type.is_none(), + "OCI artifacts are not executable images" + ); + Ok(()) +} + +pub fn parse_manifest(bytes: &[u8]) -> Result { + ensure!(bytes.len() <= MAX_MANIFEST_BYTES, "OCI manifest is too large"); + let manifest: Manifest = serde_json::from_slice(bytes).context("invalid OCI image manifest")?; + ensure!(manifest.schema_version == 2, "unsupported OCI manifest schema version"); + ensure!( + matches!(manifest.media_type.as_str(), OCI_MANIFEST | DOCKER_MANIFEST), + "unsupported image manifest media type" + ); + ensure!( + manifest.artifact_type.is_none() && manifest.subject.is_none(), + "OCI artifact manifests are not executable images" + ); + ensure!(manifest.layers.len() <= MAX_LAYERS, "too many OCI image layers"); + validate_descriptor(&manifest.config, MAX_CONFIG_BYTES as u64)?; + ensure!( + matches!(manifest.config.media_type.as_str(), OCI_CONFIG | DOCKER_CONFIG), + "unsupported image config media type" + ); + let mut total = manifest.config.size; + for layer in &manifest.layers { + validate_descriptor(layer, MAX_IMAGE_BYTES)?; + ensure!( + matches!( + layer.media_type.as_str(), + "application/vnd.oci.image.layer.v1.tar" + | "application/vnd.oci.image.layer.v1.tar+gzip" + | "application/vnd.oci.image.layer.v1.tar+zstd" + | "application/vnd.docker.image.rootfs.diff.tar" + | "application/vnd.docker.image.rootfs.diff.tar.gzip" + ), + "unsupported or foreign image layer media type" + ); + total = total.checked_add(layer.size).context("OCI image size overflow")?; + } + ensure!(total <= MAX_IMAGE_BYTES, "OCI image exceeds compressed object quota"); + Ok(manifest) +} + +/// Select exactly one executable image for the requested platform. BuildKit may +/// include attestation descriptors for unknown/unknown; they are never executed. +pub fn select_platform(bytes: &[u8], requested: &ImagePlatform) -> Result { + validate_platform(requested)?; + ensure!(bytes.len() <= MAX_MANIFEST_BYTES, "OCI image index is too large"); + let index: ImageIndex = serde_json::from_slice(bytes).context("invalid OCI image index")?; + ensure!( + index.schema_version == 2 && matches!(index.media_type.as_str(), OCI_INDEX | DOCKER_INDEX), + "unsupported image index format" + ); + ensure!( + index.manifests.len() <= MAX_INDEX_ENTRIES, + "too many image index entries" + ); + let mut selected = None; + for descriptor in index.manifests { + if !descriptor.platform.as_ref().is_some_and(|p| p.matches(requested)) { + continue; + } + validate_descriptor(&descriptor, MAX_MANIFEST_BYTES as u64)?; + ensure!( + matches!(descriptor.media_type.as_str(), OCI_MANIFEST | DOCKER_MANIFEST), + "selected platform is not an image manifest" + ); + ensure!( + selected.replace(descriptor).is_none(), + "OCI index has ambiguous images for the selected platform" + ); + } + selected.context("OCI image does not contain the selected Linux platform") +} + +fn validate_platform(platform: &ImagePlatform) -> Result<()> { + ensure!( + platform.os == "linux" && matches!(platform.architecture.as_str(), "amd64" | "arm64"), + "unsupported container platform" + ); + Ok(()) +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ImageConfig { + pub architecture: String, + pub os: String, + #[serde(default)] + pub variant: Option, + #[serde(default)] + pub config: ContainerConfig, + pub rootfs: RootFs, +} + +#[derive(Clone, Default, Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct ContainerConfig { + #[serde(default)] + pub entrypoint: Option>, + #[serde(default)] + pub cmd: Option>, + #[serde(default)] + pub user: String, + #[serde(default, rename = "WorkingDir")] + pub working_directory: String, + #[serde(default)] + pub env: Option>, + #[serde(default)] + pub volumes: Option>, +} + +impl std::fmt::Debug for ContainerConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ContainerConfig") + .field("entrypoint", &self.entrypoint) + .field("cmd", &self.cmd) + .field("user", &self.user) + .field("working_directory", &self.working_directory) + .field( + "env_keys", + &self.env.as_ref().map(|env| { + env.iter() + .map(|v| v.split('=').next().unwrap_or("")) + .collect::>() + }), + ) + .field("volumes", &self.volumes.as_ref().map(|v| v.keys().collect::>())) + .finish() + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct RootFs { + #[serde(rename = "type")] + pub kind: String, + pub diff_ids: Vec, +} + +pub fn parse_config(bytes: &[u8], manifest: &Manifest, platform: &ImagePlatform) -> Result { + validate_platform(platform)?; + ensure!(bytes.len() <= MAX_CONFIG_BYTES, "OCI image config is too large"); + verify_object(&manifest.config, bytes)?; + let image: ImageConfig = serde_json::from_slice(bytes).context("invalid OCI image config")?; + ensure!( + Platform { + os: image.os.clone(), + architecture: image.architecture.clone(), + variant: image.variant.clone(), + os_version: None, + os_features: vec![] + } + .matches(platform), + "image config platform does not match selected platform" + ); + ensure!( + image.rootfs.kind == "layers" && image.rootfs.diff_ids.len() == manifest.layers.len(), + "image rootfs does not match layer descriptors" + ); + ensure!( + image.config.volumes.as_ref().is_none_or(BTreeMap::is_empty), + "image-declared volumes are unsupported in Stage 1" + ); + image.config.environment()?; + for argv in [&image.config.entrypoint, &image.config.cmd].into_iter().flatten() { + ensure!(argv.len() <= MAX_ARGV_ENTRIES, "too many image command arguments"); + for arg in argv { + validate_string(arg)?; + } + } + validate_string(&image.config.user)?; + validate_string(&image.config.working_directory)?; + ensure!( + image.config.working_directory.is_empty() || image.config.working_directory.starts_with('/'), + "image working directory must be absolute" + ); + Ok(image) +} + +fn validate_string(value: &str) -> Result<()> { + ensure!( + value.len() < MAX_EXEC_STRING_BYTES && !value.contains('\0'), + "invalid container startup string" + ); + Ok(()) +} + +impl ContainerConfig { + /// Keep image defaults separately from the normalized spec. Environment + /// values remain in the immutable image, never in hot deployment metadata. + pub fn environment(&self) -> Result> { + let env = self.env.as_deref().unwrap_or_default(); + ensure!(env.len() <= MAX_ENV_KEYS, "too many image environment variables"); + let mut result = BTreeMap::new(); + for value in env { + validate_string(value)?; + let (key, value) = value + .split_once('=') + .context("image environment entry must contain '='")?; + ensure!( + valid_env_key(key) && !key.starts_with("SPACETIMEDB_"), + "invalid or reserved image environment key" + ); + ensure!( + result.insert(key.to_owned(), value.to_owned()).is_none(), + "duplicate image environment key" + ); + } + Ok(result) + } + + pub fn argv(&self, override_command: Option<&[String]>) -> Result> { + let argv = match override_command { + Some(argv) => argv.to_vec(), + None => self + .entrypoint + .iter() + .flatten() + .chain(self.cmd.iter().flatten()) + .cloned() + .collect(), + }; + ensure!( + !argv.is_empty() && !argv[0].is_empty() && argv.len() <= MAX_ARGV_ENTRIES, + "image needs a nonempty main command" + ); + for arg in &argv { + validate_string(arg)?; + } + Ok(argv) + } +} + +pub fn valid_env_key(key: &str) -> bool { + let mut bytes = key.bytes(); + key.len() <= 256 + && bytes.next().is_some_and(|b| b.is_ascii_alphabetic() || b == b'_') + && bytes.all(|b| b.is_ascii_alphanumeric() || b == b'_') +} + +/// Descriptor closure used to pin all required objects before desired-state +/// commit. Reject a digest repeated with conflicting lengths or media types. +pub fn object_closure(manifest_descriptor: Descriptor, manifest: &Manifest) -> Result> { + validate_descriptor(&manifest_descriptor, MAX_MANIFEST_BYTES as u64)?; + let mut seen = BTreeMap::new(); + let mut objects = Vec::with_capacity(manifest.layers.len() + 2); + for descriptor in std::iter::once(manifest_descriptor) + .chain(std::iter::once(manifest.config.clone())) + .chain(manifest.layers.iter().cloned()) + { + match seen.insert(descriptor.digest, (descriptor.size, descriptor.media_type.clone())) { + Some(previous) if previous != (descriptor.size, descriptor.media_type.clone()) => { + bail!("OCI digest has conflicting descriptors") + } + Some(_) => {} + None => objects.push(descriptor), + } + } + Ok(objects) +} + +#[cfg(test)] +mod tests; diff --git a/crates/oci/src/tests.rs b/crates/oci/src/tests.rs new file mode 100644 index 00000000000..6fe4192bfa2 --- /dev/null +++ b/crates/oci/src/tests.rs @@ -0,0 +1,166 @@ +use super::*; +use serde_json::{json, Value}; + +fn platform() -> ImagePlatform { + ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + } +} +fn config() -> Value { + json!({"architecture":"arm64","os":"linux","config":{ + "Entrypoint":["node"],"Cmd":["server.js"],"User":"1000:1000","WorkingDir":"/app", + "Env":["PATH=/usr/bin","APP_KEY=image-secret"] + },"rootfs":{"type":"layers","diff_ids":[sha256(b"uncompressed layer")]}}) +} +fn fixture(config: &Value) -> (Vec, Vec, Descriptor) { + let config_bytes = serde_json::to_vec(config).unwrap(); + let manifest = serde_json::to_vec(&json!({ + "schemaVersion":2,"mediaType":OCI_MANIFEST, + "config":{"mediaType":OCI_CONFIG,"digest":sha256(&config_bytes),"size":config_bytes.len()}, + "layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":sha256(b"layer"),"size":5}] + })) + .unwrap(); + let descriptor = Descriptor { + media_type: OCI_MANIFEST.into(), + digest: sha256(&manifest), + size: manifest.len() as u64, + platform: Some(Platform { + os: "linux".into(), + architecture: "arm64".into(), + variant: Some("v8".into()), + os_version: None, + os_features: vec![], + }), + urls: vec![], + data: None, + artifact_type: None, + }; + (config_bytes, manifest, descriptor) +} + +#[test] +fn immutable_image_preserves_defaults_and_explicit_command_replaces_all_argv() { + let (bytes, raw, descriptor) = fixture(&config()); + verify_object(&descriptor, &raw).unwrap(); + let manifest = parse_manifest(&raw).unwrap(); + let image = parse_config(&bytes, &manifest, &platform()).unwrap(); + assert_eq!(image.config.argv(None).unwrap(), ["node", "server.js"]); + assert_eq!(image.config.argv(Some(&["/bin/sh".into()])).unwrap(), ["/bin/sh"]); + assert_eq!(image.config.user, "1000:1000"); + assert_eq!(image.config.working_directory, "/app"); + assert_eq!(image.config.environment().unwrap()["APP_KEY"], "image-secret"); + assert!(!format!("{image:?}").contains("image-secret")); + assert_eq!(object_closure(descriptor, &manifest).unwrap().len(), 3); +} + +#[test] +fn content_verification_checks_actual_sha256_and_length() { + assert_eq!( + sha256(b"abc").to_string(), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let (_, bytes, mut descriptor) = fixture(&config()); + assert!(verify_object(&descriptor, b"different").is_err()); + descriptor.size -= 1; + assert!(verify_object(&descriptor, &bytes).is_err()); +} + +#[test] +fn platform_selection_ignores_attestations_and_rejects_ambiguity_or_absence() { + let (_, _, descriptor) = fixture(&config()); + let mut attestation = descriptor.clone(); + attestation.platform = Some(Platform { + os: "unknown".into(), + architecture: "unknown".into(), + variant: None, + os_version: None, + os_features: vec![], + }); + let index = |descriptors: Vec| { + serde_json::to_vec(&json!({"schemaVersion":2,"mediaType":OCI_INDEX,"manifests":descriptors})).unwrap() + }; + assert_eq!( + select_platform(&index(vec![attestation.clone(), descriptor.clone()]), &platform()).unwrap(), + descriptor + ); + assert!(select_platform(&index(vec![descriptor.clone(), descriptor]), &platform()).is_err()); + assert!(select_platform(&index(vec![attestation]), &platform()).is_err()); +} + +#[test] +fn foreign_layers_urls_inline_data_and_artifacts_fail_closed() { + let (_, bytes, _) = fixture(&config()); + let baseline: Value = serde_json::from_slice(&bytes).unwrap(); + for (key, value) in [ + ("urls", json!(["http://169.254.169.254/latest/meta-data/"])), + ("data", json!("inline")), + ("artifactType", json!("application/test")), + ] { + let mut manifest = baseline.clone(); + manifest["layers"][0][key] = value; + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + } + let mut manifest = baseline.clone(); + manifest["layers"][0]["mediaType"] = json!("application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + let mut manifest = baseline; + manifest["artifactType"] = json!("application/test"); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); +} + +#[test] +fn config_rejects_volumes_wrong_platform_rootfs_and_reserved_environment() { + for (field, value) in [ + ("Volumes", json!({"/escape":{}})), + ("Env", json!(["SPACETIMEDB_IDENTITY=forged"])), + ("Env", json!(["KEY=a", "KEY=b"])), + ("Env", json!(["NO_EQUALS"])), + ("Env", json!(["KEY=contains\u{0}nul"])), + ("WorkingDir", json!("relative/path")), + ] { + let mut config = config(); + config["config"][field] = value; + let (bytes, raw, _) = fixture(&config); + assert!( + parse_config(&bytes, &parse_manifest(&raw).unwrap(), &platform()).is_err(), + "{field}" + ); + } + for (field, value) in [ + ("architecture", json!("amd64")), + ("rootfs", json!({"type":"layers","diff_ids":[]})), + ] { + let mut config = config(); + config[field] = value; + let (bytes, raw, _) = fixture(&config); + assert!(parse_config(&bytes, &parse_manifest(&raw).unwrap(), &platform()).is_err()); + } +} + +#[test] +fn size_limits_and_conflicting_descriptors_are_enforced() { + let (_, bytes, descriptor) = fixture(&config()); + let mut manifest: Value = serde_json::from_slice(&bytes).unwrap(); + manifest["layers"][0]["size"] = json!(MAX_IMAGE_BYTES); + assert!(parse_manifest(&serde_json::to_vec(&manifest).unwrap()).is_err()); + assert!(parse_manifest(&vec![b' '; MAX_MANIFEST_BYTES + 1]).is_err()); + let mut manifest = parse_manifest(&bytes).unwrap(); + let mut conflicting = manifest.layers[0].clone(); + conflicting.size += 1; + manifest.layers.push(conflicting); + assert!(object_closure(descriptor, &manifest).is_err()); +} + +#[test] +fn empty_scratch_image_requires_explicit_main_command() { + let config = ContainerConfig::default(); + assert!(config.argv(None).is_err()); + assert!(config.argv(Some(&[])).is_err()); + assert!(config.argv(Some(&["".into()])).is_err()); + assert_eq!(config.argv(Some(&["/main".into()])).unwrap(), ["/main"]); + assert!(valid_env_key("_A0")); + assert!(!valid_env_key("0A")); + assert!(valid_env_key(&"A".repeat(256))); + assert!(!valid_env_key(&"A".repeat(257))); +} diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index f2fba813fa8..b5817afdbf8 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -204,6 +204,24 @@ pub struct AutoMigratePlan<'def> { } impl AutoMigratePlan<'_> { + /// Function authority changes are part of the published API comparison. + pub fn function_visibility_changes( + &self, + ) -> impl Iterator { + let reducers = self + .old + .reducers() + .filter(|old| old.lifecycle.is_none()) + .filter_map(|old| { + let new = self.new.reducer(&*old.name)?; + (old.visibility != new.visibility).then_some((&*old.name, &old.visibility, &new.visibility)) + }); + let procedures = self.old.procedures().filter_map(|old| { + let new = self.new.procedure(&*old.name)?; + (old.visibility != new.visibility).then_some((&*old.name, &old.visibility, &new.visibility)) + }); + reducers.chain(procedures) + } fn any_step(&self, f: impl Fn(&AutoMigrateStep) -> bool) -> bool { self.steps.iter().any(f) } @@ -467,6 +485,15 @@ pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> prechecks: Vec::new(), }; + let restricts_function_access = plan.function_visibility_changes().any(|(_, old, new)| { + [false, true] + .into_iter() + .any(|owner| old.allows_invocation(false, owner) && !new.allows_invocation(false, owner)) + }); + if restricts_function_access { + plan.ensure_disconnect_all_users(); + } + let views_ok = auto_migrate_views(&mut plan); let tables_ok = auto_migrate_tables(&mut plan); diff --git a/crates/schema/src/auto_migrate/formatter.rs b/crates/schema/src/auto_migrate/formatter.rs index 1f7377209bf..07128e20de5 100644 --- a/crates/schema/src/auto_migrate/formatter.rs +++ b/crates/schema/src/auto_migrate/formatter.rs @@ -17,6 +17,9 @@ use thiserror::Error; pub fn format_plan(f: &mut F, plan: &AutoMigratePlan) -> Result<(), FormattingErrors> { f.format_header()?; + for (name, old, new) in plan.function_visibility_changes() { + f.format_function_visibility(name, old, new)?; + } for step in &plan.steps { format_step(f, step, plan)?; @@ -149,6 +152,12 @@ pub enum Action { /// It allows for different implementations, such as ANSI formatting or plain text formatting. pub trait MigrationFormatter { fn format_header(&mut self) -> io::Result<()>; + fn format_function_visibility( + &mut self, + name: &str, + old: &crate::def::FunctionVisibility, + new: &crate::def::FunctionVisibility, + ) -> io::Result<()>; fn format_add_table(&mut self, table_info: &TableInfo) -> io::Result<()>; fn format_remove_table(&mut self, table_name: &Identifier) -> io::Result<()>; fn format_view(&mut self, view_info: &ViewInfo, action: Action) -> io::Result<()>; diff --git a/crates/schema/src/auto_migrate/termcolor_formatter.rs b/crates/schema/src/auto_migrate/termcolor_formatter.rs index 31807d14b30..bd2e3b60404 100644 --- a/crates/schema/src/auto_migrate/termcolor_formatter.rs +++ b/crates/schema/src/auto_migrate/termcolor_formatter.rs @@ -157,6 +157,15 @@ impl TermColorFormatter { } impl MigrationFormatter for TermColorFormatter { + fn format_function_visibility( + &mut self, + name: &str, + old: &crate::def::FunctionVisibility, + new: &crate::def::FunctionVisibility, + ) -> io::Result<()> { + self.write_bullet(&format!("Function {name} visibility: {old} -> {new}")) + } + fn format_header(&mut self) -> io::Result<()> { let line = "━".repeat(60); self.write_line(&line)?; diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index baae44ed76f..4a3f44a6ede 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -15,7 +15,7 @@ //! After validation, a `ModuleDef` can be converted to the `*Schema` types in `crate::schema` for use in the database. //! (Eventually, we may unify these types...) -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::{self, Debug, Write}; use std::hash::Hash; @@ -37,6 +37,9 @@ use spacetimedb_lib::db::raw_def::v10::{ RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, }; +use spacetimedb_lib::db::raw_def::v11::{ + RawModuleDefV11, RawModuleDefV11Section, RawProcedureDefV11, RawReducerDefV11, +}; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, RawMiscModuleExportV9, RawModuleDefV9, RawProcedureDefV9, RawReducerDefV9, RawRowLevelSecurityDefV9, @@ -163,6 +166,9 @@ pub struct ModuleDef { /// was authored under. #[allow(unused)] raw_module_def_version: RawModuleDefVersion, + + /// Validated module bindings capabilities. Legacy modules have none. + capabilities: BTreeSet, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -171,9 +177,19 @@ pub enum RawModuleDefVersion { V9OrEarlier, /// Represents [`RawModuleDefV10`]. V10, + /// Explicit function visibility and contextual defaults. + V11, } impl ModuleDef { + pub fn supports_hosted_auth_v1(&self) -> bool { + self.capabilities.contains(&RawIdentifier::new("hosted_auth_v1")) + } + + pub fn capabilities(&self) -> impl Iterator { + self.capabilities.iter() + } + /// The raw module definition version this module was authored under. pub fn raw_module_def_version(&self) -> RawModuleDefVersion { self.raw_module_def_version @@ -184,6 +200,22 @@ impl ModuleDef { self.tables.values() } + /// The row type of a table or view, addressed by its canonical name. + pub fn type_ref_for_table_like(&self, name: &str) -> Option { + self.table(name) + .map(|table| table.product_type_ref) + .or_else(|| self.view(name).map(|view| view.product_type_ref)) + } + + /// Serialize without reinterpreting the definition's original version semantics. + pub fn into_raw(self) -> RawModuleDef { + match self.raw_module_def_version { + RawModuleDefVersion::V9OrEarlier => RawModuleDef::V9(self.try_into().expect("same-version conversion")), + RawModuleDefVersion::V10 => RawModuleDef::V10(self.try_into().expect("same-version conversion")), + RawModuleDefVersion::V11 => RawModuleDef::V11(self.into()), + } + } + /// The indexes of the module definition. pub fn indexes(&self) -> impl Iterator { self.tables().flat_map(|table| table.indexes.values()) @@ -469,7 +501,8 @@ impl TryFrom for ModuleDef { RawModuleDef::V8BackCompat(v8_mod) => Self::try_from(v8_mod), RawModuleDef::V9(v9_mod) => Self::try_from(v9_mod), RawModuleDef::V10(v10_mod) => Self::try_from(v10_mod), - _ => unimplemented!(), + RawModuleDef::V11(v11_mod) => Self::try_from(v11_mod), + _ => Err(crate::error::ValidationError::UnsupportedModuleVersion.into()), } } } @@ -489,8 +522,14 @@ impl TryFrom for ModuleDef { validate::v9::validate(v9_mod) } } -impl From for RawModuleDefV9 { - fn from(val: ModuleDef) -> Self { +impl TryFrom for RawModuleDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ModuleDef) -> Result { + if val.raw_module_def_version != RawModuleDefVersion::V9OrEarlier { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } let ModuleDef { tables, views, @@ -506,6 +545,7 @@ impl From for RawModuleDefV9 { http_handlers: _, http_routes: _, raw_module_def_version: _, + capabilities: _, } = val; // Extract column defaults from tables before consuming tables @@ -524,18 +564,26 @@ impl From for RawModuleDefV9 { }) .collect(); - RawModuleDefV9 { + let raw_reducers = reducers + .into_values() + .map(TryInto::try_into) + .collect::>()?; + let raw_procedures = procedures + .into_values() + .map(TryInto::try_into) + .collect::, _>>()?; + Ok(RawModuleDefV9 { tables: to_raw(tables), - reducers: reducers.into_iter().map(|(_, def)| def.into()).collect(), + reducers: raw_reducers, types: to_raw(types), misc_exports: column_defaults .into_iter() - .chain(procedures.into_iter().map(|(_, def)| def.into())) + .chain(raw_procedures) .chain(views.into_iter().map(|(_, def)| def.into())) .collect(), typespace, row_level_security: row_level_security_raw.into_iter().map(|(_, def)| def).collect(), - } + }) } } @@ -547,8 +595,14 @@ impl TryFrom for ModuleDef { } } -impl From for RawModuleDefV10 { - fn from(val: ModuleDef) -> Self { +impl TryFrom for RawModuleDefV10 { + type Error = SchemaConversionError; + fn try_from(val: ModuleDef) -> Result { + if val.raw_module_def_version != RawModuleDefVersion::V10 { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V10, + }); + } let ModuleDef { tables, views, @@ -564,6 +618,7 @@ impl From for RawModuleDefV10 { http_handlers, http_routes, raw_module_def_version: _, + capabilities: _, } = val; let mut sections = Vec::new(); @@ -623,9 +678,9 @@ impl From for RawModuleDefV10 { RawIdentifier::from(rd.accessor_name.clone()), RawIdentifier::from(rd.name.clone()), ); - rd.into() + rd.try_into() }) - .collect(); + .collect::>()?; if !raw_reducers.is_empty() { sections.push(RawModuleDefV10Section::Reducers(raw_reducers)); } @@ -638,9 +693,9 @@ impl From for RawModuleDefV10 { RawIdentifier::from(pd.accessor_name.clone()), RawIdentifier::from(pd.name.clone()), ); - pd.into() + pd.try_into() }) - .collect(); + .collect::>()?; if !raw_procedures.is_empty() { sections.push(RawModuleDefV10Section::Procedures(raw_procedures)); } @@ -698,7 +753,173 @@ impl From for RawModuleDefV10 { // Always emit ExplicitNames so canonical names survive the round-trip. sections.push(RawModuleDefV10Section::ExplicitNames(explicit_names)); - RawModuleDefV10 { sections } + Ok(RawModuleDefV10 { sections }) + } +} + +impl TryFrom for ModuleDef { + type Error = ValidationErrors; + fn try_from(value: RawModuleDefV11) -> Result { + validate::v11::validate(value) + } +} + +impl From for RawModuleDefV11 { + fn from(val: ModuleDef) -> Self { + let ModuleDef { + tables, + views, + reducers, + lifecycle_reducers, + types, + typespace, + stored_in_table_def: _, + typespace_for_generate: _, + refmap: _, + row_level_security_raw, + procedures, + http_handlers, + http_routes, + raw_module_def_version: _, + capabilities, + } = val; + + let mut sections = Vec::new(); + let mut explicit_names = ExplicitNames::default(); + + sections.push(RawModuleDefV11Section::Typespace(typespace)); + + // Extract lifecycle reducer names before consuming reducers. + let raw_lifecycle: Vec = lifecycle_reducers + .into_iter() + .filter_map(|(lifecycle, reducer_id)| { + let id = reducer_id?; + let (name, _) = reducers.get_index(id.idx())?; + Some(RawLifeCycleReducerDefV10 { + lifecycle_spec: lifecycle, + function_name: name.clone().into(), + }) + }) + .collect(); + + let raw_types: Vec = types.into_values().map(Into::into).collect(); + if !raw_types.is_empty() { + sections.push(RawModuleDefV11Section::Types(raw_types)); + } + + // Collect schedules from tables (V10 stores them in a separate section). + // Also collect ExplicitNames for tables: accessor_name → source_name, name → canonical_name. + let mut schedules = Vec::new(); + let raw_tables: Vec = tables + .into_values() + .map(|td| { + // Always emit name as ExplicitNames canonical_name. + explicit_names.insert_table( + RawIdentifier::from(td.accessor_name.clone()), + RawIdentifier::from(td.name.clone()), + ); + if let Some(sched) = td.schedule.clone() { + schedules.push(RawScheduleDefV10 { + source_name: Some(sched.name.into()), + table_name: td.name.clone().into(), + schedule_at_col: sched.at_column, + function_name: sched.function_name.into(), + }); + } + td.into() + }) + .collect(); + if !raw_tables.is_empty() { + sections.push(RawModuleDefV11Section::Tables(raw_tables)); + } + + // Collect ExplicitNames for reducers: accessor_name → source_name, name → canonical_name. + let raw_reducers: Vec = reducers + .into_values() + .map(|rd| { + explicit_names.insert_function( + RawIdentifier::from(rd.accessor_name.clone()), + RawIdentifier::from(rd.name.clone()), + ); + rd.into() + }) + .collect(); + if !raw_reducers.is_empty() { + sections.push(RawModuleDefV11Section::Reducers(raw_reducers)); + } + + // Collect ExplicitNames for procedures: accessor_name → source_name, name → canonical_name. + let raw_procedures: Vec = procedures + .into_values() + .map(|pd| { + explicit_names.insert_function( + RawIdentifier::from(pd.accessor_name.clone()), + RawIdentifier::from(pd.name.clone()), + ); + pd.into() + }) + .collect(); + if !raw_procedures.is_empty() { + sections.push(RawModuleDefV11Section::Procedures(raw_procedures)); + } + + let raw_http_handlers: Vec = http_handlers + .into_values() + .map(|hd| RawHttpHandlerDefV10 { + source_name: hd.accessor_name.into(), + }) + .collect(); + if !raw_http_handlers.is_empty() { + sections.push(RawModuleDefV11Section::HttpHandlers(raw_http_handlers)); + } + + if !http_routes.is_empty() { + let raw_http_routes: Vec = http_routes + .into_iter() + .map(|route| RawHttpRouteDefV10 { + handler_function: route.handler_name.into(), + method: route.method, + path: RawIdentifier::new(route.path.as_ref()), + }) + .collect(); + sections.push(RawModuleDefV11Section::HttpRoutes(raw_http_routes)); + } + + // Collect ExplicitNames for views: accessor_name → source_name, name → canonical_name. + let raw_views: Vec = views + .into_values() + .map(|vd| { + explicit_names.insert_function( + RawIdentifier::from(vd.accessor_name.clone()), + RawIdentifier::from(vd.name.clone()), + ); + vd.into() + }) + .collect(); + if !raw_views.is_empty() { + sections.push(RawModuleDefV11Section::Views(raw_views)); + } + + if !schedules.is_empty() { + sections.push(RawModuleDefV11Section::Schedules(schedules)); + } + + if !raw_lifecycle.is_empty() { + sections.push(RawModuleDefV11Section::LifeCycleReducers(raw_lifecycle)); + } + + let raw_rls: Vec = row_level_security_raw.into_values().collect(); + if !raw_rls.is_empty() { + sections.push(RawModuleDefV11Section::RowLevelSecurity(raw_rls)); + } + + // Always emit ExplicitNames so canonical names survive the round-trip. + sections.push(RawModuleDefV11Section::ExplicitNames(explicit_names)); + + if !capabilities.is_empty() { + sections.push(RawModuleDefV11Section::Capabilities(capabilities.into_iter().collect())); + } + RawModuleDefV11 { sections } } } @@ -1710,9 +1931,36 @@ pub enum FunctionVisibility { /// Callable from client code. ClientCallable, + + /// Callable only by a host-verified internal invocation. + Internal, +} + +impl fmt::Display for FunctionVisibility { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Private => "Private", + Self::ClientCallable => "Public", + Self::Internal => "Internal", + }) + } } impl FunctionVisibility { + pub fn is_client_callable(&self) -> bool { + matches!(self, Self::ClientCallable) + } + pub fn is_internal(&self) -> bool { + matches!(self, Self::Internal) + } + /// Lifecycle event dispatch is a separate restriction from this predicate. + pub fn allows_invocation(&self, is_internal: bool, is_authorized_private_caller: bool) -> bool { + match self { + Self::Internal => is_internal, + Self::Private => is_internal || is_authorized_private_caller, + Self::ClientCallable => true, + } + } pub fn is_private(&self) -> bool { matches!(self, FunctionVisibility::Private) } @@ -1728,11 +1976,40 @@ impl From for FunctionVisibility { } } -impl From for RawFunctionVisibility { - fn from(val: FunctionVisibility) -> Self { +#[derive(Debug, Clone, thiserror::Error)] +#[error("schema cannot be represented as {target:?} without losing function visibility or source-version semantics; request schema version 11")] +pub struct SchemaConversionError { + pub target: RawModuleDefVersion, +} + +impl TryFrom for RawFunctionVisibility { + type Error = SchemaConversionError; + fn try_from(val: FunctionVisibility) -> Result { match val { - FunctionVisibility::Private => RawFunctionVisibility::Private, - FunctionVisibility::ClientCallable => RawFunctionVisibility::ClientCallable, + FunctionVisibility::Private => Ok(Self::Private), + FunctionVisibility::ClientCallable => Ok(Self::ClientCallable), + FunctionVisibility::Internal => Err(SchemaConversionError { + target: RawModuleDefVersion::V10, + }), + } + } +} + +impl From for FunctionVisibility { + fn from(value: raw_def::v11::FunctionVisibility) -> Self { + match value { + raw_def::v11::FunctionVisibility::Private => Self::Private, + raw_def::v11::FunctionVisibility::ClientCallable => Self::ClientCallable, + raw_def::v11::FunctionVisibility::Internal => Self::Internal, + } + } +} +impl From for raw_def::v11::FunctionVisibility { + fn from(value: FunctionVisibility) -> Self { + match value { + FunctionVisibility::Private => Self::Private, + FunctionVisibility::ClientCallable => Self::ClientCallable, + FunctionVisibility::Internal => Self::Internal, } } } @@ -1775,25 +2052,37 @@ pub struct ReducerDef { pub err_return_type: AlgebraicType, } -impl From for RawReducerDefV9 { - fn from(val: ReducerDef) -> Self { - RawReducerDefV9 { +impl TryFrom for RawReducerDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ReducerDef) -> Result { + if val.lifecycle.is_none() && !val.visibility.is_client_callable() { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } + Ok(RawReducerDefV9 { name: val.name.into(), params: val.params, lifecycle: val.lifecycle, - } + }) } } -impl From for RawReducerDefV10 { - fn from(val: ReducerDef) -> Self { - RawReducerDefV10 { +impl TryFrom for RawReducerDefV10 { + type Error = SchemaConversionError; + fn try_from(val: ReducerDef) -> Result { + let visibility = if val.lifecycle.is_some() { + RawFunctionVisibility::Private + } else { + val.visibility.try_into()? + }; + Ok(RawReducerDefV10 { source_name: val.accessor_name.into(), params: val.params, - visibility: val.visibility.into(), + visibility, ok_return_type: val.ok_return_type, err_return_type: val.err_return_type, - } + }) } } @@ -1853,30 +2142,38 @@ pub struct HttpRouteDef { pub path: Box, } -impl From for RawProcedureDefV9 { - fn from(val: ProcedureDef) -> Self { - RawProcedureDefV9 { +impl TryFrom for RawProcedureDefV9 { + type Error = SchemaConversionError; + fn try_from(val: ProcedureDef) -> Result { + if !val.visibility.is_client_callable() { + return Err(SchemaConversionError { + target: RawModuleDefVersion::V9OrEarlier, + }); + } + Ok(RawProcedureDefV9 { name: val.name.into(), params: val.params, return_type: val.return_type, - } + }) } } -impl From for RawProcedureDefV10 { - fn from(val: ProcedureDef) -> Self { - RawProcedureDefV10 { +impl TryFrom for RawProcedureDefV10 { + type Error = SchemaConversionError; + fn try_from(val: ProcedureDef) -> Result { + Ok(RawProcedureDefV10 { source_name: val.accessor_name.into(), params: val.params, return_type: val.return_type, - visibility: val.visibility.into(), - } + visibility: val.visibility.try_into()?, + }) } } -impl From for RawMiscModuleExportV9 { - fn from(def: ProcedureDef) -> Self { - Self::Procedure(def.into()) +impl TryFrom for RawMiscModuleExportV9 { + type Error = SchemaConversionError; + fn try_from(def: ProcedureDef) -> Result { + Ok(Self::Procedure(def.try_into()?)) } } @@ -2144,3 +2441,25 @@ mod tests { == 2)) } } + +impl From for RawReducerDefV11 { + fn from(value: ReducerDef) -> Self { + Self { + source_name: value.accessor_name.into(), + params: value.params, + declared_visibility: Some(value.visibility.into()), + ok_return_type: value.ok_return_type, + err_return_type: value.err_return_type, + } + } +} +impl From for RawProcedureDefV11 { + fn from(value: ProcedureDef) -> Self { + Self { + source_name: value.accessor_name.into(), + params: value.params, + declared_visibility: Some(value.visibility.into()), + return_type: value.return_type, + } + } +} diff --git a/crates/schema/src/def/validate.rs b/crates/schema/src/def/validate.rs index 44829a0e8b8..0b7c5d28c69 100644 --- a/crates/schema/src/def/validate.rs +++ b/crates/schema/src/def/validate.rs @@ -3,6 +3,7 @@ use crate::error::ValidationErrors; pub mod v10; +pub mod v11; pub mod v8; pub mod v9; diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 5ea6370f2d0..7ed1b846c63 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -315,6 +315,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { procedures, http_handlers, http_routes, + capabilities: Default::default(), raw_module_def_version: RawModuleDefVersion::V10, }) } @@ -356,7 +357,7 @@ fn change_scheduled_functions_and_lifetimes_visibility( for red_def in reducers.iter_mut().map(|(_, r)| r) { if red_def.lifecycle.is_some() { - red_def.visibility = crate::def::FunctionVisibility::Private; + red_def.visibility = crate::def::FunctionVisibility::Internal; } } @@ -1293,7 +1294,7 @@ mod tests { def.reducers[&check_deliveries_name].visibility, FunctionVisibility::Private, ); - assert_eq!(def.reducers[&init_name].visibility, FunctionVisibility::Private); + assert_eq!(def.reducers[&init_name].visibility, FunctionVisibility::Internal); assert_eq!( def.reducers[&extra_reducer_name].visibility, FunctionVisibility::ClientCallable diff --git a/crates/schema/src/def/validate/v11.rs b/crates/schema/src/def/validate/v11.rs new file mode 100644 index 00000000000..2bd91e23da1 --- /dev/null +++ b/crates/schema/src/def/validate/v11.rs @@ -0,0 +1,353 @@ +//! V11 reuses V10 structural validation, then resolves declarations after schedules +//! and lifecycle assignments exist. No V11 metadata is decoded as a V10 module. +use super::Result; +use crate::{ + def::{FunctionVisibility, ModuleDef, RawModuleDefVersion}, + error::ValidationError, +}; +use spacetimedb_lib::db::raw_def::{v10, v11}; +use spacetimedb_sats::raw_identifier::RawIdentifier; +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +pub fn validate(def: v11::RawModuleDefV11) -> Result { + let mut seen_sections = HashSet::new(); + let mut declared = BTreeMap::new(); + let mut sections = Vec::new(); + let mut capabilities = BTreeSet::new(); + for section in def.sections { + if !seen_sections.insert(std::mem::discriminant(§ion)) { + return Err(ValidationError::DuplicateModuleSection { + section: format!("{:?}", std::mem::discriminant(§ion)), + } + .into()); + } + if let v11::RawModuleDefV11Section::Capabilities(names) = section { + if names.len() > 32 { + return Err(ValidationError::InvalidModuleCapabilities.into()); + } + for name in names { + if name.is_empty() + || name.len() > 64 + || !name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + || !capabilities.insert(name) + { + return Err(ValidationError::InvalidModuleCapabilities.into()); + } + } + continue; + } + sections.push(match section { + v11::RawModuleDefV11Section::Reducers(rows) => v10::RawModuleDefV10Section::Reducers( + rows.into_iter() + .map(|row| { + insert_declaration(&mut declared, &row.source_name, row.declared_visibility)?; + Ok(v10::RawReducerDefV10 { + source_name: row.source_name, + params: row.params, + visibility: v10::FunctionVisibility::ClientCallable, + ok_return_type: row.ok_return_type, + err_return_type: row.err_return_type, + }) + }) + .collect::>()?, + ), + v11::RawModuleDefV11Section::Procedures(rows) => v10::RawModuleDefV10Section::Procedures( + rows.into_iter() + .map(|row| { + insert_declaration(&mut declared, &row.source_name, row.declared_visibility)?; + Ok(v10::RawProcedureDefV10 { + source_name: row.source_name, + params: row.params, + visibility: v10::FunctionVisibility::ClientCallable, + return_type: row.return_type, + }) + }) + .collect::>()?, + ), + v11::RawModuleDefV11Section::Typespace(value) => v10::RawModuleDefV10Section::Typespace(value), + v11::RawModuleDefV11Section::Types(value) => v10::RawModuleDefV10Section::Types(value), + v11::RawModuleDefV11Section::Tables(value) => v10::RawModuleDefV10Section::Tables(value), + v11::RawModuleDefV11Section::Views(value) => v10::RawModuleDefV10Section::Views(value), + v11::RawModuleDefV11Section::Schedules(value) => v10::RawModuleDefV10Section::Schedules(value), + v11::RawModuleDefV11Section::LifeCycleReducers(value) => { + v10::RawModuleDefV10Section::LifeCycleReducers(value) + } + v11::RawModuleDefV11Section::RowLevelSecurity(value) => { + v10::RawModuleDefV10Section::RowLevelSecurity(value) + } + v11::RawModuleDefV11Section::CaseConversionPolicy(value) => { + v10::RawModuleDefV10Section::CaseConversionPolicy(value) + } + v11::RawModuleDefV11Section::ExplicitNames(value) => v10::RawModuleDefV10Section::ExplicitNames(value), + v11::RawModuleDefV11Section::HttpHandlers(value) => v10::RawModuleDefV10Section::HttpHandlers(value), + v11::RawModuleDefV11Section::HttpRoutes(value) => v10::RawModuleDefV10Section::HttpRoutes(value), + _ => unreachable!("all V11 sections are handled"), + }); + } + let mut module = super::v10::validate(v10::RawModuleDefV10 { sections })?; + for reducer in module.reducers.values_mut() { + let source_name = RawIdentifier::from(reducer.accessor_name.clone()); + let declaration = declared.get(&source_name).copied().flatten(); + if reducer.lifecycle.is_some() { + if declaration.is_some_and(|visibility| visibility != v11::FunctionVisibility::Internal) { + return Err(ValidationError::InvalidLifecycleVisibility { function: source_name }.into()); + } + reducer.visibility = FunctionVisibility::Internal; + } else if let Some(visibility) = declaration { + reducer.visibility = visibility.into(); + } + } + for procedure in module.procedures.values_mut() { + if let Some(visibility) = declared + .get(&RawIdentifier::from(procedure.accessor_name.clone())) + .copied() + .flatten() + { + procedure.visibility = visibility.into(); + } + } + module.raw_module_def_version = RawModuleDefVersion::V11; + module.capabilities = capabilities; + Ok(module) +} + +fn insert_declaration( + declared: &mut BTreeMap>, + name: &RawIdentifier, + visibility: Option, +) -> Result<()> { + if declared.insert(name.clone(), visibility).is_some() { + return Err(ValidationError::DuplicateName { name: name.clone() }.into()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_lib::{db::raw_def::v9, RawModuleDef, ScheduleAt}; + use spacetimedb_sats::{AlgebraicType, ProductType}; + use v11::{FunctionVisibility as Declared, RawModuleDefV11Builder}; + + fn scheduled_module(visibility: Option, procedure: bool) -> ModuleDef { + let mut builder = RawModuleDefV11Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "Jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") + .finish(); + let params = ProductType::from([("job", AlgebraicType::Ref(row))]); + if procedure { + builder.add_procedure_with_visibility("run_job", params, AlgebraicType::unit(), visibility); + } else { + builder.add_reducer_with_visibility("run_job", params, visibility); + } + builder.add_schedule("Jobs", 1, "run_job"); + builder.finish().try_into().unwrap() + } + + #[test] + fn explicit_scheduled_visibility_overrides_the_private_default() { + for procedure in [false, true] { + for (selection, expected) in [ + (None, FunctionVisibility::Private), + (Some(Declared::Private), FunctionVisibility::Private), + (Some(Declared::Internal), FunctionVisibility::Internal), + (Some(Declared::ClientCallable), FunctionVisibility::ClientCallable), + ] { + let module = scheduled_module(selection, procedure); + let visibility = if procedure { + &module.procedure("run_job").unwrap().visibility + } else { + &module.reducer("run_job").unwrap().visibility + }; + assert_eq!(visibility, &expected); + assert_eq!(module.raw_module_def_version(), RawModuleDefVersion::V11); + } + } + } + + #[test] + fn ordinary_defaults_and_lifecycle_restrictions() { + let mut builder = RawModuleDefV11Builder::new(); + builder.add_reducer("ordinary", ProductType::unit()); + builder.add_procedure("ordinary_procedure", ProductType::unit(), AlgebraicType::unit()); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "initialize", ProductType::unit()); + let module: ModuleDef = builder.finish().try_into().unwrap(); + assert!(module.reducer("ordinary").unwrap().visibility.is_client_callable()); + assert!(module + .procedure("ordinary_procedure") + .unwrap() + .visibility + .is_client_callable()); + assert!(module.reducer("initialize").unwrap().visibility.is_internal()); + for selection in [Declared::Private, Declared::ClientCallable] { + let mut builder = RawModuleDefV11Builder::new(); + builder.add_lifecycle_reducer_with_visibility( + v9::Lifecycle::Init, + "initialize", + ProductType::unit(), + Some(selection), + ); + assert!(ModuleDef::try_from(builder.finish()) + .unwrap_err() + .to_string() + .contains("must have Internal visibility")); + } + let mut builder = RawModuleDefV11Builder::new(); + builder.add_lifecycle_reducer_with_visibility( + v9::Lifecycle::Init, + "initialize", + ProductType::unit(), + Some(Declared::Internal), + ); + assert!(ModuleDef::try_from(builder.finish()).is_ok()); + } + + #[test] + fn duplicate_definitions_sections_and_lifecycles_are_rejected() { + let mut builder = RawModuleDefV11Builder::new(); + builder.add_reducer("same", ProductType::unit()); + builder.add_procedure("same", ProductType::unit(), AlgebraicType::unit()); + assert!(ModuleDef::try_from(builder.finish()).is_err()); + let raw = v11::RawModuleDefV11 { + sections: vec![ + v11::RawModuleDefV11Section::Reducers(vec![]), + v11::RawModuleDefV11Section::Reducers(vec![]), + ], + }; + assert!(ModuleDef::try_from(raw) + .unwrap_err() + .to_string() + .contains("repeated V11 section")); + let mut builder = RawModuleDefV11Builder::new(); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "a", ProductType::unit()); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "b", ProductType::unit()); + assert!(ModuleDef::try_from(builder.finish()).is_err()); + } + + #[test] + fn resolved_v11_roundtrips_without_reapplying_defaults_and_rejects_legacy_exports() { + for selection in [None, Some(Declared::Internal), Some(Declared::ClientCallable)] { + let module = scheduled_module(selection, false); + assert!(v9::RawModuleDefV9::try_from(module.clone()).is_err()); + assert!(v10::RawModuleDefV10::try_from(module.clone()).is_err()); + let RawModuleDef::V11(raw) = module.clone().into_raw() else { + panic!("lost source version") + }; + assert!(raw.reducers().all(|reducer| reducer.declared_visibility.is_some())); + let bytes = spacetimedb_lib::bsatn::to_vec(&RawModuleDef::V11(raw)).unwrap(); + let roundtrip: RawModuleDef = spacetimedb_lib::bsatn::from_slice(&bytes).unwrap(); + let roundtrip: ModuleDef = roundtrip.try_into().unwrap(); + assert_eq!( + roundtrip.reducer("run_job").unwrap().visibility, + module.reducer("run_job").unwrap().visibility + ); + assert_eq!(roundtrip.raw_module_def_version(), RawModuleDefVersion::V11); + } + } + + #[test] + fn legacy_v9_schedules_stay_public_and_v10_schedules_stay_private() { + let mut builder = v9::RawModuleDefV9Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index(v9::btree(0), "jobs_id_idx") + .with_schedule("run_job", 1) + .finish(); + builder.add_reducer("run_job", ProductType::from([("job", row.into())]), None); + let v9: ModuleDef = builder.finish().try_into().unwrap(); + assert!(v9.reducer("run_job").unwrap().visibility.is_client_callable()); + assert!(matches!(v9.into_raw(), RawModuleDef::V9(_))); + + let mut builder = v10::RawModuleDefV10Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") + .finish(); + builder.add_reducer("run_job", ProductType::from([("job", row.into())])); + builder.add_schedule("jobs", 1, "run_job"); + let v10: ModuleDef = builder.finish().try_into().unwrap(); + assert!(v10.reducer("run_job").unwrap().visibility.is_private()); + assert!(matches!(v10.into_raw(), RawModuleDef::V10(_))); + } + + #[test] + fn capabilities_are_explicit_bounded_and_preserved() { + let bare: ModuleDef = RawModuleDefV11Builder::new().finish().try_into().unwrap(); + assert!(!bare.supports_hosted_auth_v1()); + let mut builder = RawModuleDefV11Builder::new(); + builder.add_capability("hosted_auth_v1"); + let module: ModuleDef = builder.finish().try_into().unwrap(); + assert!(module.supports_hosted_auth_v1()); + let reloaded: ModuleDef = module.into_raw().try_into().unwrap(); + assert!(reloaded.supports_hosted_auth_v1()); + for names in [ + vec!["".to_string()], + vec!["Uppercase".to_string()], + vec!["with-dash".to_string()], + vec!["a".repeat(65)], + vec!["duplicate".to_string(); 2], + (0..33).map(|i| format!("cap_{i}")).collect(), + ] { + let mut builder = RawModuleDefV11Builder::new(); + for name in names { + builder.add_capability(name); + } + assert!(ModuleDef::try_from(builder.finish()).is_err()); + } + } + + #[test] + fn narrowing_function_visibility_is_a_reported_client_break() { + let module = |visibility| { + let mut builder = RawModuleDefV11Builder::new(); + builder.add_reducer_with_visibility("run_now", ProductType::unit(), Some(visibility)); + ModuleDef::try_from(builder.finish()).unwrap() + }; + let public = module(Declared::ClientCallable); + let internal = module(Declared::Internal); + let plan = crate::auto_migrate::ponder_migrate(&public, &internal).unwrap(); + assert!(plan.breaks_client()); + let display = plan + .pretty_print(crate::auto_migrate::PrettyPrintStyle::NoColor) + .unwrap(); + assert!(display.contains("run_now")); + assert!(display.contains("Internal")); + assert!(!crate::auto_migrate::ponder_migrate(&internal, &public) + .unwrap() + .breaks_client()); + } + + #[test] + fn visibility_authority_is_cumulative_without_elevating_the_owner() { + for (visibility, external, owner, internal) in [ + (FunctionVisibility::Internal, false, false, true), + (FunctionVisibility::Private, false, true, true), + (FunctionVisibility::ClientCallable, true, true, true), + ] { + assert_eq!(visibility.allows_invocation(false, false), external); + assert_eq!(visibility.allows_invocation(false, true), owner); + assert_eq!(visibility.allows_invocation(true, false), internal); + } + } +} diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 618f8e3c9c4..aa5e1099575 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -167,6 +167,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { procedures, http_handlers: IndexMap::new(), http_routes: Vec::new(), + capabilities: Default::default(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, }) } @@ -378,7 +379,11 @@ impl ModuleValidatorV9<'_> { recursive: false, // A ProductTypeDef not stored in a Typespace cannot be recursive. }, lifecycle, - visibility: FunctionVisibility::ClientCallable, + visibility: if lifecycle.is_some() { + FunctionVisibility::Internal + } else { + FunctionVisibility::ClientCallable + }, ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }; diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 33abb0c1866..1ffe44196c9 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -22,6 +22,14 @@ pub type ValidationErrors = ErrorStream; #[derive(thiserror::Error, Debug, PartialOrd, Ord, PartialEq, Eq)] #[non_exhaustive] pub enum ValidationError { + #[error("unsupported module definition version")] + UnsupportedModuleVersion, + #[error("invalid module capabilities: at most 32 unique names of 1..64 lowercase ASCII letters, digits or underscores are allowed")] + InvalidModuleCapabilities, + #[error("lifecycle reducer `{function}` must have Internal visibility")] + InvalidLifecycleVisibility { function: RawIdentifier }, + #[error("module contains repeated V11 section `{section}`")] + DuplicateModuleSection { section: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, #[error("name `{name}` is used for multiple types")] diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index e62b9dc6963..e42b34cf6ee 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -773,7 +773,7 @@ impl TableSchema { .map(|(i, schema)| (ColId::from(i), schema)) .map(|(col_pos, schema)| ColumnSchema { col_pos, ..schema }) .collect(); - let view_primary_key = (module_def.raw_module_def_version() == RawModuleDefVersion::V10) + let view_primary_key = (module_def.raw_module_def_version() != RawModuleDefVersion::V9OrEarlier) .then_some(*primary_key) .flatten(); @@ -914,7 +914,7 @@ impl TableSchema { }; let mut constraints = vec![]; - let view_primary_key = (module_def.raw_module_def_version() == RawModuleDefVersion::V10) + let view_primary_key = (module_def.raw_module_def_version() != RawModuleDefVersion::V9OrEarlier) .then_some(primary_key.map(|pk| ColId::from(meta_cols + pk.idx()))) .flatten(); diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/smoketests/http_routes.rs index 567c3684ac8..62263da685c 100644 --- a/crates/smoketests/tests/smoketests/http_routes.rs +++ b/crates/smoketests/tests/smoketests/http_routes.rs @@ -1172,7 +1172,7 @@ fn assert_http_routes_end_to_end(server_url: &str, identity: &str) { assert_eq!(resp.text().expect("missing route body"), NO_SUCH_ROUTE_BODY); let resp = client - .get(format!("{server_url}/v1/database/{identity}/schema?version=10")) + .get(format!("{server_url}/v1/database/{identity}/schema?version=11")) .header("authorization", "Bearer not-a-jwt") .send() .expect("schema request failed"); diff --git a/crates/standalone/src/subcommands/extract_schema.rs b/crates/standalone/src/subcommands/extract_schema.rs index efc77960195..874f213c183 100644 --- a/crates/standalone/src/subcommands/extract_schema.rs +++ b/crates/standalone/src/subcommands/extract_schema.rs @@ -4,7 +4,7 @@ use anyhow::Context; use clap::{ArgMatches, CommandFactory, FromArgMatches}; use spacetimedb::host::extract_schema; use spacetimedb::messages::control_db; -use spacetimedb_lib::{db::raw_def::v10::RawModuleDefV10, sats, RawModuleDef}; +use spacetimedb_lib::sats; /// Extracts the module schema from a local module file. /// WARNING: This command is UNSTABLE and subject to breaking changes. @@ -67,7 +67,7 @@ pub async fn exec(args: &ArgMatches) -> anyhow::Result<()> { let module_def = extract_schema(program_bytes.into(), host_type.into()).await?; - let raw_def = RawModuleDef::V10(RawModuleDefV10::from(module_def)); + let raw_def = module_def.into_raw(); serde_json::to_writer(std::io::stdout().lock(), &sats::serde::SerdeWrapper(raw_def))?; diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index bf8c305526d..22cbcf943da 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -35,6 +35,8 @@ serde.workspace = true futures.workspace = true [dev-dependencies] +spacetimedb-auth.workspace = true +spacetimedb-datastore.workspace = true serial_test.workspace = true [lints] diff --git a/crates/testing/tests/hosted_invocation.rs b/crates/testing/tests/hosted_invocation.rs new file mode 100644 index 00000000000..7c9b3c5528c --- /dev/null +++ b/crates/testing/tests/hosted_invocation.rs @@ -0,0 +1,261 @@ +//! Exercise Rust Wasm bindings and host admission using actual signed proofs. +use serial_test::serial; +use spacetimedb::auth::hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}; +use spacetimedb::auth::invocation::InvocationCaller; +use spacetimedb::auth::JwtKeys; +use spacetimedb::db::deployment::install_container_fence; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_auth::identity::ConnectionAuthCtx; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + StContainerFenceRow, ST_CLIENT_ID, ST_CONNECTION_AUTH_ID, ST_CONNECTION_CREDENTIALS_ID, +}; +use spacetimedb_lib::sats::{product, AlgebraicValue, ProductValue}; +use spacetimedb_lib::{bsatn, ConnectionId, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::{Duration, SystemTime}; + +fn authenticate(source: Identity, target: Identity, lifetime: Duration) -> ConnectionAuthCtx { + let keys = JwtKeys::generate().unwrap(); + let now = SystemTime::now(); + let binding = HostedTokenBinding { + source_database: source, + target_database: target, + generation: 1, + grant_revision: 1, + lease_expires_at: now + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "test.platform", + &binding, + now, + now + lifetime, + "wasm-integration", + ) + .unwrap(); + HostedTokenValidator::new([("test.platform".into(), keys.public)]) + .unwrap() + .validate_token(&token, target, now, |issuer, requested_source, requested_target| { + (issuer == "test.platform" && requested_source == source && requested_target == target).then_some(binding) + }) + .unwrap() + .into_connection_auth() + .unwrap() +} + +fn install_fence(module: &ModuleHost, source: Identity, generation: u64, allowed: bool) { + let db = module.relational_db(); + db.with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + install_container_fence( + db, + tx, + &StContainerFenceRow { + source_identity: source.into(), + generation, + target_grant_revision: generation, + target_set_hash: spacetimedb_lib::hash_bytes(b"configured targets"), + allowed, + }, + )?; + Ok(()) + }) + .unwrap(); +} + +fn assert_connection_count(module: &ModuleHost, expected: u64) { + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + for table in [ST_CLIENT_ID, ST_CONNECTION_CREDENTIALS_ID, ST_CONNECTION_AUTH_ID] { + assert_eq!(tx.table_row_count(table), Some(expected)); + } + Ok(()) + }) + .unwrap(); +} + +fn arguments(values: ProductValue) -> FunctionArgs { + FunctionArgs::Bsatn(bsatn::to_vec(&values).unwrap().into()) +} + +async fn call( + module: &ModuleHost, + caller: impl Into + Send, + connection: Option, + reducer: &str, + args: ProductValue, +) -> anyhow::Result<()> { + module + .call_reducer(caller, connection, None, None, None, reducer, arguments(args)) + .await? + .outcome + .into_result() +} + +#[test] +#[serial] +fn hosted_wasm_calls_preserve_authority_and_disconnect_after_revocation() { + CompiledModule::compile("hosted-auth-test", CompilationMode::Debug).with_module_async( + DEFAULT_CONFIG, + |handle| async move { + let module = handle.client.module(); + let target = handle.db_identity; + let foreign = Identity::ONE; + let owner = Identity::ZERO; + let self_connection = ConnectionId::from_u128(101); + let foreign_connection = ConnectionId::from_u128(102); + let failed_disconnect = ConnectionId::from_u128(999); + let expired_connection = ConnectionId::from_u128(103); + install_fence(&module, target, 1, true); + install_fence(&module, foreign, 1, true); + let self_auth = authenticate(target, target, Duration::from_secs(30)); + let foreign_auth = authenticate(foreign, target, Duration::from_secs(30)); + let expiring_auth = authenticate(target, target, Duration::from_secs(3)); + for (auth, connection) in [ + (self_auth.clone(), self_connection), + (foreign_auth.clone(), foreign_connection), + (self_auth.clone(), failed_disconnect), + (expiring_auth.clone(), expired_connection), + ] { + module.call_identity_connected(auth, connection).await.unwrap(); + } + assert_connection_count(&module, 4); + for (auth, sender, connection, internal) in [ + (&self_auth, target, self_connection, true), + (&foreign_auth, foreign, foreign_connection, false), + ] { + call( + &module, + auth, + Some(connection), + "inspect_context", + product![sender, Some(connection), internal, true], + ) + .await + .unwrap(); + let result = module + .call_procedure( + auth, + Some(connection), + None, + "inspect_procedure", + arguments(product![sender, Some(connection), internal]), + ) + .await; + assert_eq!(result.result.unwrap().return_val, AlgebraicValue::Bool(true)); + for lifecycle in ["connected", "disconnected"] { + assert!(call(&module, auth, Some(connection), lifecycle, product![]) + .await + .is_err()); + } + } + // Ordinary owner calls and equal database identities remain external. + for ordinary in [target, owner] { + call( + &module, + ordinary, + None, + "inspect_context", + product![ordinary, Option::::None, false, false], + ) + .await + .unwrap(); + assert!(call(&module, ordinary, None, "internal_only", product![]) + .await + .is_err()); + } + call(&module, owner, None, "private_only", product![]).await.unwrap(); + call(&module, &self_auth, Some(self_connection), "internal_only", product![]) + .await + .unwrap(); + call(&module, &self_auth, Some(self_connection), "private_only", product![]) + .await + .unwrap(); + for reducer in ["internal_only", "private_only"] { + assert!( + call(&module, &foreign_auth, Some(foreign_connection), reducer, product![]) + .await + .is_err() + ); + } + call(&module, owner, None, "schedule_check", product![]).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let result = module + .call_procedure(owner, None, None, "scheduled_finished", FunctionArgs::Nullary) + .await; + if result.result.unwrap().return_val == AlgebraicValue::Bool(true) { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("scheduled call did not observe internal authority"); + // Tokens remain cryptographically valid, but their persisted grants are revoked. + install_fence(&module, target, 2, false); + install_fence(&module, foreign, 2, false); + assert!( + call(&module, &self_auth, Some(self_connection), "internal_only", product![]) + .await + .is_err() + ); + let result = module + .call_procedure( + &self_auth, + Some(self_connection), + None, + "inspect_procedure", + arguments(product![target, Some(self_connection), true]), + ) + .await; + assert!(result.result.is_err()); + if let Ok(remaining) = expiring_auth + .hosted + .as_ref() + .unwrap() + .expires_at() + .duration_since(SystemTime::now()) + { + tokio::time::sleep(remaining + Duration::from_millis(20)).await; + } + assert!(call( + &module, + &expiring_auth, + Some(expired_connection), + "internal_only", + product![] + ) + .await + .is_err()); + // Host cleanup retains captured flags and JWT sender after revocation and expiry. + for (sender, connection) in [ + (target, self_connection), + (foreign, foreign_connection), + (target, failed_disconnect), + (target, expired_connection), + ] { + module.call_identity_disconnected(sender, connection).await.unwrap(); + } + assert_connection_count(&module, 0); + for (connection, sender, internal, disconnected) in [ + (self_connection, target, true, true), + (foreign_connection, foreign, false, true), + (failed_disconnect, target, true, false), + (expired_connection, target, true, true), + ] { + call( + &module, + owner, + None, + "inspect_observation", + product![connection, sender, internal, disconnected], + ) + .await + .unwrap(); + } + }, + ); +} diff --git a/modules/hosted-auth-test/Cargo.toml b/modules/hosted-auth-test/Cargo.toml new file mode 100644 index 00000000000..b37b9b44332 --- /dev/null +++ b/modules/hosted-auth-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "hosted-auth-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/hosted-auth-test/src/lib.rs b/modules/hosted-auth-test/src/lib.rs new file mode 100644 index 00000000000..f6380f686eb --- /dev/null +++ b/modules/hosted-auth-test/src/lib.rs @@ -0,0 +1,141 @@ +//! Actual host integration fixture for verified container authentication. +use spacetimedb::{ConnectionId, Identity, ProcedureContext, ReducerContext, Table}; + +#[spacetimedb::table(accessor = observations)] +pub struct Observation { + #[primary_key] + connection: ConnectionId, + sender: Identity, + internal: bool, + jwt_identity: Identity, + disconnected: bool, +} + +#[spacetimedb::reducer(client_connected)] +pub fn connected(ctx: &ReducerContext) { + ctx.db.observations().insert(Observation { + connection: ctx.connection_id().unwrap(), + sender: ctx.sender(), + internal: ctx.sender_auth().is_internal(), + jwt_identity: ctx.sender_auth().jwt().unwrap().identity(), + disconnected: false, + }); +} + +#[spacetimedb::reducer(client_disconnected)] +pub fn disconnected(ctx: &ReducerContext) -> Result<(), String> { + let connection = ctx.connection_id().unwrap(); + let mut observation = ctx.db.observations().connection().find(connection).unwrap(); + assert_eq!(ctx.sender(), observation.sender); + assert_eq!(ctx.sender_auth().is_internal(), observation.internal); + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), observation.jwt_identity); + // Exercise host fallback cleanup after a user callback rejects disconnect. + if connection == ConnectionId::from_u128(999) { + return Err("intentional disconnect failure".into()); + } + observation.disconnected = true; + ctx.db.observations().connection().update(observation); + Ok(()) +} + +#[spacetimedb::reducer] +pub fn inspect_context( + ctx: &ReducerContext, + sender: Identity, + connection: Option, + internal: bool, + jwt: bool, +) { + assert_eq!(ctx.sender(), sender); + assert_eq!(ctx.connection_id(), connection); + assert_eq!(ctx.sender_auth().is_internal(), internal); + assert_eq!(ctx.sender_auth().has_jwt(), jwt); + if jwt { + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), sender); + } +} + +#[spacetimedb::reducer(internal)] +pub fn internal_only(ctx: &ReducerContext) { + assert!(ctx.sender_auth().is_internal()); +} + +#[spacetimedb::reducer(private)] +pub fn private_only(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer] +pub fn inspect_observation( + ctx: &ReducerContext, + connection: ConnectionId, + sender: Identity, + internal: bool, + disconnected: bool, +) { + let observation = ctx.db.observations().connection().find(connection).unwrap(); + assert_eq!(observation.sender, sender); + assert_eq!(observation.jwt_identity, sender); + assert_eq!(observation.internal, internal); + assert_eq!(observation.disconnected, disconnected); +} + +#[spacetimedb::procedure] +pub fn inspect_procedure( + ctx: &mut ProcedureContext, + sender: Identity, + connection: Option, + internal: bool, +) -> bool { + assert_eq!(ctx.sender(), sender); + assert_eq!(ctx.connection_id(), connection); + assert_eq!(ctx.sender_auth().is_internal(), internal); + assert_eq!(ctx.sender_auth().jwt().unwrap().identity(), sender); + ctx.with_tx(|tx| { + assert_eq!(tx.sender(), sender); + assert_eq!(tx.connection_id(), connection); + assert_eq!(tx.sender_auth().is_internal(), internal); + assert_eq!(tx.sender_auth().jwt().unwrap().identity(), sender); + }); + true +} + +#[spacetimedb::table(accessor = scheduled_checks, scheduled(scheduled_check))] +pub struct ScheduledCheck { + #[primary_key] + #[auto_inc] + id: u64, + scheduled_at: spacetimedb::ScheduleAt, +} + +#[spacetimedb::reducer] +pub fn schedule_check(ctx: &ReducerContext) { + ctx.db.scheduled_checks().insert(ScheduledCheck { + id: 0, + scheduled_at: ctx.timestamp.into(), + }); +} + +#[spacetimedb::reducer] +pub fn scheduled_check(ctx: &ReducerContext, _job: ScheduledCheck) { + assert_eq!(ctx.sender(), ctx.database_identity()); + assert_eq!(ctx.connection_id(), None); + assert!(ctx.sender_auth().is_internal()); + assert!(!ctx.sender_auth().has_jwt()); + ctx.db.observations().insert(Observation { + connection: ConnectionId::from_u128(777), + sender: ctx.sender(), + internal: true, + jwt_identity: ctx.sender(), + disconnected: false, + }); +} + +#[spacetimedb::procedure] +pub fn scheduled_finished(ctx: &mut ProcedureContext) -> bool { + ctx.with_tx(|tx| { + tx.db + .observations() + .connection() + .find(ConnectionId::from_u128(777)) + .is_some() + }) +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index b8f0c01d558..831abd5f657 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -510,7 +510,7 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; try { const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=9` + `http://localhost:3000/v1/database/${module_identity}/schema?version=11` ); return response.text(); } catch (e) { diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index 56e6b288e2d..0932377b59d 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -545,7 +545,7 @@ fn with_tx(ctx: &mut ProcedureContext) { fn get_my_schema_via_http(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=9" + "http://localhost:3000/v1/database/{module_identity}/schema?version=11" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => format!("{e}"), diff --git a/modules/sdk-test-procedure-ts/src/index.ts b/modules/sdk-test-procedure-ts/src/index.ts index f89aa76665a..5ab74792193 100644 --- a/modules/sdk-test-procedure-ts/src/index.ts +++ b/modules/sdk-test-procedure-ts/src/index.ts @@ -93,7 +93,7 @@ export const will_panic = spacetimedb.procedure(t.unit(), _ctx => { export const read_my_schema = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=9` + `http://localhost:3000/v1/database/${module_identity}/schema?version=11` ); return response.text(); }); diff --git a/modules/sdk-test-procedure/src/lib.rs b/modules/sdk-test-procedure/src/lib.rs index 95ae9b523b1..a1c5a38efc3 100644 --- a/modules/sdk-test-procedure/src/lib.rs +++ b/modules/sdk-test-procedure/src/lib.rs @@ -44,7 +44,7 @@ fn will_panic(_ctx: &mut ProcedureContext) { fn read_my_schema(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=9" + "http://localhost:3000/v1/database/{module_identity}/schema?version=11" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => panic!("{e}"), diff --git a/sdks/rust/tests/procedure-client/src/test_handlers.rs b/sdks/rust/tests/procedure-client/src/test_handlers.rs index d3f75c0698a..e280ddc2d90 100644 --- a/sdks/rust/tests/procedure-client/src/test_handlers.rs +++ b/sdks/rust/tests/procedure-client/src/test_handlers.rs @@ -1,7 +1,7 @@ use crate::module_bindings::*; use anyhow::Context; use core::time::Duration; -use spacetimedb_lib::db::raw_def::v9::{RawMiscModuleExportV9, RawModuleDefV9}; +use spacetimedb_lib::db::raw_def::v11::{RawModuleDefV11Section, RawModuleDefV11}; use spacetimedb_sdk::{DbConnectionBuilder, DbContext, Table}; use test_counter::TestCounter; @@ -249,7 +249,7 @@ async fn exec_insert_with_tx_rollback(db_name: &str) { /// Test that a procedure can perform an HTTP request and return a string derived from the response. /// /// Invoke the procedure `read_my_schema`, -/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV9`], +/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV11`], /// then (in the client) deserialize the response and assert that it contains a description of that procedure. async fn exec_procedure_http_ok(db_name: &str) { let test_counter = TestCounter::new(); @@ -263,12 +263,12 @@ async fn exec_procedure_http_ok(db_name: &str) { #[allow(clippy::redundant_closure_call)] (|| { anyhow::ensure!(res.is_ok(), "Expected Ok result but got {res:?}"); - let module_def: RawModuleDefV9 = spacetimedb_lib::de::serde::deserialize_from( + let module_def: RawModuleDefV11 = spacetimedb_lib::de::serde::deserialize_from( &mut serde_json::Deserializer::from_str(&res.unwrap()), )?; - anyhow::ensure!(module_def.misc_exports.iter().any(|misc_export| { - if let RawMiscModuleExportV9::Procedure(procedure_def) = misc_export { - &*procedure_def.name == "read_my_schema" + anyhow::ensure!(module_def.sections.iter().any(|section| { + if let RawModuleDefV11Section::Procedures(procedures) = section { + procedures.iter().any(|procedure| &*procedure.source_name == "read_my_schema") } else { false } From 21272fcd23252904f7539daaf83cc3305543af2f Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Mon, 7 Sep 2026 20:52:03 -0400 Subject: [PATCH 02/23] Commit deployment metadata with module changes and expose database environment reads --- Cargo.lock | 7 + Cargo.toml | 1 + crates/auth/src/hosted.rs | 23 +- .../include/spacetimedb/abi/FFI.h | 1 + .../include/spacetimedb/abi/abi.h | 6 + .../include/spacetimedb/bsatn/reader.h | 6 +- .../include/spacetimedb/environment.h | 34 +++ .../include/spacetimedb/procedure_context.h | 3 + .../include/spacetimedb/reducer_context.h | 3 + .../include/spacetimedb/tx_context.h | 2 + .../include/spacetimedb/view_context.h | 4 + .../tests/unit/hosted_auth_unit_tests.cpp | 33 +++ .../diag/snapshots/Module#FFI.verified.cs | 3 + .../snapshots/Module#FFI.verified.cs | 3 + .../server/snapshots/Module#FFI.verified.cs | 3 + crates/bindings-csharp/Codegen/Module.cs | 3 + .../Runtime/DatabaseEnvironment.cs | 26 +++ .../bindings-csharp/Runtime/HandlerContext.cs | 2 + .../bindings-csharp/Runtime/Internal/FFI.cs | 15 ++ .../Runtime/ProcedureContext.cs | 2 + crates/bindings-csharp/Runtime/bindings.c | 4 + crates/bindings-sys/src/lib.rs | 19 ++ .../src/lib/environment.ts | 5 + .../bindings-typescript/src/lib/reducers.ts | 2 + .../src/server/environment.ts | 6 + .../src/server/http_handlers.ts | 2 + .../bindings-typescript/src/server/index.ts | 2 + .../src/server/procedures.ts | 3 + .../bindings-typescript/src/server/runtime.ts | 5 + .../bindings-typescript/src/server/sys.d.ts | 5 + .../bindings-typescript/src/server/views.ts | 3 + .../tests/hosted_auth.test.ts | 1 + crates/bindings/src/lib.rs | 29 +++ crates/bindings/src/rt.rs | 7 + crates/client-api/src/lib.rs | 22 ++ crates/client-api/src/routes/database.rs | 5 + crates/core/src/auth/hosted_tokens.rs | 20 +- crates/core/src/auth/token_validation.rs | 4 +- crates/core/src/db/deployment.rs | 64 ++++++ crates/core/src/db/environment.rs | 131 +++++++++++ crates/core/src/db/mod.rs | 3 +- crates/core/src/error.rs | 4 + crates/core/src/host/empty_module.rs | 47 ++++ crates/core/src/host/empty_module/README.md | 23 ++ crates/core/src/host/empty_module/generate.py | 113 ++++++++++ crates/core/src/host/empty_module/tests.rs | 126 +++++++++++ .../src/host/empty_module/v1.schema.bsatn | Bin 0 -> 33 bytes crates/core/src/host/empty_module/v1.sha256 | 1 + crates/core/src/host/empty_module/v1.wasm | Bin 0 -> 250 bytes crates/core/src/host/host_controller.rs | 146 +++++++++--- .../host/host_controller/deployment_tests.rs | 213 ++++++++++++++++++ crates/core/src/host/instance_env.rs | 126 +++++++++++ crates/core/src/host/mod.rs | 2 + crates/core/src/host/module_host.rs | 115 +++++++++- crates/core/src/host/v8/mod.rs | 41 +++- crates/core/src/host/v8/syscall/mod.rs | 1 + crates/core/src/host/v8/syscall/v2.rs | 18 ++ crates/core/src/host/wasm_common.rs | 3 + .../src/host/wasm_common/module_host_actor.rs | 109 ++++++++- .../src/host/wasmtime/wasm_instance_env.rs | 41 +++- .../core/src/host/wasmtime/wasmtime_module.rs | 2 +- crates/core/src/sql/execute.rs | 12 +- crates/lib/src/container.rs | 6 +- crates/lib/src/deployment.rs | 14 +- crates/lib/src/environment.rs | 53 +++++ crates/lib/src/lib.rs | 3 +- crates/testing/src/modules.rs | 6 + crates/testing/tests/deployment_publish.rs | 204 +++++++++++++++++ crates/testing/tests/environment.rs | 134 +++++++++++ modules/environment-test/Cargo.toml | 13 ++ modules/environment-test/src/lib.rs | 54 +++++ modules/module-test-cpp/src/lib.cpp | 12 + modules/module-test-cs/EnvironmentTests.cs | 25 ++ modules/module-test-ts/src/index.ts | 23 ++ 74 files changed, 2121 insertions(+), 91 deletions(-) create mode 100644 crates/bindings-cpp/include/spacetimedb/environment.h create mode 100644 crates/bindings-csharp/Runtime/DatabaseEnvironment.cs create mode 100644 crates/bindings-typescript/src/lib/environment.ts create mode 100644 crates/bindings-typescript/src/server/environment.ts create mode 100644 crates/core/src/db/environment.rs create mode 100644 crates/core/src/host/empty_module.rs create mode 100644 crates/core/src/host/empty_module/README.md create mode 100644 crates/core/src/host/empty_module/generate.py create mode 100644 crates/core/src/host/empty_module/tests.rs create mode 100644 crates/core/src/host/empty_module/v1.schema.bsatn create mode 100644 crates/core/src/host/empty_module/v1.sha256 create mode 100644 crates/core/src/host/empty_module/v1.wasm create mode 100644 crates/core/src/host/host_controller/deployment_tests.rs create mode 100644 crates/lib/src/environment.rs create mode 100644 crates/testing/tests/deployment_publish.rs create mode 100644 crates/testing/tests/environment.rs create mode 100644 modules/environment-test/Cargo.toml create mode 100644 modules/environment-test/src/lib.rs create mode 100644 modules/module-test-cs/EnvironmentTests.cs diff --git a/Cargo.lock b/Cargo.lock index 1e0387e5b94..f080620b262 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2105,6 +2105,13 @@ dependencies = [ "log", ] +[[package]] +name = "environment-test" +version = "0.0.0" +dependencies = [ + "spacetimedb 2.3.0", +] + [[package]] name = "equivalent" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 2e1417f1322..815bdc7f62c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ members = [ "modules/perf-test", "modules/module-test", "modules/hosted-auth-test", + "modules/environment-test", "templates/basic-rs/spacetimedb", "templates/chat-console-rs/spacetimedb", "modules/sdk-test", diff --git a/crates/auth/src/hosted.rs b/crates/auth/src/hosted.rs index 97e8ec32362..3043ec47952 100644 --- a/crates/auth/src/hosted.rs +++ b/crates/auth/src/hosted.rs @@ -142,8 +142,27 @@ impl VerifiedHostedAuth { /// Classifies the reserved namespace only. A positive result grants no authority. /// All reserved versions are rejected by ordinary OIDC validation and token exchange. pub fn has_reserved_hosted_token_kind(token: &str) -> anyhow::Result { + classify_reserved_token(token, is_reserved_hosted_type, is_reserved_hosted_kind) +} + +/// Operational container proofs are never client Identity credentials. Reserve +/// their entire versioned namespace so a lease/registry proof cannot enter +/// OIDC discovery, ordinary JWT validation, or the Identity token exchange. +pub fn has_reserved_platform_token_kind(token: &str) -> anyhow::Result { + classify_reserved_token( + token, + |kind| is_reserved_hosted_type(kind) || kind.starts_with("spacetimedb-container-"), + |kind| is_reserved_hosted_kind(kind) || kind.starts_with("spacetimedb_container_"), + ) +} + +fn classify_reserved_token( + token: &str, + reserved_type: impl FnOnce(&str) -> bool, + reserved_kind: impl FnOnce(&str) -> bool, +) -> anyhow::Result { let header = decode_header(token)?; - if header.typ.as_deref().is_some_and(is_reserved_hosted_type) { + if header.typ.as_deref().is_some_and(reserved_type) { return Ok(true); } let mut validation = Validation::new(Algorithm::ES256); @@ -156,7 +175,7 @@ pub fn has_reserved_hosted_token_kind(token: &str) -> anyhow::Result { .claims .get("kind") .and_then(serde_json::Value::as_str) - .is_some_and(is_reserved_hosted_kind)) + .is_some_and(reserved_kind)) } pub fn is_reserved_hosted_kind(kind: &str) -> bool { diff --git a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h index d9ce82b6aca..138cbdd6ce6 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/FFI.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/FFI.h @@ -74,6 +74,7 @@ using ::identity; // ===== JWT ===== using ::get_jwt; using ::get_call_auth_flags; +using ::env_get; // ===== Procedure Transactions ===== #ifdef SPACETIMEDB_UNSTABLE_FEATURES diff --git a/crates/bindings-cpp/include/spacetimedb/abi/abi.h b/crates/bindings-cpp/include/spacetimedb/abi/abi.h index 91066510861..136868a2c2c 100644 --- a/crates/bindings-cpp/include/spacetimedb/abi/abi.h +++ b/crates/bindings-cpp/include/spacetimedb/abi/abi.h @@ -42,6 +42,9 @@ #define STDB_IMPORT_10_6(name) \ __attribute__((import_module("spacetime_10.6"), import_name(#name))) extern +#define STDB_IMPORT_10_7(name) \ + __attribute__((import_module("spacetime_10.7"), import_name(#name))) extern + // Import opaque types into global namespace for C compatibility using SpacetimeDB::Status; using SpacetimeDB::TableId; @@ -62,6 +65,9 @@ using SpacetimeDB::ConsoleTimerId; extern "C" { +STDB_IMPORT_10_7(env_get) +Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out); + // Verified invocation authority. Bit 0 is INTERNAL; JWT presence is independent. STDB_IMPORT_10_6(get_call_auth_flags) uint32_t get_call_auth_flags(); diff --git a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h index 5fcba095c76..a61fc3bcc59 100644 --- a/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h +++ b/crates/bindings-cpp/include/spacetimedb/bsatn/reader.h @@ -124,10 +124,10 @@ namespace SpacetimeDB::bsatn { template std::optional read_optional() { uint8_t tag = read_u8(); - if (tag == 0) { - return std::nullopt; - } else if (tag == 1) { + if (tag == 0) { // Some, matching the canonical BSATN option type. return SpacetimeDB::bsatn::deserialize(*this); + } else if (tag == 1) { // None. + return std::nullopt; } else { std::abort(); // Invalid optional tag in BSATN deserialization } diff --git a/crates/bindings-cpp/include/spacetimedb/environment.h b/crates/bindings-cpp/include/spacetimedb/environment.h new file mode 100644 index 00000000000..f2df08234d0 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/environment.h @@ -0,0 +1,34 @@ +#ifndef SPACETIMEDB_ENVIRONMENT_H +#define SPACETIMEDB_ENVIRONMENT_H +#include +#include +#include +#include +#include +#include + +namespace SpacetimeDB { +/// Read-only database environment. Reads use the current transaction, or a +/// short snapshot in a procedure outside a transaction. Values are not cached. +class Environment { +public: + std::optional get(std::string_view key) const { + if (key.empty() || key.size() > 256) LOG_PANIC("invalid environment variable name"); + BytesSource source{0}; + if (FFI::env_get(reinterpret_cast(key.data()), static_cast(key.size()), &source) != Status(0)) + LOG_PANIC("environment read failed"); + if (source == BytesSource{0}) return std::nullopt; + std::array buffer; + std::string value; + for (;;) { + size_t len = buffer.size(); + const auto status = FFI::bytes_source_read(source, buffer.data(), &len); + if ((status != 0 && status != -1) || len > buffer.size()) LOG_PANIC("environment source read failed"); + value.append(reinterpret_cast(buffer.data()), len); + if (status == -1) return value; + if (len == 0) LOG_PANIC("environment source made no progress"); + } + } +}; +} +#endif diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index 8a146a82706..75f6847c580 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -17,6 +17,8 @@ #include #include +#include + namespace SpacetimeDB { /** @@ -60,6 +62,7 @@ struct ProcedureContext { AuthCtx sender_auth_ = AuthCtx::internal(); public: + Environment env; // Timestamp when the procedure was invoked Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/reducer_context.h b/crates/bindings-cpp/include/spacetimedb/reducer_context.h index 41865b14f3a..14a6028c606 100644 --- a/crates/bindings-cpp/include/spacetimedb/reducer_context.h +++ b/crates/bindings-cpp/include/spacetimedb/reducer_context.h @@ -13,6 +13,8 @@ // Include database for DatabaseContext #include +#include + namespace SpacetimeDB { // Enhanced ReducerContext with database access - matches Rust pattern @@ -21,6 +23,7 @@ struct ReducerContext { Identity sender_; public: + Environment env; // Core fields - sender is exposed via sender() like Rust, other fields remain directly accessible std::optional connection_id; Timestamp timestamp; diff --git a/crates/bindings-cpp/include/spacetimedb/tx_context.h b/crates/bindings-cpp/include/spacetimedb/tx_context.h index 1a04ef027e1..c874a22b4ff 100644 --- a/crates/bindings-cpp/include/spacetimedb/tx_context.h +++ b/crates/bindings-cpp/include/spacetimedb/tx_context.h @@ -56,6 +56,7 @@ struct TxContext { // In C++, we explicitly expose references where possible and provide // accessors for fields exposed as methods on ReducerContext. DatabaseContext& db; + const Environment& env; const Timestamp& timestamp; const std::optional& connection_id; @@ -63,6 +64,7 @@ struct TxContext { explicit TxContext(ReducerContext& ctx) : ctx_(ctx), db(ctx.db), + env(ctx.env), timestamp(ctx.timestamp), connection_id(ctx.connection_id) {} diff --git a/crates/bindings-cpp/include/spacetimedb/view_context.h b/crates/bindings-cpp/include/spacetimedb/view_context.h index 3a7fdd577f0..12898664059 100644 --- a/crates/bindings-cpp/include/spacetimedb/view_context.h +++ b/crates/bindings-cpp/include/spacetimedb/view_context.h @@ -6,6 +6,8 @@ #include // For ReadOnlyDatabaseContext #include +#include + namespace SpacetimeDB { /** @@ -40,6 +42,7 @@ struct ViewContext { public: // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; // Constructors ViewContext() = default; @@ -74,6 +77,7 @@ struct ViewContext { struct AnonymousViewContext { // Read-only database access - no mutations allowed ReadOnlyDatabaseContext db; + Environment env; // Constructors AnonymousViewContext() = default; diff --git a/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp b/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp index 914261554e2..cfa432bf30b 100644 --- a/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp +++ b/crates/bindings-cpp/tests/unit/hosted_auth_unit_tests.cpp @@ -38,6 +38,14 @@ extern "C" Status get_jwt(const uint8_t*, BytesSource* out) { return Status{0}; } +extern "C" Status env_get(const uint8_t* key, uint32_t key_len, BytesSource* out) { + payload_offset = 0; + const std::string name(reinterpret_cast(key), key_len); + if (name == "ERROR") return Status{1}; + *out = BytesSource{name == "MISSING" ? 0u : 1u}; + return Status{0}; +} + extern "C" int16_t bytes_source_read(BytesSource, uint8_t* out, size_t* len) { *len = std::min(*len, jwt_payload.size() - payload_offset); std::memcpy(out, jwt_payload.data() + payload_offset, *len); @@ -102,3 +110,28 @@ TEST_CASE(procedure_transactions_preserve_authority_connection_and_sender) { ASSERT_EQ(size_t{1}, flag_reads); } } + +TEST_CASE(environment_preserves_missing_empty_and_all_chunks_without_caching) { + Environment env; + reset_host(0); + ASSERT_TRUE(!env.get("MISSING").has_value()); + ASSERT_EQ(std::string{}, env.get("EMPTY").value()); + jwt_payload = std::string(8192, 'x'); + ASSERT_EQ(jwt_payload, env.get("LARGE").value()); + jwt_payload = std::string("a\0b", 3); + ASSERT_EQ(jwt_payload, env.get("NUL").value()); + jwt_payload = "updated"; + ASSERT_EQ(jwt_payload, env.get("NUL").value()); + ProcedureContext procedure(verified_sender(), Timestamp::from_micros_since_epoch(0), ConnectionId(0)); + ASSERT_EQ(jwt_payload, procedure.env.get("VALUE").value()); + procedure.with_tx([&](TxContext& tx) { ASSERT_EQ(jwt_payload, tx.env.get("VALUE").value()); }); +} + +TEST_CASE(optional_reader_matches_canonical_bsatn_tags_and_preserves_following_bytes) { + const std::vector bytes{1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 'a', 0, 'b', 42}; + bsatn::Reader reader(bytes.data(), bytes.size()); + ASSERT_TRUE(!bsatn::deserialize>(reader).has_value()); + ASSERT_EQ(std::string{}, bsatn::deserialize>(reader).value()); + ASSERT_EQ(std::string("a\0b", 3), bsatn::deserialize>(reader).value()); + ASSERT_EQ(uint8_t{42}, reader.read_u8()); +} diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index dd48f15e495..667077f7b6e 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -649,6 +649,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -896,6 +897,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -909,6 +911,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index 126924858f3..1b4f68886d4 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -51,6 +51,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -279,6 +280,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -292,6 +294,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 3a8d63aabb0..346631b0583 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -493,6 +493,7 @@ public static class Handlers { } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -730,6 +731,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -743,6 +745,7 @@ public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index 4ac8ca7139d..097918f3df5 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -2507,6 +2507,7 @@ public static class Handlers { ))}} } public sealed record ReducerContext : DbContext, Internal.IReducerContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public readonly Identity Sender; public readonly ConnectionId? ConnectionId; public readonly Random Rng; @@ -2709,6 +2710,7 @@ public sealed record ViewContext : DbContext, Internal.I { public Identity Sender { get; } + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal ViewContext(Identity sender, Internal.LocalReadOnly db) @@ -2720,6 +2722,7 @@ internal ViewContext(Identity sender, Internal.LocalReadOnly db) public sealed record AnonymousViewContext : DbContext, Internal.IAnonymousViewContext { + public global::SpacetimeDB.DatabaseEnvironment Env => default; public QueryBuilder From => default; internal AnonymousViewContext(Internal.LocalReadOnly db) diff --git a/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs new file mode 100644 index 00000000000..59ca2c26ec2 --- /dev/null +++ b/crates/bindings-csharp/Runtime/DatabaseEnvironment.cs @@ -0,0 +1,26 @@ +namespace SpacetimeDB; + +/// +/// Read-only database environment. Values are plaintext and accessible to database +/// collaborators. Procedure reads outside a transaction use a short snapshot. +/// +public readonly struct DatabaseEnvironment +{ + internal static readonly DatabaseEnvironment Instance = new(); + + /// Return null for a missing key, or an empty string for a present empty value. + public unsafe string? Get(string key) + { + ArgumentNullException.ThrowIfNull(key); + var bytes = System.Text.Encoding.UTF8.GetBytes(key); + fixed (byte* ptr = bytes) + { + Internal.FFI.env_get(ptr, checked((uint)bytes.Length), out var source); + if (source == Internal.BytesSource.INVALID) + { + return null; + } + return System.Text.Encoding.UTF8.GetString(Internal.Module.Consume(source)); + } + } +} diff --git a/crates/bindings-csharp/Runtime/HandlerContext.cs b/crates/bindings-csharp/Runtime/HandlerContext.cs index 8ad7fe14239..861cd3f803f 100644 --- a/crates/bindings-csharp/Runtime/HandlerContext.cs +++ b/crates/bindings-csharp/Runtime/HandlerContext.cs @@ -7,6 +7,7 @@ namespace SpacetimeDB; public abstract class HandlerContextBase { public Random Rng => txState.Rng; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => txState.Timestamp; // NOTE: The host rejects procedure HTTP requests while a mut transaction is open @@ -90,6 +91,7 @@ public abstract class HandlerTxContextBase(Internal.TxContext inner) : IRefresha void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Timestamp Timestamp => Inner.Timestamp; public Random Rng => Inner.Rng; } diff --git a/crates/bindings-csharp/Runtime/Internal/FFI.cs b/crates/bindings-csharp/Runtime/Internal/FFI.cs index 946d66ad2db..89871c2d706 100644 --- a/crates/bindings-csharp/Runtime/Internal/FFI.cs +++ b/crates/bindings-csharp/Runtime/Internal/FFI.cs @@ -109,6 +109,21 @@ internal static partial class FFI [LibraryImport(StdbNamespace10_6)] public static partial uint get_call_auth_flags(); + const string StdbNamespace10_7 = +#if EXPERIMENTAL_WASM_AOT + "spacetime_10.7" +#else + "bindings" +#endif + ; + + [LibraryImport(StdbNamespace10_7)] + public static unsafe partial CheckedStatus env_get( + byte* key, + uint keyLen, + out BytesSource source + ); + [NativeMarshalling(typeof(Marshaller))] public struct CheckedStatus { diff --git a/crates/bindings-csharp/Runtime/ProcedureContext.cs b/crates/bindings-csharp/Runtime/ProcedureContext.cs index 9c5a197aa1e..bb9fa307992 100644 --- a/crates/bindings-csharp/Runtime/ProcedureContext.cs +++ b/crates/bindings-csharp/Runtime/ProcedureContext.cs @@ -5,6 +5,7 @@ namespace SpacetimeDB; #pragma warning disable STDB_UNSTABLE public abstract class ProcedureContextBase : Internal.IInternalProcedureContext { + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public static Identity Identity => Internal.IProcedureContext.GetIdentity(); public Identity Sender { get; } public ConnectionId? ConnectionId { get; } @@ -104,6 +105,7 @@ public abstract class ProcedureTxContextBase(Internal.TxContext inner) : IRefres void IRefreshableTxContext.Refresh(Internal.TxContext inner) => Refresh(inner); public LocalBase Db => (LocalBase)Inner.Db; + public DatabaseEnvironment Env { get; } = DatabaseEnvironment.Instance; public Identity Sender => Inner.Sender; public ConnectionId? ConnectionId => Inner.ConnectionId; public Timestamp Timestamp => Inner.Timestamp; diff --git a/crates/bindings-csharp/Runtime/bindings.c b/crates/bindings-csharp/Runtime/bindings.c index 1ee1d745f7e..60e376cab61 100644 --- a/crates/bindings-csharp/Runtime/bindings.c +++ b/crates/bindings-csharp/Runtime/bindings.c @@ -139,6 +139,10 @@ IMPORT(Status, datastore_clear, IMPORT(uint32_t, get_call_auth_flags, (void), ()); #undef SPACETIME_MODULE_VERSION +#define SPACETIME_MODULE_VERSION "spacetime_10.7" +IMPORT(Status, env_get, (const uint8_t* key, uint32_t key_len, BytesSource* source), (key, key_len, source)); +#undef SPACETIME_MODULE_VERSION + #ifndef EXPERIMENTAL_WASM_AOT static MonoClass* ffi_class; diff --git a/crates/bindings-sys/src/lib.rs b/crates/bindings-sys/src/lib.rs index c629595ddfb..36ee7f03f28 100644 --- a/crates/bindings-sys/src/lib.rs +++ b/crates/bindings-sys/src/lib.rs @@ -893,6 +893,17 @@ pub mod raw { pub fn get_call_auth_flags() -> u32; } + #[link(wasm_import_module = "spacetime_10.7")] + unsafe extern "C" { + /// Read a UTF-8 environment value. Writes INVALID for a missing key; + /// present empty strings have a valid BytesSource. Returns ordinary errno. + /// Invalid keys return HOST_CALL_FAILURE. NO_SPACE means 256 byte + /// sources remain unconsumed; consume a source before retrying. + /// Calls outside a reducer/view + /// transaction or procedure return NOT_IN_TRANSACTION. + pub fn env_get(key: *const u8, key_len: usize, out: *mut BytesSource) -> u16; + } + /// What strategy does the database index use? /// /// See also: @@ -1504,6 +1515,14 @@ pub fn get_jwt(connection_id: [u8; 16]) -> Option { } } +/// Read a database environment value without exposing the system table. +#[inline] +pub fn env_get(key: &str) -> Option { + let source = unsafe { call(|out| raw::env_get(key.as_ptr(), key.len(), out)) } + .unwrap_or_else(|errno: Errno| panic!("Error reading environment: {errno}")); + (source != raw::BytesSource::INVALID).then_some(source) +} + pub struct RowIter { raw: raw::RowIter, } diff --git a/crates/bindings-typescript/src/lib/environment.ts b/crates/bindings-typescript/src/lib/environment.ts new file mode 100644 index 00000000000..3798cfb43d4 --- /dev/null +++ b/crates/bindings-typescript/src/lib/environment.ts @@ -0,0 +1,5 @@ +/** Read-only database environment access. Missing keys return null; empty values return "". */ +export interface Environment { + /** Reads the current transaction, or a short snapshot outside a procedure transaction. */ + get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index ebf386261f5..6e3bee1c3f5 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -1,3 +1,4 @@ +import type { Environment } from './environment'; import type { DbView } from '../server/db_view'; import type { Random } from '../server/rng'; import type { ConnectionId } from './connection_id'; @@ -109,6 +110,7 @@ export type ReducerCtx = Readonly<{ timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + env: Environment; senderAuth: AuthCtx; newUuidV4(): Uuid; newUuidV7(): Uuid; diff --git a/crates/bindings-typescript/src/server/environment.ts b/crates/bindings-typescript/src/server/environment.ts new file mode 100644 index 00000000000..349e8ffed8c --- /dev/null +++ b/crates/bindings-typescript/src/server/environment.ts @@ -0,0 +1,6 @@ +import { env_get } from 'spacetime:sys@2.3'; +import type { Environment } from '../lib/environment'; + +/** Values are not cached: transaction and procedure reads retain host semantics. */ +export const environment: Environment = Object.freeze({ get: env_get }); +export type { Environment } from '../lib/environment'; diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index a092f513611..0659461a74a 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -1,3 +1,4 @@ +import type { Environment } from '../lib/environment'; import type { Identity } from '../lib/identity'; import type { HttpMethod, @@ -214,6 +215,7 @@ export class Request { export interface HandlerContext { readonly timestamp: Timestamp; readonly http: HttpClient; + readonly env: Environment; readonly identity: Identity; readonly random: Random; withTx(body: (ctx: TransactionCtx) => T): T; diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index 604d9f60ea7..231f6b2f5ea 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -36,3 +36,5 @@ export { export type { HandlerContext, HttpHandlerExport } from './http'; import './polyfills'; // Ensure polyfills are loaded + +export type { Environment } from './environment'; diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index 0867c1dfbc6..0b056b8d660 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -1,3 +1,4 @@ +import { environment, type Environment } from './environment'; import { AlgebraicType, ProductType, @@ -83,6 +84,7 @@ export interface ProcedureOpts { } export interface ProcedureCtx { + readonly env: Environment; readonly sender: Identity; readonly databaseIdentity: Identity; /** @deprecated Use `databaseIdentity` instead. */ @@ -192,6 +194,7 @@ const ProcedureCtxImpl = class ProcedureCtx #random: Random | undefined; #dbView: () => DbView; readonly senderAuth: AuthCtx; + readonly env = environment; constructor( readonly sender: Identity, diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index e6fe0ea1841..bbf17b81c9d 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -1,3 +1,4 @@ +import { environment } from './environment'; import * as _syscalls2_0 from 'spacetime:sys@2.0'; import * as _syscalls2_1 from 'spacetime:sys@2.1'; import * as _syscalls2_2 from 'spacetime:sys@2.2'; @@ -220,6 +221,7 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + readonly env = environment; constructor( sender: Identity, @@ -425,6 +427,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = moduleCtx.views[id]; const ctx: ViewCtx = freeze({ + env: environment, sender: new Identity(sender), // this is the non-readonly DbView, but the typing for the user will be // the readonly one, and if they do call mutating functions it will fail @@ -450,6 +453,7 @@ class ModuleHooksImpl implements ModuleHooks { const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = moduleCtx.anonViews[id]; const ctx: AnonymousViewCtx = freeze({ + env: environment, // this is the non-readonly DbView, but the typing for the user will be // the readonly one, and if they do call mutating functions it will fail // at runtime @@ -520,6 +524,7 @@ const BINARY_READER = new BinaryReader(new Uint8Array()); class HandlerContextImpl implements HandlerContext { + readonly env = environment; #identity: Identity | undefined; #uuidCounter: { value: number } | undefined; #random: Random | undefined; diff --git a/crates/bindings-typescript/src/server/sys.d.ts b/crates/bindings-typescript/src/server/sys.d.ts index 97282ae189e..32addabb9e7 100644 --- a/crates/bindings-typescript/src/server/sys.d.ts +++ b/crates/bindings-typescript/src/server/sys.d.ts @@ -128,3 +128,8 @@ declare module 'spacetime:sys@2.2' { /** Verified invocation flags. Bit 0 is INTERNAL; JWT presence is independent. */ export function get_call_auth_flags(): number; } + +declare module 'spacetime:sys@2.3' { + /** Null means missing; an empty string is a present value. */ + export function env_get(key: string): string | null; +} diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index 6d2b475b177..fb0bd1ccde7 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -1,3 +1,4 @@ +import type { Environment } from '../lib/environment'; import { AlgebraicType, ProductType, @@ -75,11 +76,13 @@ export function makeAnonViewExport< export type ViewCtx = Readonly<{ sender: Identity; db: ReadonlyDbView; + env: Environment; from: QueryBuilder; }>; export type AnonymousViewCtx = Readonly<{ db: ReadonlyDbView; + env: Environment; from: QueryBuilder; }>; diff --git a/crates/bindings-typescript/tests/hosted_auth.test.ts b/crates/bindings-typescript/tests/hosted_auth.test.ts index db7d14d8433..a2c2a3b6774 100644 --- a/crates/bindings-typescript/tests/hosted_auth.test.ts +++ b/crates/bindings-typescript/tests/hosted_auth.test.ts @@ -19,6 +19,7 @@ vi.mock('spacetime:sys@2.0', () => ({ }, })); vi.mock('spacetime:sys@2.1', () => ({})); +vi.mock('spacetime:sys@2.3', () => ({ env_get: () => null })); vi.mock('spacetime:sys@2.2', () => ({ get_call_auth_flags: () => { host.flagReads++; diff --git a/crates/bindings/src/lib.rs b/crates/bindings/src/lib.rs index 2f108f9dd9c..1f47dba6955 100644 --- a/crates/bindings/src/lib.rs +++ b/crates/bindings/src/lib.rs @@ -912,10 +912,29 @@ pub use spacetimedb_bindings_macro::view; pub struct QueryBuilder {} pub use query_builder::{Query, RawQuery}; +/// Read-only access to this database's environment store. +/// +/// Reads use the current transaction. In a procedure outside a transaction, +/// each read uses a short snapshot; use `with_tx` to read related keys together. +/// Values are stored in plaintext and may be read by database collaborators. +#[derive(Clone, Copy, Debug, Default)] +pub struct Environment { + _private: (), +} + +impl Environment { + /// Return None for a missing key and Some("") for a present empty value. + /// Keys must be POSIX environment names of at most 256 bytes. + pub fn get(&self, key: &str) -> Option { + rt::env_get(key) + } +} + /// One of two possible types that can be passed as the first argument to a `#[view]`. /// The other is [`ViewContext`]. /// Use this type if the view does not depend on the caller's identity. pub struct AnonymousViewContext { + pub env: Environment, pub db: LocalReadOnly, pub from: QueryBuilder, } @@ -923,6 +942,7 @@ pub struct AnonymousViewContext { impl Default for AnonymousViewContext { fn default() -> Self { Self { + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -932,6 +952,7 @@ impl Default for AnonymousViewContext { /// The other is [`AnonymousViewContext`]. /// Use this type if the view depends on the caller's identity. pub struct ViewContext { + pub env: Environment, sender: Identity, pub db: LocalReadOnly, pub from: QueryBuilder, @@ -941,6 +962,7 @@ impl ViewContext { pub fn new(sender: Identity) -> Self { Self { sender, + env: Environment::default(), db: LocalReadOnly {}, from: QueryBuilder {}, } @@ -971,6 +993,8 @@ impl ViewContext { /// Implements the `DbContext` trait for accessing views into a database. #[non_exhaustive] pub struct ReducerContext { + /// Read-only access to the database environment in this transaction. + pub env: Environment, /// The `Identity` of the client that invoked the reducer. sender: Identity, @@ -1035,6 +1059,7 @@ impl ReducerContext { #[doc(hidden)] pub fn __dummy() -> Self { Self { + env: Environment::default(), db: Local {}, sender: Identity::__dummy(), timestamp: Timestamp::UNIX_EPOCH, @@ -1061,6 +1086,7 @@ impl ReducerContext { sender_auth: AuthCtx, ) -> Self { Self { + env: Environment::default(), db, sender, timestamp, @@ -1268,6 +1294,8 @@ fn with_tx( #[non_exhaustive] #[cfg(feature = "unstable")] pub struct ProcedureContext { + /// Read-only access to the database environment. + pub env: Environment, /// The `Identity` of the client that invoked the procedure. sender: Identity, @@ -1302,6 +1330,7 @@ impl ProcedureContext { timestamp, connection_id, sender_auth: AuthCtx::from_invocation(sender, connection_id), + env: Environment::default(), http: http::HttpClient {}, #[cfg(feature = "rand08")] rng: std::cell::OnceCell::new(), diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index e66b70cd96c..e73c1340a7e 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1347,6 +1347,13 @@ pub fn get_jwt(connection_id: ConnectionId) -> Option { Some(std::str::from_utf8(&buf).unwrap().to_string()) } +pub(crate) fn env_get(key: &str) -> Option { + let source = sys::env_get(key)?; + let mut buf = IterBuf::take(); + read_bytes_source_into(source, &mut buf); + Some(String::from_utf8(buf.to_vec()).expect("host environment values are UTF-8")) +} + /// Read `source` from the host fully into `buf`. pub(crate) fn read_bytes_source_into(source: BytesSource, buf: &mut Vec) { const INVALID: i16 = NO_SUCH_BYTES as i16; diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index b80799e7c83..2495d01bc06 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -213,6 +213,28 @@ impl Host { .update_module_host(database, host_type, self.replica_id, program_bytes, policy) .await } + + /// Used only by an authenticated publication coordinator after control + /// admission and quiescing. This does not authorize or start a container. + pub async fn update_with_deployment( + &self, + database: Database, + host_type: HostType, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + deployment: spacetimedb::db::deployment::DeploymentCommit, + ) -> anyhow::Result { + self.host_controller + .update_module_host_with_deployment( + database, + host_type, + self.replica_id, + program_bytes, + policy, + Some(deployment), + ) + .await + } } /// Parameters for publishing a database. /// diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index c5ec4f9dc8d..e57830bbbc0 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -1081,6 +1081,11 @@ pub async fn publish( | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { tx_offset, durable_offset, + } + | UpdateDatabaseResult::DeploymentAlreadyCommitted { + tx_offset, + durable_offset, + .. }, ) => { timeout(confirmation_timeout.min(MAX_UPDATE_CONFIRMATION_TIMEOUT), async { diff --git a/crates/core/src/auth/hosted_tokens.rs b/crates/core/src/auth/hosted_tokens.rs index f1e6c356460..77622b2706a 100644 --- a/crates/core/src/auth/hosted_tokens.rs +++ b/crates/core/src/auth/hosted_tokens.rs @@ -3,8 +3,8 @@ use anyhow::{ensure, Context}; use jsonwebtoken::DecodingKey; pub use spacetimedb_auth::hosted::{ - has_reserved_hosted_token_kind, sign_hosted_token, HostedTokenBinding, HostedTokenClaims, VerifiedHostedAuth, HOSTED_TOKEN_KIND, HOSTED_TOKEN_TYPE, - MAX_HOSTED_TOKEN_LIFETIME, + has_reserved_hosted_token_kind, sign_hosted_token, HostedTokenBinding, HostedTokenClaims, VerifiedHostedAuth, + HOSTED_TOKEN_KIND, HOSTED_TOKEN_TYPE, MAX_HOSTED_TOKEN_LIFETIME, }; use spacetimedb_auth::hosted::{unverified_hosted_token_claims, verify_hosted_token}; use spacetimedb_lib::Identity; @@ -258,7 +258,7 @@ mod tests { } #[tokio::test] - async fn ordinary_validation_rejects_reserved_hosted_kinds_and_types() { + async fn ordinary_validation_rejects_reserved_platform_kinds_and_types() { let (keys, _, binding, _) = fixture(); let now = SystemTime::now(); let binding = HostedTokenBinding { @@ -281,8 +281,20 @@ mod tests { claims.as_object_mut().unwrap().remove("kind"); header.typ = Some("spacetimedb-hosted-v2+jwt".into()); }), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_container_lease_v1"); + header.typ = Some("JWT".into()); + }), + change(&token, &keys, |claims, header| { + claims.as_object_mut().unwrap().remove("kind"); + header.typ = Some("spacetimedb-container-lease+jwt".into()); + }), + change(&token, &keys, |claims, header| { + claims["kind"] = json!("spacetimedb_container_registry_future"); + header.typ = Some("JWT".into()); + }), ] { - assert!(has_reserved_hosted_token_kind(&reserved).unwrap()); + assert!(spacetimedb_auth::hosted::has_reserved_platform_token_kind(&reserved).unwrap()); assert!(keys.public.validate_token(&reserved).await.is_err()); assert!(ordinary.validate_token(&reserved).await.is_err()); } diff --git a/crates/core/src/auth/token_validation.rs b/crates/core/src/auth/token_validation.rs index 77aca274baf..e165be43ced 100644 --- a/crates/core/src/auth/token_validation.rs +++ b/crates/core/src/auth/token_validation.rs @@ -258,9 +258,9 @@ fn get_raw_issuer(token: &str) -> Result, TokenValidationError> { } fn reject_reserved_hosted_credentials(token: &str) -> Result<(), TokenValidationError> { - if spacetimedb_auth::hosted::has_reserved_hosted_token_kind(token)? { + if spacetimedb_auth::hosted::has_reserved_platform_token_kind(token)? { return Err(anyhow::anyhow!( - "hosted credentials require dedicated target-bound validation and cannot be exchanged" + "platform container credentials require their dedicated validator and cannot be exchanged" ) .into()); } diff --git a/crates/core/src/db/deployment.rs b/crates/core/src/db/deployment.rs index 3c794194c93..5b6285577b8 100644 --- a/crates/core/src/db/deployment.rs +++ b/crates/core/src/db/deployment.rs @@ -22,6 +22,12 @@ use spacetimedb_sats::AlgebraicValue; #[derive(Debug, thiserror::Error)] pub enum DeploymentError { + #[error("this database requires the deployment publication protocol")] + CoordinatorRequired, + #[error("the supplied module does not match the prepared deployment")] + ProgramMismatch, + #[error("the prepared module must advertise hosted_auth_v1 to attach a container")] + UnsupportedHostedModule, #[error("the publication coordinator no longer owns the database fence")] PublicationFenced, #[error("the expected deployment revision does not match the database")] @@ -86,6 +92,64 @@ pub enum CommitAdmission { Ready, } +/// Check the actual program selected by the host, not a caller-provided +/// capability bit. The program bytes are hashed here because `Program` also +/// has a public constructor which accepts a previously computed hash. +pub fn validate_deployment_program( + request: &DeploymentCommit, + program: &spacetimedb_datastore::traits::Program, + module: &spacetimedb_schema::def::ModuleDef, +) -> Result<(), DeploymentError> { + use spacetimedb_datastore::system_tables::ModuleKind; + use spacetimedb_lib::deployment::{ModuleComponent, UserModuleKind}; + if hash_bytes(&program.bytes) != program.hash { + return Err(DeploymentError::ProgramMismatch); + } + match &request.deployment.current().module { + ModuleComponent::User(expected) + if expected.program_hash == program.hash + && matches!( + (expected.kind, program.kind), + (UserModuleKind::Wasm, ModuleKind::WASM) | (UserModuleKind::Js, ModuleKind::JS) + ) => {} + ModuleComponent::SystemEmpty(version) if crate::host::empty_module::matches_program(*version, program) => {} + _ => return Err(DeploymentError::ProgramMismatch), + } + if request.deployment.current().container.is_some() && !module.supports_hosted_auth_v1() { + return Err(DeploymentError::UnsupportedHostedModule); + } + Ok(()) +} + +/// Legacy raw module publication must be routed through the coordinator once +/// a database has deployment metadata or a publication fence, including after +/// an attempt aborts or its container is removed. In particular a legacy +/// request cannot race the first prepared container publication. +/// Call while holding the transaction that changes the program/schema. +pub fn require_unmanaged_publication(tx: &MutTx) -> Result<(), DeploymentError> { + if current_deployment(tx)?.is_some() || singleton(tx, ST_PUBLISH_FENCE_ID)?.is_some() { + return Err(DeploymentError::CoordinatorRequired); + } + Ok(()) +} + +/// Foreign callers depend on the receiving module's bindings just as self +/// callers do. A module replacement cannot silently erase that capability +/// while an admitted generation can still address the database. +pub fn validate_active_hosted_grants( + tx: &MutTx, + module: &spacetimedb_schema::def::ModuleDef, +) -> Result<(), DeploymentError> { + if !module.supports_hosted_auth_v1() { + for row in tx.iter(ST_CONTAINER_FENCE_ID)? { + if StContainerFenceRow::try_from(row)?.allowed { + return Err(DeploymentError::UnsupportedHostedModule); + } + } + } + Ok(()) +} + fn singleton( state: &S, table: TableId, diff --git a/crates/core/src/db/environment.rs b/crates/core/src/db/environment.rs new file mode 100644 index 00000000000..87b7a7231f7 --- /dev/null +++ b/crates/core/src/db/environment.rs @@ -0,0 +1,131 @@ +//! Dedicated access to the private environment store. +//! +//! Mutation callers must authorize owner/admin access before calling these +//! helpers and commit through the normal module transaction machinery so +//! dependent views refresh. Helpers never acquire a second transaction. + +use super::relational_db::{MutTx, RelationalDB}; +use crate::error::DBError; +use spacetimedb_datastore::error::DatastoreError; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{StEnvFields, StEnvRow, ST_ENV_ID}; +use spacetimedb_lib::environment::{validate_key, validate_value, EnvironmentValidationError, MAX_ENV_VARS}; +use spacetimedb_sats::AlgebraicValue; +use std::collections::BTreeMap; + +#[derive(Debug, thiserror::Error)] +pub enum EnvironmentError { + #[error(transparent)] + Validation(#[from] EnvironmentValidationError), + #[error(transparent)] + Datastore(#[from] DatastoreError), + #[error(transparent)] + Database(#[from] DBError), +} + +/// Read from exactly the caller's snapshot, preserving missing versus empty. +pub fn get(state: &impl StateView, key: &str) -> Result, EnvironmentError> { + validate_key(key)?; + state + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| Ok(StEnvRow::try_from(row)?.value)) + .transpose() +} + +pub fn snapshot(state: &impl StateView) -> Result, EnvironmentError> { + state + .iter(ST_ENV_ID)? + .map(|row| { + let row = StEnvRow::try_from(row)?; + Ok((row.key, row.value)) + }) + .collect() +} + +/// Insert or replace one key. Validation occurs before any mutation. +pub fn set(db: &RelationalDB, tx: &mut MutTx, key: &str, value: &str) -> Result<(), EnvironmentError> { + validate_key(key)?; + validate_value(value)?; + let previous = get(tx, key)?; + if previous.is_none() && tx.table_row_count(ST_ENV_ID).unwrap_or(0) >= MAX_ENV_VARS as u64 { + return Err(EnvironmentValidationError::TooManyVariables.into()); + } + if previous.as_deref() == Some(value) { + return Ok(()); + } + delete(db, tx, key)?; + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: key.into(), + value: value.into(), + }, + )?; + Ok(()) +} + +pub fn delete(db: &RelationalDB, tx: &mut MutTx, key: &str) -> Result { + validate_key(key)?; + let pointer = tx + .iter_by_col_eq(ST_ENV_ID, StEnvFields::Key, &AlgebraicValue::String(key.into()))? + .next() + .map(|row| row.pointer()); + if let Some(pointer) = pointer { + db.delete(tx, ST_ENV_ID, [pointer]); + return Ok(true); + } + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use spacetimedb_datastore::execution_context::Workload; + + #[test] + fn missing_empty_nul_update_and_rollback() { + let db = TestDB::in_memory().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + assert_eq!(get(tx, "EMPTY")?, None); + set(&db, tx, "EMPTY", "")?; + set(&db, tx, "NUL", "a\0b")?; + assert_eq!(get(tx, "EMPTY")?, Some(String::new())); + assert_eq!(get(tx, "NUL")?, Some("a\0b".into())); + Ok(()) + }) + .unwrap(); + let result = db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + set(&db, tx, "EMPTY", "changed")?; + delete(&db, tx, "NUL")?; + Err(EnvironmentValidationError::InvalidKey.into()) + }); + assert!(result.is_err()); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!( + snapshot(tx).unwrap(), + BTreeMap::from([("EMPTY".into(), "".into()), ("NUL".into(), "a\0b".into())]) + ); + }); + } + + #[test] + fn capacity_and_value_limits_precede_mutation() { + let db = TestDB::in_memory().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> Result<(), EnvironmentError> { + for i in 0..MAX_ENV_VARS { + set(&db, tx, &format!("K{i}"), "")?; + } + assert!(set(&db, tx, "EXTRA", "").is_err()); + set(&db, tx, "K0", "updated")?; + assert!(set(&db, tx, "K0", &"x".repeat(8193)).is_err()); + assert_eq!(get(tx, "K0")?.as_deref(), Some("updated")); + assert!(delete(&db, tx, "K1")?); + assert!(!delete(&db, tx, "MISSING")?); + set(&db, tx, "EXTRA", "")?; + Ok(()) + }) + .unwrap(); + } +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 86686e6a133..e206ffba2fa 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -9,8 +9,9 @@ use crate::subscription::ExecutionCounters; use spacetimedb_datastore::execution_context::WorkloadType; use spacetimedb_datastore::{locking_tx_datastore::datastore::TxMetrics, traits::TxData}; -mod durability; pub mod deployment; +mod durability; +pub mod environment; pub mod persistence; pub mod relational_db; pub mod snapshot; diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 5eaa9e2f02b..596c30b2c68 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -237,6 +237,10 @@ pub enum LogReplayError { #[derive(Error, Debug)] pub enum NodesError { + #[error("invalid environment variable name")] + InvalidEnvironmentKey, + #[error("too many outstanding byte sources for environment read")] + EnvironmentSourceLimit, #[error("hosted invocation rejected: {0}")] HostedInvocationRejected(String), #[error("Failed to decode row: {0}")] diff --git a/crates/core/src/host/empty_module.rs b/crates/core/src/host/empty_module.rs new file mode 100644 index 00000000000..cfd89fbe29d --- /dev/null +++ b/crates/core/src/host/empty_module.rs @@ -0,0 +1,47 @@ +//! Versioned built-in module for a database published with only a container. +//! +//! This is a real Wasm program, not an empty byte string. It declares an empty +//! V11 user schema and hosted_auth_v1, so normal database initialization, system +//! tables, subscriptions, and later migration use the existing host machinery. +//! Its required reducer ABI entry point traps because no reducer is declared. +//! +//! Version 1 bytes are immutable. Reproduce them with +//! `python3 crates/core/src/host/empty_module/generate.py --check` from public/. +//! New program bytes require a new version, preserving old deployment replay. + +use spacetimedb_datastore::{system_tables::ModuleKind, traits::Program}; +use spacetimedb_lib::{hash_bytes, Hash}; +use std::sync::OnceLock; + +pub const VERSION_1: u32 = 1; +pub use spacetimedb_lib::deployment::SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH as VERSION_1_PROGRAM_HASH; + +pub const VERSION_1_BYTES: &[u8] = include_bytes!("empty_module/v1.wasm"); + +/// Return the exact bundled program for a recognized system module version. +/// Unknown versions fail closed instead of silently selecting the latest one. +pub fn program(version: u32) -> Option { + (version == VERSION_1 && v1_hash() == VERSION_1_PROGRAM_HASH).then(|| Program { + hash: v1_hash(), + bytes: VERSION_1_BYTES.into(), + kind: ModuleKind::WASM, + }) +} + +/// Validate a system-empty deployment against immutable platform bytes, not +/// against an empty-looking schema supplied by a publisher or a claimed hash. +pub fn matches_program(version: u32, candidate: &Program) -> bool { + version == VERSION_1 + && candidate.kind == ModuleKind::WASM + && candidate.hash == VERSION_1_PROGRAM_HASH + && v1_hash() == VERSION_1_PROGRAM_HASH + && candidate.bytes.as_ref() == VERSION_1_BYTES +} + +fn v1_hash() -> Hash { + static HASH: OnceLock = OnceLock::new(); + *HASH.get_or_init(|| hash_bytes(VERSION_1_BYTES)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/host/empty_module/README.md b/crates/core/src/host/empty_module/README.md new file mode 100644 index 00000000000..4b9b15057d7 --- /dev/null +++ b/crates/core/src/host/empty_module/README.md @@ -0,0 +1,23 @@ +# System empty module, version 1 + +The host uses this real Wasm module for a database published with a container and no user module. It declares V11 with an empty typespace and `hosted_auth_v1`; there are no user tables, reducers, procedures, views, schedules, or HTTP handlers. + +`generate.py` is the complete source. It uses only the Python standard library and explicitly encodes the Wasm sections and BSATN description. It produces a 250-byte Wasm binary with one fixed memory page, a schema describer, and the required reducer ABI entry point. The reducer entry point always traps because no reducer is declared. The `spacetime_10.6::get_call_auth_flags` import requires the host ABI corresponding to the advertised capability; the empty module has no invocation contexts to construct. + +From the public repository, reproduce and verify the checked-in files with: + +```sh +python3 crates/core/src/host/empty_module/generate.py --check +``` + +Omit `--check` to regenerate the three files. This does not require a Rust module compiler, WASI toolchain, WAT compiler, or container image builder. + +The SHA-256 checksum of `v1.wasm` is recorded in `v1.sha256`. SpacetimeDB's separate Keccak-256 program identity is: + +```text +9b6cf2db3644c1d321d97ae0fcd4ab3fc065fc94f54a49f61c7a1dfa40a02612 +``` + +Version 1 is immutable. A change to its schema or Wasm requires a new version and new files. Keep version 1 available for existing deployments and replay. The Rust helper checks the known program hash as well as the exact bytes, kind, and version, so a different publisher-supplied module with an empty-looking schema cannot qualify as the system empty module. `Program::empty` is also rejected. + +Core tests compare the BSATN fixture against the current V11 Rust wire types, load the Wasm through the actual host, and initialize a real database through `HostController`. They verify that initialization stores the bundled program and the database's metadata without invoking a user reducer. diff --git a/crates/core/src/host/empty_module/generate.py b/crates/core/src/host/empty_module/generate.py new file mode 100644 index 00000000000..f7eb227023a --- /dev/null +++ b/crates/core/src/host/empty_module/generate.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Reproduce the version-1 system empty module using only Python's standard library. + +The binary has one page of memory, a V11 schema describer, and a reducer entry +point which always traps. There are no declared user tables or functions. + +Equivalent code (the data payload below is BSATN, not WebAssembly encoding): + (import "spacetime_10.0" "bytes_sink_write" (func (param i32 i32 i32) (result i32))) + (import "spacetime_10.6" "get_call_auth_flags" (func (result i32))) + (memory (export "memory") 1 1) + (func (export "__describe_module__") (param i32) + local.get 0 i32.const 32 i32.const 16 call 0 + if unreachable end) + (func (export "__call_reducer__") + (param i32 i64 i64 i64 i64 i64 i64 i64 i32 i32) (result i32) + unreachable) + +The unused 10.6 import explicitly requires the host ABI which supports captured +invocation flags. No function or invocation context exists in this module. +""" + +import argparse +import hashlib +from pathlib import Path +import struct + + +def leb(value): + encoded = bytearray() + while value >= 128: + encoded.append((value & 127) | 128) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def string(value): + value = value.encode("utf-8") + return leb(len(value)) + value + + +def vector(items): + return leb(len(items)) + b"".join(items) + + +def section(tag, payload): + return bytes([tag]) + leb(len(payload)) + payload + + +def function_type(params, results): + return b"\x60" + vector(params) + vector(results) + + +def generate(): + u32 = lambda value: struct.pack("!#HE~N#) literal 0 HcmV?d00001 diff --git a/crates/core/src/host/empty_module/v1.sha256 b/crates/core/src/host/empty_module/v1.sha256 new file mode 100644 index 00000000000..1a265014eb8 --- /dev/null +++ b/crates/core/src/host/empty_module/v1.sha256 @@ -0,0 +1 @@ +5448d1205c0915f7c169368dd27dc5fbe57fea11acc349a866b2fc755b5a9c9a v1.wasm diff --git a/crates/core/src/host/empty_module/v1.wasm b/crates/core/src/host/empty_module/v1.wasm new file mode 100644 index 0000000000000000000000000000000000000000..5a90221beb3df142c951db962f45220d68d8ee38 GIT binary patch literal 250 zcmY+9L2JV>427TNY=edcLw6qSA1En>>; /// The registry of all running hosts. type Hosts = Arc>>; +#[cfg(test)] +mod deployment_tests; + +#[cfg(test)] +static FAIL_NEXT_DEPLOYMENT_ACTIVATION: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); + pub type ExternalDurability = (Arc>, DiskSizeFn); #[async_trait] pub trait ExternalStorage: Send + Sync + 'static { async fn lookup(&self, program_hash: Hash) -> anyhow::Result>>; + + /// Resolve the authorized initial deployment from the same control + /// transaction that created the database. A failed lookup must return an + /// error, never None: None is reserved for legacy module-only databases. + /// Called only when no initialized program exists in the user database. + async fn initial_deployment(&self, _database: &Database) -> anyhow::Result> { + Ok(None) + } } #[async_trait] impl ExternalStorage for F @@ -395,6 +411,23 @@ impl HostController { replica_id: u64, program_bytes: Box<[u8]>, policy: MigrationPolicy, + ) -> anyhow::Result { + self.update_module_host_with_deployment(database, host_type, replica_id, program_bytes, policy, None) + .await + } + + /// Authorized coordinator entry point. The deployment and its operation + /// receipt are committed in the module migration transaction. Process + /// startup and the control database mirror remain separate reconciliation. + #[allow(clippy::too_many_arguments)] + pub async fn update_module_host_with_deployment( + &self, + database: Database, + host_type: HostType, + replica_id: u64, + program_bytes: Box<[u8]>, + policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { let program = Program::from_bytes(host_type.into(), program_bytes); trace!( @@ -438,20 +471,34 @@ impl HostController { host } }; + let mut database_committed = false; let update_result = host .update_module( this.runtimes.clone(), program, policy, + deployment, this.energy_monitor.clone(), this.unregister_fn(replica_id), this.db_cores.take(), + &mut database_committed, ) - .await?; + .await; - *guard = Some(host); + if update_result.is_err() && database_committed { + // Schema/program/receipt already committed. The previous + // executable cannot be retained after activation failure. + // Close clients/scheduler and reconstruct from stored program + // on the next leader lookup or reconciliation attempt. + let module = host.module.borrow().clone(); + module.exit().await; + host.replica_ctx.relational_db().shutdown().await; + drop(host); + } else { + *guard = Some(host); + } - Ok::<_, anyhow::Error>(update_result) + update_result }) .await??; @@ -813,30 +860,24 @@ impl ModuleLauncher { /// If the `db` is not initialized yet (i.e. its program hash is `None`), /// return an error. /// -/// Otherwise, if `db.program_hash` matches the given `program_hash`, do -/// nothing and return an empty `UpdateDatabaseResult`. -/// -/// Otherwise, invoke `module.update_database` and return the result. +/// Admission, including unchanged programs and idempotent deployment retries, +/// is serialized inside the migration transaction. async fn update_module( db: &RelationalDB, module: &ModuleHost, program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { let addr = db.database_identity(); match stored_program_hash(db)? { None => Err(anyhow!("database `{addr}` not yet initialized")), Some(stored) => { - let res = if stored == program.hash { - info!("database `{}` up to date with program `{}`", addr, program.hash); - UpdateDatabaseResult::NoUpdateNeeded - } else { - info!("updating `{}` from {} to {}", addr, stored, program.hash); - module.update_database(program, old_module_info, policy).await? - }; - - Ok(res) + info!("publishing `{}` from {} to {}", addr, stored, program.hash); + module + .update_database_with_deployment(program, old_module_info, policy, deployment) + .await } } } @@ -936,7 +977,7 @@ impl Host { (db, clients) } }; - let (mut program, program_needs_init) = match db.program()? { + let (mut program, program_needs_init, initial_deployment) = match db.program()? { // Launch module with program from existing database. Some(program) => { info!( @@ -944,7 +985,7 @@ impl Host { program.hash, HostType::from(program.kind) ); - (program, false) + (program, false, None) } // Database is empty, load program from external storage and run // initialization. @@ -953,13 +994,14 @@ impl Host { "loading program {} from external storage host-type={}", database.initial_program, database.host_type ); + let initial_deployment = program_storage.initial_deployment(&database).await?; let program_bytes = load_program(program_storage, database.initial_program).await?; let program = Program { hash: database.initial_program, bytes: program_bytes, kind: database.host_type.into(), }; - (program, true) + (program, true, initial_deployment) } }; @@ -1054,7 +1096,10 @@ impl Host { }; if program_needs_init { - let call_result = launched.module_host.init_database(program).await?; + let call_result = launched + .module_host + .init_database_with_deployment(program, initial_deployment) + .await?; if let Some(call_result) = call_result { Result::from(call_result)?; } @@ -1168,9 +1213,11 @@ impl Host { runtimes: Arc, program: Program, policy: MigrationPolicy, + deployment: Option, energy_monitor: Arc, on_panic: impl Fn() + Send + Sync + 'static, core: AllocatedJobCore, + database_committed: &mut bool, ) -> anyhow::Result { let replica_ctx = &self.replica_ctx; let (scheduler, scheduler_starter) = Scheduler::open(self.replica_ctx.relational_db().clone()); @@ -1189,15 +1236,37 @@ impl Host { // Get the old module info to diff against when building a migration plan. let old_module_info = self.module.borrow().info.clone(); - let update_result = - update_module(replica_ctx.relational_db(), &module, program, old_module_info, policy).await?; + let update_result = update_module( + replica_ctx.relational_db(), + &module, + program, + old_module_info, + policy, + deployment, + ) + .await?; + + *database_committed = matches!( + update_result, + UpdateDatabaseResult::UpdatePerformed { .. } + | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } + ); + + #[cfg(test)] + if *database_committed + && FAIL_NEXT_DEPLOYMENT_ACTIVATION + .lock() + .remove(&replica_ctx.database_identity) + { + bail!("injected scheduler activation failure after deployment commit"); + } // Only replace the module + scheduler if the update succeeded. // Otherwise, we want the database to continue running with the old state. match update_result { UpdateDatabaseResult::NoUpdateNeeded | UpdateDatabaseResult::UpdatePerformed { .. } => { - self.scheduler = scheduler; scheduler_starter.start(&module)?; + self.scheduler = scheduler; let old_module = self.module.send_replace(module); old_module.exit().await; } @@ -1206,26 +1275,29 @@ impl Host { UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } => { // Replace the module first, so that new clients get the new module. let old_watcher = std::mem::replace(&mut self.module, watch::Sender::new(module.clone())); + let old_module = old_watcher.borrow().clone(); // Disconnect all clients connected to the old module. - let connected_clients = replica_ctx.relational_db().connected_clients()?; - for (identity, connection_id) in connected_clients { - let client_actor_id = ClientActorId { - identity, - connection_id, - name: ClientName(0), - }; - //NOTE: This will call disconnect reducer of the new module, not the old one. - //It makes sense, as relationaldb is already updated to the new module. - module.disconnect_client(client_actor_id).await; + let activation = async { + let connected_clients = replica_ctx.relational_db().connected_clients()?; + for (identity, connection_id) in connected_clients { + let client_actor_id = ClientActorId { + identity, + connection_id, + name: ClientName(0), + }; + // Disconnect uses the newly committed module. + module.disconnect_client(client_actor_id).await; + } + scheduler_starter.start(&module)?; + Ok::<_, anyhow::Error>(()) } - - self.scheduler = scheduler; - scheduler_starter.start(&module)?; + .await; // exit the old module, drop the `old_watcher` afterwards, // which will signal websocket clients that the module is gone. - let old_module = old_watcher.borrow().clone(); old_module.exit().await; + activation?; + self.scheduler = scheduler; } _ => {} } diff --git a/crates/core/src/host/host_controller/deployment_tests.rs b/crates/core/src/host/host_controller/deployment_tests.rs new file mode 100644 index 00000000000..348d90a695d --- /dev/null +++ b/crates/core/src/host/host_controller/deployment_tests.rs @@ -0,0 +1,213 @@ +use super::*; +use crate::db::deployment::{current_deployment, install_publication_fence}; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::empty_module; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ST_DEPLOYMENT_OPERATION_ID, ST_PUBLISH_FENCE_ID}; +use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; +use spacetimedb_lib::{hash_bytes, Uuid}; +use spacetimedb_paths::FromPathUnchecked; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +struct InitialStorage { + program: Program, + request: Option, + initial_lookups: AtomicUsize, +} + +#[async_trait] +impl ExternalStorage for InitialStorage { + async fn lookup(&self, hash: Hash) -> anyhow::Result>> { + Ok((self.program.hash == hash).then(|| self.program.bytes.clone())) + } + async fn initial_deployment(&self, _: &Database) -> anyhow::Result> { + self.initial_lookups.fetch_add(1, Ordering::SeqCst); + Ok(self.request.clone()) + } +} + +fn request(program: &Program, sequence: u64, initial: bool) -> DeploymentCommit { + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + DeploymentCommit { + operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(sequence)), + publication_epoch: sequence, + publisher: Identity::ONE, + expected_revision: None, + prepared_manifest_hash: hash_bytes(sequence.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: if initial { + ModuleComponent::SystemEmpty(1) + } else { + ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: program.hash, + }) + }, + container: Some(ContainerSpec { + image_manifest: OciDigest::sha256([4; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 1000, + }), + }), + } +} + +fn fixture(id: u64, bootstrap: bool) -> (tempfile::TempDir, HostController, Database, Arc) { + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = empty_module::program(1).unwrap(); + let storage = Arc::new(InitialStorage { + request: bootstrap.then(|| request(&program, 1, true)), + program: program.clone(), + initial_lookups: AtomicUsize::new(0), + }); + let controller = HostController::new( + data_dir.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + storage.clone(), + Arc::new(NullEnergyMonitor), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + }; + (directory, controller, database, storage) +} + +#[tokio::test(flavor = "multi_thread")] +async fn bootstrap_commits_fence_program_and_deployment_and_reopens_from_disk() { + let (_directory, controller, database, storage) = fixture(0xdd01, true); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let expected = storage.request.as_ref().unwrap().deployment.revision().unwrap(); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, expected); + assert_eq!(tx.table_row_count(ST_PUBLISH_FENCE_ID), Some(1)); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + assert_eq!( + module.relational_db().program().unwrap().unwrap().hash, + database.initial_program + ); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!( + storage.initial_lookups.load(Ordering::SeqCst), + 1, + "replay must not re-run bootstrap intent" + ); + module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, expected); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn activation_failure_after_commit_closes_old_host_and_recovers_committed_program() { + let (_directory, controller, database, _storage) = fixture(0xdd02, false); + let old_module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let watcher = controller.watch_module_host(database.id).await.unwrap(); + let mut bytes = empty_module::VERSION_1_BYTES.to_vec(); + bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let newer = Program::from_bytes(ModuleKind::WASM, bytes); + let publication = request(&newer, 1, false); + old_module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, publication.publication_epoch, publication.operation_id) + }) + .unwrap(); + FAIL_NEXT_DEPLOYMENT_ACTIVATION + .lock() + .insert(database.database_identity); + let error = controller + .update_module_host_with_deployment( + database.clone(), + HostType::Wasm, + database.id, + newer.bytes.clone(), + MigrationPolicy::Compatible, + Some(publication.clone()), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("injected scheduler activation failure")); + assert!( + controller.get_module_host(database.id).await.is_err(), + "old executable must not remain available" + ); + assert!(watcher.has_changed().is_err(), "old client watcher must close"); + assert_eq!(old_module.relational_db().program().unwrap().unwrap().hash, newer.hash); + old_module.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!( + current_deployment(tx).unwrap().unwrap().0, + publication.deployment.revision().unwrap() + ); + }); + drop(watcher); + drop(old_module); + let recovered = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(recovered.info.module_hash, newer.hash); + recovered.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!( + current_deployment(tx).unwrap().unwrap().0, + publication.deployment.revision().unwrap() + ); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(1)); + }); + drop(recovered); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 25614821750..b811a9efd9a 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -304,6 +304,27 @@ impl InstanceEnv { self.replica_ctx.relational_db() } + /// Dedicated read-only environment access. Missing reads also register a + /// dependency so a later insert refreshes a view that observed absence. + pub(crate) fn env_get(&self, key: &str) -> Result, NodesError> { + use crate::db::environment; + use spacetimedb_datastore::system_tables::ST_ENV_ID; + spacetimedb_lib::environment::validate_key(key).map_err(|_| NodesError::InvalidEnvironmentKey)?; + let read = |state: &_| environment::get(state, key).map_err(|err| NodesError::from(DBError::Other(err.into()))); + if let Ok(mut tx) = self.get_tx() { + tx.record_table_scan(&self.func_type, ST_ENV_ID); + return read(&*tx); + } + if !matches!(self.func_type, FuncCallType::Procedure) { + return Err(NodesError::NotInTransaction); + } + self.relational_db().with_read_only(Workload::Internal, |tx| { + check_hosted_admission(tx, *self.database_identity(), self.hosted_auth.as_deref()) + .map_err(|err| NodesError::HostedInvocationRejected(err.to_string()))?; + environment::get(tx, key).map_err(|err| NodesError::from(DBError::Other(err.into()))) + }) + } + pub(crate) fn get_jwt_payload(&self, connection_id: ConnectionId) -> Result, NodesError> { if let Ok(tx) = self.get_tx() { return Ok(tx.get_jwt_payload(connection_id).map_err(DBError::from)?); @@ -1455,6 +1476,111 @@ mod test { Ok(db) } + #[test] + fn environment_reads_use_active_transaction_and_track_missing_view_dependency() -> Result<()> { + use crate::db::environment; + use spacetimedb_datastore::locking_tx_datastore::ViewCallInfo; + use spacetimedb_datastore::system_tables::ST_ENV_ID; + use spacetimedb_primitives::{ViewFnPtr, ViewId}; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + assert!(matches!(env.env_get("A"), Err(NodesError::NotInTransaction))); + env.func_type = FuncCallType::Procedure; + assert_eq!(env.env_get("A")?, None); + let mut tx = begin_mut_tx(&db); + environment::set(&db, &mut tx, "A", "uncommitted")?; + env.tx.set_raw(tx); + assert_eq!(env.env_get("A")?.as_deref(), Some("uncommitted")); + let view = ViewCallInfo { + view_id: ViewId(88), + table_id: ST_ENV_ID, + fn_ptr: ViewFnPtr(0), + sender: None, + }; + env.func_type = FuncCallType::View(view.clone()); + assert_eq!(env.env_get("MISSING")?, None); + let tx = env.tx.take()?; + db.commit_tx(tx)?; + let mut tx = begin_mut_tx(&db); + environment::set(&db, &mut tx, "MISSING", "")?; + assert!(tx.views_for_refresh().any(|dependency| dependency == &view)); + let (_, metrics, reducer) = db.rollback_mut_tx(tx); + db.report_mut_tx_metrics(reducer, metrics, None); + env.func_type = FuncCallType::Procedure; + assert_eq!(env.env_get("MISSING")?, None); + assert!(matches!(env.env_get("A=B"), Err(NodesError::InvalidEnvironmentKey))); + Ok(()) + } + + #[test] + fn procedure_environment_snapshot_rechecks_durable_fence_and_expiry() -> Result<()> { + use crate::auth::{ + hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, + JwtKeys, + }; + use crate::db::{deployment::install_container_fence, environment}; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + use std::time::SystemTime; + let db = relational_db()?; + let (mut env, _runtime) = instance_env(db.clone())?; + env.func_type = FuncCallType::Procedure; + let keys = JwtKeys::generate()?; + let validator = HostedTokenValidator::new([("platform.test".into(), keys.public)])?; + let mint = |issued: SystemTime| -> Result<_> { + let binding = HostedTokenBinding { + source_database: db.database_identity(), + target_database: db.database_identity(), + generation: 1, + grant_revision: 1, + lease_expires_at: issued + Duration::from_secs(30), + }; + let token = sign_hosted_token( + &keys.private, + "platform.test", + &binding, + issued, + issued + Duration::from_secs(20), + "env-test", + )?; + Ok(Arc::new(validator.validate_token( + &token, + db.database_identity(), + issued, + |_, _, _| Some(binding), + )?)) + }; + let fence = |generation, allowed| StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: generation, + target_set_hash: Hash::ZERO, + allowed, + }; + db.with_auto_commit(Workload::ForTests, |tx| -> Result<()> { + install_container_fence(&db, tx, &fence(1, true))?; + environment::set(&db, tx, "VALUE", "available")?; + Ok(()) + })?; + env.set_hosted_auth(Some(mint(SystemTime::now())?)); + assert_eq!(env.env_get("VALUE")?.as_deref(), Some("available")); + env.set_hosted_auth(Some(mint(SystemTime::now() - Duration::from_secs(60))?)); + assert!(matches!( + env.env_get("VALUE"), + Err(NodesError::HostedInvocationRejected(_)) + )); + env.set_hosted_auth(Some(mint(SystemTime::now())?)); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &fence(2, false)) + })?; + assert!(matches!( + env.env_get("VALUE"), + Err(NodesError::HostedInvocationRejected(_)) + )); + env.set_hosted_auth(None); + assert_eq!(env.env_get("VALUE")?.as_deref(), Some("available")); + Ok(()) + } + /// Generate a `ProductValue` for use in [create_table_with_index] fn product_row(i: usize) -> ProductValue { let str = i.to_string(); diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index c25c47bb99b..0be568862c9 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -11,6 +11,7 @@ use spacetimedb_schema::def::deserialize::{ArgsSeed, FunctionDef}; use spacetimedb_schema::def::ModuleDef; mod disk_storage; +pub mod empty_module; mod host_controller; mod module_common; #[allow(clippy::too_many_arguments)] @@ -188,6 +189,7 @@ pub enum AbiCall { JwtLength, GetJwt, GetCallAuthFlags, + EnvGet, VolatileNonatomicScheduleImmediate, diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 9232be528c0..78dab16c3ab 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -7,6 +7,7 @@ use crate::auth::invocation::{check_hosted_admission, InvocationCaller, SqlCallA use crate::client::messages::{OneOffQueryResponseMessage, ProcedureResultMessage, SerializableMessage}; use crate::client::{ClientActorId, ClientConnectionSender, WsVersion}; use crate::database_logger::{DatabaseLogger, LogLevel, Record}; +use crate::db::deployment::{self, CommitAdmission, DeploymentCommit, PublishResult as DeploymentPublishResult}; use crate::db::relational_db::{RelationalDB, Tx}; use crate::energy::EnergyQuanta; use crate::error::DBError; @@ -592,18 +593,33 @@ fn extract_trapped(res: Result<(T, bool), E>) -> (Result, bool) { pub(crate) fn init_database( replica_ctx: &ReplicaContext, module_def: &ModuleDef, + module_hash: Hash, program: Program, + deployment: Option, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResult, bool), ) -> (anyhow::Result>, bool) { - extract_trapped(init_database_inner(replica_ctx, module_def, program, call_reducer)) + extract_trapped(init_database_inner( + replica_ctx, + module_def, + module_hash, + program, + deployment, + call_reducer, + )) } fn init_database_inner( replica_ctx: &ReplicaContext, module_def: &ModuleDef, + module_hash: Hash, program: Program, + deployment: Option, call_reducer: impl FnOnce(Option, CallReducerParams) -> (ReducerCallResult, bool), ) -> anyhow::Result<(Option, bool)> { + anyhow::ensure!( + module_hash == program.hash && spacetimedb_lib::hash_bytes(&program.bytes) == program.hash, + "program does not match the instantiated module" + ); log::debug!("init database"); let timestamp = Timestamp::now(); let stdb = replica_ctx.relational_db(); @@ -612,6 +628,28 @@ fn init_database_inner( let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); let auth_ctx = AuthCtx::for_current(owner_identity); + let (tx, admission) = stdb.with_auto_rollback(tx, |tx| { + if let Some(request) = &deployment { + deployment::validate_deployment_program(request, &program, module_def)?; + // New databases install the bootstrap fence, schema, program and + // receipt in this one transaction. An interrupted initialization + // cannot leave a successful partial deployment behind. + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use spacetimedb_datastore::system_tables::ST_MODULE_ID; + if tx.iter(ST_MODULE_ID)?.next().is_none() { + deployment::install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + } + deployment::check_deployment_commit(tx, request, timestamp, &Default::default()) + } else { + deployment::require_unmanaged_publication(tx)?; + Ok(CommitAdmission::Ready) + } + })?; + if matches!(admission, CommitAdmission::AlreadyCommitted(_)) { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + return Ok((None, false)); + } let (tx, ()) = stdb .with_auto_rollback(tx, |tx| { // Create all in-memory tables defined by the module, @@ -645,6 +683,10 @@ fn init_database_inner( stdb.set_initialized(tx, program)?; + if let Some(request) = &deployment { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default())?; + } + anyhow::Ok(()) }) .inspect_err(|e| log::error!("{e:?}"))?; @@ -1443,6 +1485,13 @@ pub struct WeakModuleHost { #[derive(Debug)] pub enum UpdateDatabaseResult { NoUpdateNeeded, + /// A prior successful commit is returned without replacing the current + /// module, which may already be newer than this retried publication. + DeploymentAlreadyCommitted { + result: DeploymentPublishResult, + tx_offset: TransactionOffset, + durable_offset: Option, + }, UpdatePerformed { /// The transaction offset of the successful database update. tx_offset: TransactionOffset, @@ -1467,6 +1516,7 @@ impl UpdateDatabaseResult { self, UpdateDatabaseResult::UpdatePerformed { .. } | UpdateDatabaseResult::NoUpdateNeeded + | UpdateDatabaseResult::DeploymentAlreadyCommitted { .. } | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } ) } @@ -3097,14 +3147,49 @@ impl ModuleHost { } pub async fn init_database(&self, program: Program) -> Result, InitDatabaseError> { - call_instance!( + self.init_database_with_deployment(program, None).await + } + + /// Host-only initialization following an authorized, durable control intent. + /// A new database installs its bootstrap fence in the initialization + /// transaction. Successful deployment initialization waits for durability + /// before its scheduler or container can be activated. + pub async fn init_database_with_deployment( + &self, + program: Program, + deployment: Option, + ) -> Result, InitDatabaseError> { + let confirm_deployment = deployment.is_some(); + let result = call_instance!( self, "", - program, - |p, inst| inst.init_database(p), - |p, inst| inst.init_database(p).await, + (program, deployment), + |(p, d), inst| inst.init_database(p, d), + |(p, d), inst| inst.init_database(p, d).await, )? - .map_err(InitDatabaseError::Other) + .map_err(InitDatabaseError::Other)?; + if confirm_deployment + && result.as_ref().is_none_or(|result| result.is_ok()) + && let Some(mut durability) = self.relational_db().durable_tx_offset() + { + // A read barrier after init includes its receipt even when the + // module has no init reducer. Do not block the async worker on + // the datastore lock while capturing that barrier. + let db = self.relational_db().clone(); + let offset = tokio::task::spawn_blocking(move || { + let tx = db.begin_tx(Workload::Internal); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + offset + }) + .await + .map_err(|error| InitDatabaseError::Other(error.into()))?; + durability + .wait_for(offset) + .await + .map_err(|error| InitDatabaseError::Other(error.into()))?; + } + Ok(result) } pub async fn update_database( @@ -3112,13 +3197,25 @@ impl ModuleHost { program: Program, old_module_info: Arc, policy: MigrationPolicy, + ) -> Result { + self.update_database_with_deployment(program, old_module_info, policy, None) + .await + } + + /// Commit the prepared deployment in the same transaction as migration. + pub async fn update_database_with_deployment( + &self, + program: Program, + old_module_info: Arc, + policy: MigrationPolicy, + deployment: Option, ) -> Result { call_instance!( self, "", - (program, old_module_info, policy), - |(a, b, c), inst| inst.update_database(a, b, c), - |(a, b, c), inst| inst.update_database(a, b, c).await, + (program, old_module_info, policy, deployment), + |(a, b, c, d), inst| inst.update_database(a, b, c, d), + |(a, b, c, d), inst| inst.update_database(a, b, c, d).await, )? } diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 9c6237d9826..d6c49aef093 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -488,11 +488,13 @@ impl JsMainInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { self.request(UpdateDatabaseRequest { program, old_module_info, policy, + deployment, }) .await } @@ -548,8 +550,12 @@ impl JsMainInstance { self.request(DisconnectClientRequest { client_id }).await } - pub async fn init_database(&self, program: Program) -> anyhow::Result> { - self.request(InitDatabaseRequest { program }).await + pub async fn init_database( + &self, + program: Program, + deployment: Option, + ) -> anyhow::Result> { + self.request(InitDatabaseRequest { program, deployment }).await } pub async fn call_view(&self, cmd: ViewCommand) -> ViewCommandResult { @@ -633,6 +639,7 @@ js_main_request! { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, } => "update_database", anyhow::Result, UpdateDatabase } @@ -675,6 +682,7 @@ js_main_request! { js_main_request! { InitDatabaseRequest { program: Program, + deployment: Option, } => "init_database", anyhow::Result>, InitDatabase } @@ -817,6 +825,7 @@ enum JsMainWorkerRequest { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, }, /// See [`JsMainInstance::call_reducer`]. CallReducer { @@ -877,6 +886,7 @@ enum JsMainWorkerRequest { InitDatabase { reply_tx: JsReplyTx>>, program: Program, + deployment: Option, }, } @@ -1407,8 +1417,9 @@ fn handle_main_worker_request( program, old_module_info, policy, + deployment, } => handle_worker_request("update_database", reply_tx, || { - let res = instance_common.update_database(program, old_module_info, policy, inst); + let res = instance_common.update_database(program, old_module_info, policy, deployment, inst); (res, false) }), JsMainWorkerRequest::CallReducer { reply_tx, params } => { @@ -1496,14 +1507,22 @@ fn handle_main_worker_request( (res, trapped) }) } - JsMainWorkerRequest::InitDatabase { reply_tx, program } => { - handle_worker_request("init_database", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); - let (res, trapped): (Result, anyhow::Error>, bool) = - init_database(replica_ctx, &info.module_def, program, call_reducer); - (res, trapped) - }) - } + JsMainWorkerRequest::InitDatabase { + reply_tx, + program, + deployment, + } => handle_worker_request("init_database", reply_tx, || { + let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let (res, trapped): (Result, anyhow::Error>, bool) = init_database( + replica_ctx, + &info.module_def, + info.module_hash, + program, + deployment, + call_reducer, + ); + (res, trapped) + }), } } diff --git a/crates/core/src/host/v8/syscall/mod.rs b/crates/core/src/host/v8/syscall/mod.rs index 467e7f26562..bb942a47f2d 100644 --- a/crates/core/src/host/v8/syscall/mod.rs +++ b/crates/core/src/host/v8/syscall/mod.rs @@ -63,6 +63,7 @@ fn resolve_sys_module_inner<'scope>( (2, 0) => Ok(v2::sys_v2_0(scope)), (2, 1) => Ok(v2::sys_v2_1(scope)), (2, 2) => Ok(v2::sys_v2_2(scope)), + (2, 3) => Ok(v2::sys_v2_3(scope)), _ => Err(TypeError(format!( "Could not import {spec:?}, likely because this module was built for a newer version of SpacetimeDB.\n\ It requires sys module v{major}.{minor}, but that version is not supported by the database." diff --git a/crates/core/src/host/v8/syscall/v2.rs b/crates/core/src/host/v8/syscall/v2.rs index e34c9fa9cfe..4f174855182 100644 --- a/crates/core/src/host/v8/syscall/v2.rs +++ b/crates/core/src/host/v8/syscall/v2.rs @@ -178,6 +178,24 @@ pub(super) fn sys_v2_2<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope ) } +pub(super) fn sys_v2_3<'scope>(scope: &mut PinScope<'scope, '_>) -> Local<'scope, Module> { + create_synthetic_module!(scope, "spacetime:sys@2.3", (with_sys_result, AbiCall::EnvGet, env_get),) +} + +fn env_get<'s>( + scope: &mut PinScope<'s, '_>, + args: FunctionCallbackArguments<'s>, +) -> SysCallResult> { + let key: String = deserialize_js(scope, args.get(0))?; + match get_env(scope)?.instance_env.env_get(&key)? { + Some(value) => Ok(value + .into_string(scope) + .map_err(|_| RangeError("environment value could not be represented").throw(scope))? + .into()), + None => Ok(v8::null(scope).into()), + } +} + fn get_call_auth_flags(scope: &mut PinScope<'_, '_>, _args: FunctionCallbackArguments<'_>) -> SysCallResult { Ok(get_env(scope)?.instance_env.get_call_auth_flags()) } diff --git a/crates/core/src/host/wasm_common.rs b/crates/core/src/host/wasm_common.rs index 17af2c7bb5f..26569d072af 100644 --- a/crates/core/src/host/wasm_common.rs +++ b/crates/core/src/host/wasm_common.rs @@ -362,6 +362,8 @@ pub fn err_to_errno(err: NodesError) -> Result<(NonZeroU16, Option), Nod NodesError::DecodeRow(_) => errno::BSATN_DECODE_ERROR, NodesError::DecodeValue(_) => errno::BSATN_DECODE_ERROR, NodesError::TableNotFound => errno::NO_SUCH_TABLE, + NodesError::InvalidEnvironmentKey => errno::HOST_CALL_FAILURE, + NodesError::EnvironmentSourceLimit => errno::NO_SPACE, NodesError::IndexNotFound => errno::NO_SUCH_INDEX, NodesError::IndexNotUnique => errno::INDEX_NOT_UNIQUE, NodesError::IndexRowNotFound => errno::NO_SUCH_ROW, @@ -443,6 +445,7 @@ macro_rules! abi_funcs { "spacetime_10.5"::datastore_clear, "spacetime_10.6"::get_call_auth_flags, + "spacetime_10.7"::env_get, } $link_async! { diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 7087ba93aec..a33c6660c8f 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -4,6 +4,7 @@ use crate::auth::hosted_tokens::VerifiedHostedAuth; use crate::auth::invocation::check_hosted_admission; use crate::client::ClientActorId; use crate::database_logger; +use crate::db::deployment::{self, CommitAdmission, DeploymentCommit}; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; use crate::host::host_controller::CallProcedureReturn; @@ -477,9 +478,10 @@ impl WasmModuleInstance { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, ) -> anyhow::Result { self.common - .update_database(program, old_module_info, policy, &mut self.instance) + .update_database(program, old_module_info, policy, deployment, &mut self.instance) } pub fn call_reducer(&mut self, params: CallReducerParams) -> ReducerCallResult { @@ -533,11 +535,23 @@ impl WasmModuleInstance { res } - pub fn init_database(&mut self, program: Program) -> anyhow::Result> { - let module_def = &self.common.info.clone().module_def; + pub fn init_database( + &mut self, + program: Program, + deployment: Option, + ) -> anyhow::Result> { + let info = self.common.info.clone(); + let module_def = &info.module_def; let replica_ctx = &self.instance.replica_ctx().clone(); let call_reducer = |tx, params| self.call_reducer_with_tx(tx, params); - let (res, trapped) = init_database(replica_ctx, module_def, program, call_reducer); + let (res, trapped) = init_database( + replica_ctx, + module_def, + info.module_hash, + program, + deployment, + call_reducer, + ); self.trapped = trapped; res } @@ -632,12 +646,82 @@ impl InstanceCommon { program: Program, old_module_info: Arc, policy: MigrationPolicy, + deployment: Option, inst: &mut I, ) -> Result { let replica_ctx = inst.replica_ctx().clone(); let system_logger = replica_ctx.logger.system_logger(); let stdb = &replica_ctx.relational_db(); + let timestamp = Timestamp::now(); + let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, (admission, unchanged)) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<_> { + ensure!( + self.info.module_hash == program.hash, + "program does not match the instantiated module" + ); + let admission = if let Some(request) = &deployment { + deployment::validate_deployment_program(request, &program, &self.info.module_def)?; + deployment::check_deployment_commit(tx, request, timestamp, &Default::default())? + } else { + deployment::require_unmanaged_publication(tx)?; + CommitAdmission::Ready + }; + if matches!(admission, CommitAdmission::AlreadyCommitted(_)) { + return Ok((admission, false)); + } + deployment::validate_active_hosted_grants(tx, &self.info.module_def)?; + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use spacetimedb_datastore::system_tables::{read_hash_from_col, StModuleFields, ST_MODULE_ID}; + let row = tx.iter(ST_MODULE_ID)?.next().context("database is not initialized")?; + let stored_hash = read_hash_from_col(row, StModuleFields::ProgramHash)?; + ensure!( + stored_hash == old_module_info.module_hash, + "module changed before publication admission" + ); + Ok((admission, stored_hash == program.hash)) + })?; + if let CommitAdmission::AlreadyCommitted(result) = admission { + let (offset, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + let (sender, tx_offset) = tokio::sync::oneshot::channel(); + let _ = sender.send(offset); + return Ok(UpdateDatabaseResult::DeploymentAlreadyCommitted { + result, + tx_offset, + durable_offset: stdb.durable_tx_offset(), + }); + } + if unchanged { + let Some(request) = &deployment else { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); + return Ok(UpdateDatabaseResult::NoUpdateNeeded); + }; + let (tx, _) = stdb.with_auto_rollback(tx, |tx| { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default()) + })?; + let event = ModuleEvent { + timestamp, + caller_identity: request.publisher, + caller_connection_id: None, + function_call: ModuleFunctionCall::update(), + status: EventStatus::Committed(DatabaseUpdate::default()), + reducer_return_value: None, + energy_quanta_used: FunctionBudget::ZERO.into(), + host_execution_duration: Duration::ZERO, + request_id: None, + timer: None, + }; + let durable_offset = stdb.durable_tx_offset(); + let CommitAndBroadcastEventSuccess { tx_offset, .. } = + commit_and_broadcast_event(&self.info.subscriptions, None, event, tx); + return Ok(UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + }); + } + let plan: MigratePlan = match policy.try_migrate( self.info.database_identity, old_module_info.module_hash, @@ -647,17 +731,24 @@ impl InstanceCommon { ) { Ok(plan) => plan, Err(e) => { + let (_, metrics, reducer) = stdb.rollback_mut_tx(tx); + stdb.report_mut_tx_metrics(reducer, metrics, None); return match e { MigrationPolicyError::AutoMigrateFailure(e) => Ok(UpdateDatabaseResult::AutoMigrateError(e.into())), _ => Ok(UpdateDatabaseResult::ErrorExecutingMigration(e.into())), - } + }; } }; let program_hash = program.hash; let host_type = HostType::from(program.kind); - let tx = stdb.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| stdb.update_program(tx, program))?; + let (mut tx, _) = stdb.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + stdb.update_program(tx, program)?; + if let Some(request) = &deployment { + deployment::record_deployment_commit(tx, request, timestamp, &Default::default())?; + } + Ok(()) + })?; system_logger.info(&format!("Updated program to {program_hash}")); let auth_ctx = AuthCtx::for_current(replica_ctx.database.owner_identity); @@ -750,7 +841,9 @@ impl InstanceCommon { tx: MutTxId, inst: &mut I, ) -> Result<(ViewCallResult, bool), anyhow::Error> { - let view_calls = collect_subscribed_view_calls(&tx, &self.info.module_def, self.info.owner_identity)?; + let (tx, view_calls) = inst.replica_ctx().relational_db().with_auto_rollback(tx, |tx| { + collect_subscribed_view_calls(tx, &self.info.module_def, self.info.owner_identity) + })?; Ok(self.execute_view_calls(tx, view_calls, inst)) } diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index 82daff1bb20..13f427b3af2 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -28,6 +28,10 @@ use std::sync::Arc; use std::time::Instant; use wasmtime::{AsContext, Caller, StoreContextMut}; +/// Env reads may retain at most 2 MiB of value bytes outside the Wasm heap. +/// Other outstanding byte sources count against this interface's handle limit. +const MAX_OUTSTANDING_ENV_SOURCES: usize = 256; + /// A stream of bytes which the WASM module can read from /// using [`WasmInstanceEnv::bytes_source_read`]. /// @@ -188,7 +192,14 @@ impl WasmInstanceEnv { // This allows the module to avoid allocating and make a system call in those cases. if bytes.is_empty() { Ok(BytesSourceId::INVALID) - } else if bytes.len() > u32::MAX as usize { + } else { + self.create_present_bytes_source(bytes) + } + } + + /// Allocate a valid source even for an empty value when zero means absence. + fn create_present_bytes_source(&mut self, bytes: bytes::Bytes) -> RtResult { + if bytes.len() > u32::MAX as usize { // There's no inherent reason we need to error here, // other than that it makes it impossible to report the length in `bytes_source_remaining_length` // and that all of our usage of `BytesSource`s as of writing (pgoldman 2025-09-26) @@ -1544,6 +1555,34 @@ impl WasmInstanceEnv { }) } + /// Read an environment value as a nullable BytesSource. Zero means missing; + /// a present empty string always receives a nonzero, consumable source. + pub fn env_get( + caller: Caller<'_, Self>, + key: WasmPtr, + key_len: u32, + target_ptr: WasmPtr, + ) -> RtResult { + Self::cvt_ret(caller, AbiCall::EnvGet, target_ptr, |caller| { + if key_len == 0 || key_len > spacetimedb_lib::environment::MAX_ENV_KEY_BYTES as u32 { + return Err(crate::error::NodesError::InvalidEnvironmentKey.into()); + } + let (mem, env) = Self::mem_env(caller); + let key = mem.deref_str(key, key_len)?; + match env.instance_env.env_get(key)? { + None => Ok(0), + Some(value) => { + // These buffers live on the host heap until consumed or the + // invocation ends. Bound retained reads from hand-written Wasm. + if env.bytes_sources.len() >= MAX_OUTSTANDING_ENV_SOURCES { + return Err(crate::error::NodesError::EnvironmentSourceLimit.into()); + } + Ok(env.create_present_bytes_source(bytes::Bytes::from(value))?.0) + } + } + }) + } + /// Returns host-verified invocation flags. Bit 0 is internal authority. /// This does not read tables and is available outside transactions. pub fn get_call_auth_flags(caller: Caller<'_, Self>) -> u32 { diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index bb11b15e191..65d1b6b5bb9 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -55,7 +55,7 @@ impl WasmtimeModule { WasmtimeModule { module } } - pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 6); + pub const IMPLEMENTED_ABI: abi::VersionTuple = abi::VersionTuple::new(10, 7); pub(super) fn link_imports(linker: &mut Linker) -> anyhow::Result<()> { link_imports(linker, AsyncImportMode::SyncStub) diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 175de2fd9a0..1e5fac1d8c8 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -96,12 +96,12 @@ fn run_inner( let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { check_hosted_admission(tx, db.database_identity(), auth.hosted.as_deref())?; let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; - if let Statement::DML(dml) = &stmt { - if spacetimedb_datastore::system_tables::is_host_managed_deployment_table(dml.table_id()) { - return Err(anyhow!( - "Deployment and container authorization metadata may only be changed by the host" - )); - } + if let Statement::DML(dml) = &stmt + && spacetimedb_datastore::system_tables::is_host_managed_deployment_table(dml.table_id()) + { + return Err(anyhow!( + "Deployment and container authorization metadata may only be changed by the host" + )); } Ok(stmt) })?; diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs index 885d64abca0..e29690d34cd 100644 --- a/crates/lib/src/container.rs +++ b/crates/lib/src/container.rs @@ -19,7 +19,7 @@ pub const EXEC_RESERVED_BYTES: usize = 4096; pub const DEFAULT_STOP_GRACE_MS: u32 = 30_000; pub const MAX_STOP_GRACE_MS: u32 = 120_000; -/// The digest of an OCI object. This is never a SpacetimeDB BLAKE3 object key. +/// The digest of an OCI object. This is never a SpacetimeDB Keccak-256 program key. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, SpacetimeType)] #[sats(crate = crate)] pub enum OciDigest { @@ -242,6 +242,10 @@ pub struct ContainerSpec { feature = "serde", serde(tag = "action", content = "value", rename_all = "snake_case", deny_unknown_fields) )] +#[expect( + clippy::large_enum_variant, + reason = "the normalized publish request owns its single container spec" +)] pub enum ContainerAction { #[default] Keep, diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs index fecc63055e9..a81b2b19637 100644 --- a/crates/lib/src/deployment.rs +++ b/crates/lib/src/deployment.rs @@ -8,6 +8,12 @@ use crate::{bsatn, hash_bytes, Hash, SpacetimeType, Uuid}; pub const PUBLISH_PROTOCOL_VERSION: u32 = 1; pub const SYSTEM_EMPTY_MODULE_VERSION: u32 = 1; +/// Immutable Keccak-256 program identity of the version-1 bundled empty Wasm +/// module. Control can verify initial program bytes without linking the host. +pub const SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH: Hash = Hash::from_byte_array([ + 0x9b, 0x6c, 0xf2, 0xdb, 0x36, 0x44, 0xc1, 0xd3, 0x21, 0xd9, 0x7a, 0xe0, 0xfc, 0xd4, 0xab, 0x3f, 0xc0, 0x65, 0xfc, + 0x94, 0xf5, 0x4a, 0x49, 0xf6, 0x1c, 0x7a, 0x1d, 0xfa, 0x40, 0xa0, 0x26, 0x12, +]); pub const MAX_DEPLOYMENT_BYTES: usize = 256 * 1024; pub const PUBLISH_RETRY_WINDOW_MS: u64 = 7 * 24 * 60 * 60 * 1000; pub const MAX_OPERATION_CLOCK_SKEW_MS: u64 = 5 * 60 * 1000; @@ -131,10 +137,10 @@ impl DeploymentSpec { pub fn normalize(self, limits: &ContainerSpecLimits) -> Result { let Self::V1(mut spec) = self; - if let ModuleComponent::SystemEmpty(version) = spec.module { - if version != SYSTEM_EMPTY_MODULE_VERSION { - return Err(DeploymentValidationError::UnsupportedEmptyModule); - } + if let ModuleComponent::SystemEmpty(version) = spec.module + && version != SYSTEM_EMPTY_MODULE_VERSION + { + return Err(DeploymentValidationError::UnsupportedEmptyModule); } spec.container = spec.container.map(|spec| spec.normalize(limits)).transpose()?; let spec = Self::V1(spec); diff --git a/crates/lib/src/environment.rs b/crates/lib/src/environment.rs new file mode 100644 index 00000000000..4963a6f8718 --- /dev/null +++ b/crates/lib/src/environment.rs @@ -0,0 +1,53 @@ +//! Limits shared by the database environment store and its clients. + +pub const MAX_ENV_KEY_BYTES: usize = 256; +pub const MAX_ENV_VALUE_BYTES: usize = 8 * 1024; +pub const MAX_ENV_VARS: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub enum EnvironmentValidationError { + #[error("invalid POSIX environment variable name (maximum 256 bytes)")] + InvalidKey, + #[error("environment value exceeds 8192 UTF-8 bytes")] + ValueTooLarge, + #[error("environment store exceeds 256 variables")] + TooManyVariables, +} + +pub fn validate_key(key: &str) -> Result<(), EnvironmentValidationError> { + let bytes = key.as_bytes(); + if bytes.is_empty() + || bytes.len() > MAX_ENV_KEY_BYTES + || !(bytes[0].is_ascii_alphabetic() || bytes[0] == b'_') + || !bytes.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'_') + { + return Err(EnvironmentValidationError::InvalidKey); + } + Ok(()) +} + +/// NUL is representable in the database. Container launch separately rejects it. +pub fn validate_value(value: &str) -> Result<(), EnvironmentValidationError> { + if value.len() > MAX_ENV_VALUE_BYTES { + return Err(EnvironmentValidationError::ValueTooLarge); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validates_utf8_byte_limits_and_posix_keys_without_container_policy() { + for key in ["", "1FIRST", "A=B", "A\0B", "é", "A-B"] { + assert_eq!(validate_key(key), Err(EnvironmentValidationError::InvalidKey)); + } + assert!(validate_key(&"A".repeat(256)).is_ok()); + assert!(validate_key(&"A".repeat(257)).is_err()); + assert!(validate_key("SPACETIMEDB_USER_DATA").is_ok()); + assert!(validate_value("\0").is_ok()); + assert!(validate_value(&"é".repeat(4096)).is_ok()); + assert!(validate_value(&"é".repeat(4097)).is_err()); + } +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index 547b78d0e26..a8febe31d9b 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -12,9 +12,10 @@ use std::collections::{btree_map, BTreeMap}; pub mod connection_id; pub mod container; -pub mod deployment; pub mod db; +pub mod deployment; mod direct_index_key; +pub mod environment; pub mod error; mod filterable_value; pub mod http; diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index 21fea57fe96..5a07c3b6f60 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -61,6 +61,12 @@ pub struct ModuleHandle { } impl ModuleHandle { + /// Access the real standalone control/host environment for integration + /// tests that publish, migrate, or recover the running module. + pub fn environment(&self) -> &StandaloneEnv { + &self._env + } + async fn call_reducer(&self, reducer: &str, args: FunctionArgs) -> anyhow::Result<()> { let result = self .client diff --git a/crates/testing/tests/deployment_publish.rs b/crates/testing/tests/deployment_publish.rs new file mode 100644 index 00000000000..dd3b7323967 --- /dev/null +++ b/crates/testing/tests/deployment_publish.rs @@ -0,0 +1,204 @@ +//! Exercise publication against the real host and Wasm module, including the +//! unchanged-program path and retries after a later module has been installed. +use serial_test::serial; +use spacetimedb::db::deployment::{current_deployment, install_publication_fence, DeploymentCommit}; +use spacetimedb::host::{FunctionArgs, UpdateDatabaseResult}; +use spacetimedb::messages::control_db::HostType; +use spacetimedb_client_api::{ControlStateReadAccess, NodeDelegate}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::ST_DEPLOYMENT_OPERATION_ID; +use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; +use spacetimedb_lib::{hash_bytes, sats::product, ConnectionId, Hash, Identity, Uuid}; +use spacetimedb_schema::auto_migrate::{MigrationPolicy, MigrationToken}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +fn prepared(bytes: &[u8], epoch: u64, previous: Option, command: &str) -> DeploymentCommit { + let now_ms = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(); + DeploymentCommit { + operation_id: Uuid::from_u128((now_ms << 80) | (0x7000u128 << 64) | (0x8000u128 << 48) | u128::from(epoch)), + publication_epoch: epoch, + publisher: Identity::ZERO, + expected_revision: previous, + prepared_manifest_hash: hash_bytes(epoch.to_le_bytes()), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Wasm, + program_hash: hash_bytes(bytes), + }), + container: Some(ContainerSpec { + image_manifest: OciDigest::sha256([3; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec![command.into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 1000, + }), + }), + } +} + +fn fence(module: &spacetimedb::host::ModuleHost, request: &DeploymentCommit) { + module + .relational_db() + .with_auto_commit(Workload::Internal, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); +} + +async fn confirmed(result: UpdateDatabaseResult) { + let (offset, durable) = match result { + UpdateDatabaseResult::UpdatePerformed { + tx_offset, + durable_offset, + } + | UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { + tx_offset, + durable_offset, + } + | UpdateDatabaseResult::DeploymentAlreadyCommitted { + tx_offset, + durable_offset, + .. + } => (tx_offset, durable_offset), + other => panic!("expected a durable publication result, got {other:?}"), + }; + tokio::time::timeout(Duration::from_secs(10), async { + let offset = offset.await.unwrap(); + if let Some(mut durable) = durable { + durable.wait_for(offset).await.unwrap(); + } + }) + .await + .unwrap(); +} + +#[test] +#[serial] +fn container_only_changes_and_old_retries_preserve_the_current_module() { + let compiled = CompiledModule::compile("hosted-auth-test", CompilationMode::Debug); + let bytes = compiled.program_bytes(); + compiled.with_module_async(DEFAULT_CONFIG, |handle| async move { + let env = handle.environment(); + let database = env.get_database_by_identity(&handle.db_identity).await.unwrap().unwrap(); + let host = env.leader(database.id).await.unwrap(); + let module = host.module().await.unwrap(); + let first = prepared(&bytes, 1, None, "/app/first"); + fence(&module, &first); + // Even the first in-progress container publication fences the legacy + // raw API, including a byte-identical module update. + assert!(host.update(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible).await.is_err()); + // The rejected updater must restore the old host in its controller. + assert_eq!(host.module().await.unwrap().info.module_hash, hash_bytes(&bytes)); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); + confirmed(result).await; + let first_revision = first.deployment.revision().unwrap(); + let second = prepared(&bytes, 2, Some(first_revision), "/app/second"); + fence(&host.module().await.unwrap(), &second); + confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, second.clone()).await.unwrap()).await; + assert_ne!(first_revision, second.deployment.revision().unwrap()); + + // A valid custom section changes program bytes/hash without changing + // its definition. This exercises the actual Wasm migration and swap. + let mut newer_bytes = bytes.to_vec(); + newer_bytes.extend_from_slice(&[0, 3, 1, b'x', 1]); + let third = prepared(&newer_bytes, 3, Some(second.deployment.revision().unwrap()), "/app/third"); + fence(&host.module().await.unwrap(), &third); + confirmed(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, third.clone()).await.unwrap()).await; + let newest = third.deployment.revision().unwrap(); + assert_eq!(host.module().await.unwrap().info.module_hash, hash_bytes(&newer_bytes)); + let before_retry = host.module().await.unwrap(); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, first.clone()).await.unwrap(); + assert!(matches!(&result, UpdateDatabaseResult::DeploymentAlreadyCommitted { result, .. } if result.revision == first_revision)); + confirmed(result).await; + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, before_retry.info.module_hash); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + }); + current.call_reducer(Identity::ZERO, None, None, None, None, "private_only", FunctionArgs::Nullary).await.unwrap().outcome.into_result().unwrap(); + + // A stale CAS and a forged association between bytes and declaration + // cannot change either component, and rejection leaves service usable. + let stale = prepared(&bytes, 4, Some(first_revision), "/app/stale"); + fence(¤t, &stale); + assert!(host.update_with_deployment(database.clone(), HostType::Wasm, bytes.to_vec().into(), MigrationPolicy::Compatible, stale).await.is_err()); + let mismatched = prepared(&bytes, 5, Some(newest), "/app/mismatch"); + fence(¤t, &mismatched); + assert!(host.update_with_deployment(database.clone(), HostType::Wasm, newer_bytes.clone().into(), MigrationPolicy::Compatible, mismatched).await.is_err()); + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, hash_bytes(&newer_bytes)); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + }); + + let empty = spacetimedb::host::empty_module::program(1).unwrap(); + let mut remove_module = prepared(&empty.bytes, 6, Some(newest), "/app/image-only"); + let DeploymentSpec::V1(spec) = &mut remove_module.deployment; + spec.module = ModuleComponent::SystemEmpty(1); + // An init request cannot execute this instance's schema while storing + // the bytes of a different valid, prepared program. + assert!(current.init_database_with_deployment(empty.clone(), Some(remove_module.clone())).await.is_err()); + + // Removing a nonempty table fails during migration execution, after + // the tentative program and deployment/receipt writes. All roll back. + let observation_name = current.info.module_def.tables() + .find(|table| table.name.to_ascii_lowercase().contains("observation")) + .unwrap().name.to_string(); + let observation_table = current.relational_db().with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<_> { + let table = tx.table_id_from_name(&observation_name)?.unwrap(); + tx.insert_via_serialize_bsatn(table, &product![ConnectionId::from_u128(700), Identity::ZERO, false, Identity::ZERO, false])?; + Ok(table) + }).unwrap(); + let policy = MigrationPolicy::BreakClients(MigrationToken { + database_identity: handle.db_identity, + old_module_hash: current.info.module_hash, + new_module_hash: empty.hash, + }.hash()); + fence(¤t, &remove_module); + let result = host.update_with_deployment(database.clone(), HostType::Wasm, empty.bytes.clone(), policy.clone(), remove_module.clone()).await.unwrap(); + assert!(matches!(result, UpdateDatabaseResult::ErrorExecutingMigration(ref error) if error.to_string().contains("table contains data")), "{result:?}"); + let current = host.module().await.unwrap(); + assert_eq!(current.info.module_hash, hash_bytes(&newer_bytes)); + assert_eq!(current.relational_db().program().unwrap().unwrap().hash, hash_bytes(&newer_bytes)); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, newest); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(3)); + assert_eq!(tx.table_row_count(observation_table), Some(1)); + }); + current.relational_db().with_auto_commit(Workload::Internal, |tx| -> anyhow::Result<()> { + tx.clear_table(observation_table)?; + Ok(()) + }).unwrap(); + // The same operation remains eligible after an execution rollback. + confirmed(host.update_with_deployment(database, HostType::Wasm, empty.bytes.clone(), policy, remove_module.clone()).await.unwrap()).await; + let current = host.module().await.unwrap(); + assert!(current.info.module_def.tables().next().is_none()); + assert!(spacetimedb::host::empty_module::matches_program(1, ¤t.relational_db().program().unwrap().unwrap())); + current.relational_db().with_read_only(Workload::Internal, |tx| { + assert_eq!(current_deployment(tx).unwrap().unwrap().0, remove_module.deployment.revision().unwrap()); + assert_eq!(tx.table_row_count(ST_DEPLOYMENT_OPERATION_ID), Some(4)); + }); + }); +} diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs new file mode 100644 index 00000000000..cb0086bdfaf --- /dev/null +++ b/crates/testing/tests/environment.rs @@ -0,0 +1,134 @@ +//! Actual module calls exercise environment ABI, bindings, and snapshot semantics. +use serial_test::serial; +use spacetimedb::db::environment; +use spacetimedb::host::FunctionArgs; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; +use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; + +fn exercise_fixture(name: &str) { + CompiledModule::compile(name, CompilationMode::Debug).with_module_async(DEFAULT_CONFIG, |handle| async move { + let module = handle.client.module(); + let db = module.relational_db(); + for (key, expected) in [ + ("MISSING", None), + ("EMPTY", Some("".to_string())), + ("UTF8", Some("héllo 🌍".to_string())), + ("NUL", Some("before\0after".to_string())), + ("MAXIMUM", Some("é".repeat(4096))), + ] { + if let Some(value) = &expected { + db.with_auto_commit(Workload::Internal, |tx| environment::set(db, tx, key, value)) + .unwrap(); + } + let args = product![key, expected.clone()]; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + ) + .await; + let result = result + .map_err(anyhow::Error::from) + .and_then(|r| r.outcome.into_result()); + assert!( + result.is_ok(), + "{name} {key}: {result:?}; module log: {}", + handle.read_log(None).await + ); + let read = || FunctionArgs::Bsatn(bsatn::to_vec(&product![key]).unwrap().into()); + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(expected.clone())); + if expected.is_some() { + db.with_auto_commit(Workload::Internal, |tx| environment::set(db, tx, key, "updated")) + .unwrap(); + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(Some("updated".to_string()))); + db.with_auto_commit(Workload::Internal, |tx| environment::delete(db, tx, key)) + .unwrap(); + let result = module + .call_procedure(Identity::ZERO, None, None, "read_environment", read()) + .await + .result + .unwrap() + .return_val; + assert_eq!(result, AlgebraicValue::from(None::)); + } + } + if name == "environment-test" { + db.with_auto_commit(Workload::Internal, |tx| { + environment::set(db, tx, "LIMIT", &"x".repeat(8192)) + }) + .unwrap(); + for _ in 0..2 { + module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "bounded_environment_sources", + FunctionArgs::Nullary, + ) + .await + .unwrap() + .outcome + .into_result() + .unwrap(); + } + } + let args = product!["A=B", None::]; + let result = module + .call_reducer( + Identity::ZERO, + None, + None, + None, + None, + "expect_environment", + FunctionArgs::Bsatn(bsatn::to_vec(&args).unwrap().into()), + ) + .await; + assert!(result.is_err() || result.unwrap().outcome.into_result().is_err()); + }); +} + +#[test] +#[serial] +fn rust_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("environment-test"); +} + +#[test] +#[serial] +fn typescript_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-ts"); +} + +#[test] +#[serial] +fn cpp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-cpp"); +} + +#[test] +#[serial] +fn csharp_environment_reads_are_not_cached_and_preserve_missing_empty_utf8_and_nul() { + exercise_fixture("module-test-cs"); +} diff --git a/modules/environment-test/Cargo.toml b/modules/environment-test/Cargo.toml new file mode 100644 index 00000000000..9ae6466b734 --- /dev/null +++ b/modules/environment-test/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "environment-test" +version = "0.0.0" +edition.workspace = true +license-file = "../../LICENSE.txt" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies.spacetimedb] +workspace = true +features = ["unstable"] diff --git a/modules/environment-test/src/lib.rs b/modules/environment-test/src/lib.rs new file mode 100644 index 00000000000..fd4d1159df9 --- /dev/null +++ b/modules/environment-test/src/lib.rs @@ -0,0 +1,54 @@ +use spacetimedb::{AnonymousViewContext, ProcedureContext, ReducerContext, SpacetimeType}; + +#[spacetimedb::reducer] +pub fn expect_environment(ctx: &ReducerContext, key: String, expected: Option) { + assert_eq!(ctx.env.get(&key), expected); + assert_eq!(ctx.as_read_only().env.get(&key), expected); + assert_eq!(ctx.as_anonymous_read_only().env.get(&key), expected); +} + +#[spacetimedb::procedure] +pub fn read_environment(ctx: &mut ProcedureContext, key: String) -> Option { + let outside = ctx.env.get(&key); + ctx.with_tx(|tx| assert_eq!(tx.env.get(&key), outside)); + outside +} + +#[derive(SpacetimeType)] +pub struct EnvironmentValue { + pub value: Option, +} + +#[spacetimedb::view(accessor = environment_value, public)] +pub fn environment_value(ctx: &AnonymousViewContext) -> Option { + Some(EnvironmentValue { + value: ctx.env.get("WATCHED"), + }) +} + +/// Hand-written ABI callers cannot retain unbounded host allocations. +#[spacetimedb::reducer] +pub fn bounded_environment_sources(_ctx: &ReducerContext) { + use spacetimedb::sys::raw::{self, BytesSource}; + let mut sources = Vec::new(); + for i in 0..=256 { + let mut source = BytesSource::INVALID; + let status = unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }; + if i == 256 { + assert_eq!(status, 9); // NO_SPACE + } else { + assert_eq!(status, 0); + assert!(source != BytesSource::INVALID); + sources.push(source); + } + } + let mut buffer = [0u8; 8192]; + let mut len = buffer.len(); + let status = unsafe { raw::bytes_source_read(sources[0], buffer.as_mut_ptr(), &mut len) }; + assert_eq!(status, -1); + assert_eq!(len, buffer.len()); + let mut source = BytesSource::INVALID; + assert_eq!(unsafe { raw::env_get(b"LIMIT".as_ptr(), 5, &mut source) }, 0); + assert!(source != BytesSource::INVALID); + // The remaining sources are released when this invocation ends. +} diff --git a/modules/module-test-cpp/src/lib.cpp b/modules/module-test-cpp/src/lib.cpp index bc5305b1f16..b5035c68257 100644 --- a/modules/module-test-cpp/src/lib.cpp +++ b/modules/module-test-cpp/src/lib.cpp @@ -703,3 +703,15 @@ SPACETIMEDB_PROCEDURE(std::string, get_my_schema_via_http, ProcedureContext ctx) return result.error(); } } + +SPACETIMEDB_REDUCER(expect_environment, ReducerContext ctx, std::string key, std::optional expected) { + if (ctx.env.get(key) != expected) LOG_PANIC("environment value mismatch"); + return Ok(); +} +SPACETIMEDB_PROCEDURE(std::optional, read_environment, ProcedureContext ctx, std::string key) { + const auto outside = ctx.env.get(key); + ctx.with_tx([&](TxContext& tx) { + if (tx.env.get(key) != outside) LOG_PANIC("transaction environment value mismatch"); + }); + return outside; +} diff --git a/modules/module-test-cs/EnvironmentTests.cs b/modules/module-test-cs/EnvironmentTests.cs new file mode 100644 index 00000000000..1798d7bda44 --- /dev/null +++ b/modules/module-test-cs/EnvironmentTests.cs @@ -0,0 +1,25 @@ +#pragma warning disable STDB_UNSTABLE +namespace SpacetimeDB.Modules.ModuleTestCs; + +using SpacetimeDB; + +public static partial class EnvironmentTests +{ + [Reducer] + public static void expect_environment(ReducerContext ctx, string key, string? expected) + { + if (ctx.Env.Get(key) != expected) throw new Exception("environment value mismatch"); + } + + [Procedure] + public static string? read_environment(ProcedureContext ctx, string key) + { + var outside = ctx.Env.Get(key); + ctx.WithTx(tx => + { + if (tx.Env.Get(key) != outside) throw new Exception("transaction environment value mismatch"); + return true; + }); + return outside; + } +} diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 831abd5f657..a361112b266 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -520,3 +520,26 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { throw e; } }); + +// Dedicated environment ABI integration exercised by crates/testing. +export const expect_environment = spacetimedb.reducer( + { key: t.string(), expected: t.option(t.string()) }, + (ctx, { key, expected }) => { + if (ctx.env.get(key) !== (expected ?? null)) { + throw new Error('environment value mismatch'); + } + } +); +export const read_environment = spacetimedb.procedure( + { key: t.string() }, + t.option(t.string()), + (ctx, { key }) => { + const outside = ctx.env.get(key); + ctx.withTx(tx => { + if (tx.env.get(key) !== outside) { + throw new Error('transaction environment value mismatch'); + } + }); + return outside ?? undefined; + } +); From 2108552505ed5ca4452512fdfe78562fcf076178 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Mon, 7 Sep 2026 21:51:22 -0400 Subject: [PATCH 03/23] Extend V10 visibility without introducing V11 definitions --- crates/bindings-cpp/README.md | 2 +- .../internal/autogen/FunctionVisibility.g.h | 2 + .../autogen/FunctionVisibilityV11.g.h | 23 -- .../internal/autogen/RawModuleDef.g.h | 3 +- .../autogen/RawModuleDefV10Section.g.h | 2 +- .../internal/autogen/RawModuleDefV11.g.h | 27 -- .../autogen/RawModuleDefV11Section.g.h | 32 -- .../internal/autogen/RawProcedureDefV11.g.h | 35 -- .../internal/autogen/RawReducerDefV11.g.h | 37 -- .../spacetimedb/internal/v10_builder.h | 34 +- crates/bindings-cpp/src/internal/Module.cpp | 8 +- .../bindings-cpp/src/internal/v10_builder.cpp | 88 ++--- .../unit/function_visibility_unit_tests.cpp | 46 ++- crates/bindings-csharp/Codegen.Tests/Tests.cs | 12 +- .../diag/snapshots/Module#FFI.verified.cs | 32 +- .../snapshots/Module#FFI.verified.cs | 8 +- .../server/snapshots/Module#FFI.verified.cs | 24 +- crates/bindings-csharp/Codegen/Module.cs | 30 +- crates/bindings-csharp/README.md | 2 +- .../Runtime.Tests/FunctionVisibilityTests.cs | 44 ++- .../Internal/Autogen/FunctionVisibility.g.cs | 2 + .../Autogen/FunctionVisibilityV11.g.cs | 17 - .../Internal/Autogen/RawModuleDef.g.cs | 3 +- .../Autogen/RawModuleDefV10Section.g.cs | 3 +- .../Internal/Autogen/RawModuleDefV11.g.cs | 29 -- .../Autogen/RawModuleDefV11Section.g.cs | 27 -- .../Internal/Autogen/RawProcedureDefV11.g.cs | 45 --- .../Internal/Autogen/RawReducerDefV11.g.cs | 50 --- .../Runtime/Internal/IReducer.cs | 2 +- .../Runtime/Internal/Module.cs | 54 +-- .../Runtime/Internal/Procedure.cs | 2 +- crates/bindings-typescript/README.md | 5 +- .../src/lib/autogen/types.ts | 97 +---- crates/bindings-typescript/src/lib/schema.ts | 10 +- .../src/server/function_visibility.ts | 16 +- .../src/server/procedures.ts | 7 +- .../src/server/reducers.ts | 9 +- .../bindings-typescript/src/server/runtime.ts | 2 +- .../tests/hosted_auth.test.ts | 143 +++++-- crates/bindings/src/rt.rs | 11 +- .../tests/pass/function_visibility.rs | 30 +- crates/bindings/tests/ui/tables.stderr | 4 +- crates/cli/src/api.rs | 6 +- crates/cli/src/subcommands/describe.rs | 4 + crates/client-api/src/routes/database.rs | 10 +- crates/codegen/src/util.rs | 4 +- crates/core/src/auth/invocation/tests.rs | 4 +- crates/core/src/host/empty_module.rs | 2 +- crates/core/src/host/empty_module/README.md | 6 +- crates/core/src/host/empty_module/generate.py | 6 +- crates/core/src/host/empty_module/tests.rs | 14 +- .../src/host/empty_module/v1.schema.bsatn | Bin 33 -> 33 bytes crates/core/src/host/empty_module/v1.sha256 | 2 +- crates/core/src/host/empty_module/v1.wasm | Bin 250 -> 250 bytes crates/core/src/host/host_controller.rs | 5 +- .../subscription/module_subscription_actor.rs | 2 - crates/lib/src/db/raw_def.rs | 2 - crates/lib/src/db/raw_def/v10.rs | 229 ++++++++++- crates/lib/src/db/raw_def/v11.rs | 269 ------------- crates/lib/src/deployment.rs | 4 +- crates/lib/src/deployment/tests.rs | 6 +- crates/lib/src/lib.rs | 1 - crates/schema/src/def.rs | 289 +++----------- crates/schema/src/def/validate.rs | 1 - crates/schema/src/def/validate/v10.rs | 354 +++++++++++++++++- crates/schema/src/def/validate/v11.rs | 353 ----------------- crates/schema/src/error.rs | 2 +- .../tests/smoketests/http_routes.rs | 2 +- modules/module-test-ts/src/index.ts | 2 +- modules/module-test/src/lib.rs | 2 +- modules/sdk-test-procedure-ts/src/index.ts | 2 +- modules/sdk-test-procedure/src/lib.rs | 2 +- .../procedure-client/src/test_handlers.rs | 12 +- 73 files changed, 1042 insertions(+), 1614 deletions(-) delete mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibilityV11.g.h delete mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11.g.h delete mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11Section.g.h delete mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV11.g.h delete mode 100644 crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV11.g.h delete mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibilityV11.g.cs delete mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11.g.cs delete mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11Section.g.cs delete mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/RawProcedureDefV11.g.cs delete mode 100644 crates/bindings-csharp/Runtime/Internal/Autogen/RawReducerDefV11.g.cs delete mode 100644 crates/lib/src/db/raw_def/v11.rs delete mode 100644 crates/schema/src/def/validate/v11.rs diff --git a/crates/bindings-cpp/README.md b/crates/bindings-cpp/README.md index bedb5f515d8..4fbfc2a2d86 100644 --- a/crates/bindings-cpp/README.md +++ b/crates/bindings-cpp/README.md @@ -24,7 +24,7 @@ functions also admit the owner, and public functions admit any client. independent of connection and JWT presence, so an internal call can have a JWT. JWT identity is the verified sender supplied by the host. Procedures preserve this authentication in `with_tx` and `try_with_tx`. Newly compiled modules emit -schema V11 and advertise `hosted_auth_v1`, requiring a compatible host. +schema V10 and advertise `hosted_auth_v1`, requiring a compatible host. ## Current State diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h index 423276de9b4..9795b3bd2d7 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibility.g.h @@ -18,5 +18,7 @@ namespace SpacetimeDB::Internal { enum class FunctionVisibility : uint8_t { Private = 0, ClientCallable = 1, + Internal = 2, + ExplicitClientCallable = 3, }; } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibilityV11.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibilityV11.g.h deleted file mode 100644 index e8f1ffadaa7..00000000000 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/FunctionVisibilityV11.g.h +++ /dev/null @@ -1,23 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -// This was generated using spacetimedb codegen. - -#pragma once - -#include -#include -#include -#include -#include -#include "../autogen_base.h" -#include "spacetimedb/bsatn/bsatn.h" - -namespace SpacetimeDB::Internal { - -enum class FunctionVisibilityV11 : uint8_t { - Private = 0, - ClientCallable = 1, - Internal = 2, -}; -} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h index 6987a9edd0e..c7f144eb07d 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDef.g.h @@ -12,12 +12,11 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawModuleDefV11.g.h" #include "RawModuleDefV8.g.h" #include "RawModuleDefV9.g.h" #include "RawModuleDefV10.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDef, SpacetimeDB::Internal::RawModuleDefV8, SpacetimeDB::Internal::RawModuleDefV9, SpacetimeDB::Internal::RawModuleDefV10, SpacetimeDB::Internal::RawModuleDefV11) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDef, SpacetimeDB::Internal::RawModuleDefV8, SpacetimeDB::Internal::RawModuleDefV9, SpacetimeDB::Internal::RawModuleDefV10) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index 7b8a01cdb51..d5a27b62571 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -28,5 +28,5 @@ namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11.g.h deleted file mode 100644 index c787ba63fb4..00000000000 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11.g.h +++ /dev/null @@ -1,27 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -// This was generated using spacetimedb codegen. - -#pragma once - -#include -#include -#include -#include -#include -#include "../autogen_base.h" -#include "spacetimedb/bsatn/bsatn.h" -#include "RawModuleDefV11Section.g.h" - -namespace SpacetimeDB::Internal { - -SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawModuleDefV11) { - std::vector sections; - - void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { - ::SpacetimeDB::bsatn::serialize(writer, sections); - } - SPACETIMEDB_PRODUCT_TYPE_EQUALITY(sections) -}; -} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11Section.g.h deleted file mode 100644 index 359648e5304..00000000000 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV11Section.g.h +++ /dev/null @@ -1,32 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -// This was generated using spacetimedb codegen. - -#pragma once - -#include -#include -#include -#include -#include -#include "../autogen_base.h" -#include "spacetimedb/bsatn/bsatn.h" -#include "RawProcedureDefV11.g.h" -#include "RawTableDefV10.g.h" -#include "RawScheduleDefV10.g.h" -#include "CaseConversionPolicy.g.h" -#include "ExplicitNames.g.h" -#include "RawHttpHandlerDefV10.g.h" -#include "RawHttpRouteDefV10.g.h" -#include "RawTypeDefV10.g.h" -#include "RawReducerDefV11.g.h" -#include "Typespace.g.h" -#include "RawLifeCycleReducerDefV10.g.h" -#include "RawViewDefV10.g.h" -#include "RawRowLevelSecurityDefV9.g.h" - -namespace SpacetimeDB::Internal { - -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV11Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector) -} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV11.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV11.g.h deleted file mode 100644 index 2ac5d359352..00000000000 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV11.g.h +++ /dev/null @@ -1,35 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -// This was generated using spacetimedb codegen. - -#pragma once - -#include -#include -#include -#include -#include -#include "../autogen_base.h" -#include "spacetimedb/bsatn/bsatn.h" -#include "FunctionVisibilityV11.g.h" -#include "ProductType.g.h" -#include "AlgebraicType.g.h" - -namespace SpacetimeDB::Internal { - -SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawProcedureDefV11) { - std::string source_name; - SpacetimeDB::Internal::ProductType params; - std::optional declared_visibility; - SpacetimeDB::Internal::AlgebraicType return_type; - - void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { - ::SpacetimeDB::bsatn::serialize(writer, source_name); - ::SpacetimeDB::bsatn::serialize(writer, params); - ::SpacetimeDB::bsatn::serialize(writer, declared_visibility); - ::SpacetimeDB::bsatn::serialize(writer, return_type); - } - SPACETIMEDB_PRODUCT_TYPE_EQUALITY(source_name, params, declared_visibility, return_type) -}; -} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV11.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV11.g.h deleted file mode 100644 index 1eee0f8fb76..00000000000 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV11.g.h +++ /dev/null @@ -1,37 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -// This was generated using spacetimedb codegen. - -#pragma once - -#include -#include -#include -#include -#include -#include "../autogen_base.h" -#include "spacetimedb/bsatn/bsatn.h" -#include "ProductType.g.h" -#include "FunctionVisibilityV11.g.h" -#include "AlgebraicType.g.h" - -namespace SpacetimeDB::Internal { - -SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawReducerDefV11) { - std::string source_name; - SpacetimeDB::Internal::ProductType params; - std::optional declared_visibility; - SpacetimeDB::Internal::AlgebraicType ok_return_type; - SpacetimeDB::Internal::AlgebraicType err_return_type; - - void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { - ::SpacetimeDB::bsatn::serialize(writer, source_name); - ::SpacetimeDB::bsatn::serialize(writer, params); - ::SpacetimeDB::bsatn::serialize(writer, declared_visibility); - ::SpacetimeDB::bsatn::serialize(writer, ok_return_type); - ::SpacetimeDB::bsatn::serialize(writer, err_return_type); - } - SPACETIMEDB_PRODUCT_TYPE_EQUALITY(source_name, params, declared_visibility, ok_return_type, err_return_type) -}; -} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h index 2c4dbdda684..5b893d9bc6f 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h @@ -21,11 +21,11 @@ #include "autogen/SumType.g.h" #include "autogen/ProductType.g.h" #include "autogen/ProductTypeElement.g.h" -#include "autogen/RawModuleDefV11.g.h" +#include "autogen/RawModuleDefV10.g.h" #include "autogen/Typespace.g.h" #include "autogen/RawTableDefV10.g.h" -#include "autogen/RawReducerDefV11.g.h" -#include "autogen/RawProcedureDefV11.g.h" +#include "autogen/RawReducerDefV10.g.h" +#include "autogen/RawProcedureDefV10.g.h" #include "autogen/RawViewDefV10.g.h" #include "autogen/RawScheduleDefV10.g.h" #include "autogen/RawLifeCycleReducerDefV10.g.h" @@ -48,7 +48,7 @@ void fail_reducer(std::string message); namespace Internal { -// The historical facade name is retained; newly compiled modules serialize V11. +// Builds the V10 module definition with explicit function visibility. class V10Builder { public: V10Builder() = default; @@ -384,10 +384,10 @@ class V10Builder { }(std::make_index_sequence{}, params, param_names, type_reg); } - RawReducerDefV11 reducer_def{ + RawReducerDefV10 reducer_def{ reducer_name, std::move(params), - std::nullopt, + FunctionVisibility::ClientCallable, MakeUnitAlgebraicType(), MakeStringAlgebraicType(), }; @@ -434,10 +434,10 @@ class V10Builder { } RegisterReducerHandler(reducer_name, handler, lifecycle); - RawReducerDefV11 reducer_def{ + RawReducerDefV10 reducer_def{ reducer_name, ProductType{}, - std::nullopt, + FunctionVisibility::Internal, MakeUnitAlgebraicType(), MakeStringAlgebraicType(), }; @@ -581,11 +581,11 @@ class V10Builder { }(std::make_index_sequence{}, params, param_names, type_reg); } - RawProcedureDefV11 procedure_def{ + RawProcedureDefV10 procedure_def{ procedure_name, std::move(params), - std::nullopt, return_type, + FunctionVisibility::ClientCallable, }; UpsertProcedure(procedure_def); } @@ -628,15 +628,15 @@ class V10Builder { void SetFunctionVisibility(const std::string& source_name, ::SpacetimeDB::FunctionVisibility visibility); void RegisterExplicitIndexName(const std::string& source_name, const std::string& canonical_name); - RawModuleDefV11 BuildModuleDef() const; + RawModuleDefV10 BuildModuleDef() const; Typespace& GetTypespace() { return typespace_; } const Typespace& GetTypespace() const { return typespace_; } std::vector& GetTypeDefs() { return types_; } const std::vector& GetTypeDefs() const { return types_; } std::vector& GetTables() { return tables_; } const std::vector& GetTables() const { return tables_; } - std::vector& GetReducers() { return reducers_; } - const std::vector& GetReducers() const { return reducers_; } + std::vector& GetReducers() { return reducers_; } + const std::vector& GetReducers() const { return reducers_; } const std::optional& GetCaseConversionPolicy() const { return case_conversion_policy_; } const std::vector& GetExplicitNames() const { return explicit_names_; } const std::vector& GetHttpHandlers() const { return http_handlers_; } @@ -648,8 +648,8 @@ class V10Builder { } void UpsertTable(const RawTableDefV10& table); void UpsertLifecycleReducer(const RawLifeCycleReducerDefV10& lifecycle); - void UpsertReducer(const RawReducerDefV11& reducer); - void UpsertProcedure(const RawProcedureDefV11& procedure); + void UpsertReducer(const RawReducerDefV10& reducer); + void UpsertProcedure(const RawProcedureDefV10& procedure); void UpsertView(const RawViewDefV10& view); void UpsertHttpHandler(const RawHttpHandlerDefV10& handler); RawIndexDefV10 CreateBTreeIndex(const std::string& table_name, @@ -667,8 +667,8 @@ class V10Builder { std::vector explicit_names_; std::unordered_map> column_defaults_by_table_; std::vector tables_; - std::vector reducers_; - std::vector procedures_; + std::vector reducers_; + std::vector procedures_; std::vector views_; std::vector http_handlers_; std::vector http_routes_; diff --git a/crates/bindings-cpp/src/internal/Module.cpp b/crates/bindings-cpp/src/internal/Module.cpp index d1a751264c1..901bb5e3e86 100644 --- a/crates/bindings-cpp/src/internal/Module.cpp +++ b/crates/bindings-cpp/src/internal/Module.cpp @@ -5,7 +5,7 @@ #include "spacetimedb/internal/Module.h" #include "spacetimedb/internal/buffer_pool.h" #include "spacetimedb/internal/autogen/RawModuleDef.g.h" -#include "spacetimedb/internal/autogen/RawModuleDefV11.g.h" +#include "spacetimedb/internal/autogen/RawModuleDefV10.g.h" #include "spacetimedb/internal/autogen/RawTypeDefV10.g.h" #include "spacetimedb/internal/v9_builder.h" #include "spacetimedb/internal/v10_builder.h" @@ -373,9 +373,9 @@ void __preinit__99_validate_types() { std::vector Internal::Module::SerializeModuleDef() { - RawModuleDefV11 v11_module = getV10Builder().BuildModuleDef(); + RawModuleDefV10 v10_module = getV10Builder().BuildModuleDef(); RawModuleDef versioned_module; - versioned_module.set<3>(std::move(v11_module)); + versioned_module.set<2>(std::move(v10_module)); std::vector buffer; bsatn::Writer writer(buffer); @@ -383,7 +383,7 @@ std::vector Internal::Module::SerializeModuleDef() { return buffer; } -// FFI export - V11 serialization +// FFI export - V10 serialization void Internal::Module::__describe_module__(BytesSink sink) { // The preinit functions should have already been called by SpacetimeDB // Including our validation preinit which checks for recursive types diff --git a/crates/bindings-cpp/src/internal/v10_builder.cpp b/crates/bindings-cpp/src/internal/v10_builder.cpp index 336a8151478..d7eadd210ae 100644 --- a/crates/bindings-cpp/src/internal/v10_builder.cpp +++ b/crates/bindings-cpp/src/internal/v10_builder.cpp @@ -2,10 +2,10 @@ #include "spacetimedb/internal/autogen/AlgebraicType.g.h" #include "spacetimedb/internal/autogen/ProductType.g.h" #include "spacetimedb/internal/autogen/ProductTypeElement.g.h" -#include "spacetimedb/internal/autogen/RawModuleDefV11Section.g.h" +#include "spacetimedb/internal/autogen/RawModuleDefV10Section.g.h" #include "spacetimedb/internal/autogen/RawTypeDefV10.g.h" #include "spacetimedb/internal/autogen/RawScopedTypeNameV10.g.h" -#include "spacetimedb/internal/autogen/FunctionVisibilityV11.g.h" +#include "spacetimedb/internal/autogen/FunctionVisibility.g.h" #include "spacetimedb/internal/autogen/ExplicitNames.g.h" #include "spacetimedb/router.h" #include @@ -120,7 +120,7 @@ void V10Builder::UpsertLifecycleReducer(const RawLifeCycleReducerDefV10& lifecyc } } -void V10Builder::UpsertReducer(const RawReducerDefV11& reducer) { +void V10Builder::UpsertReducer(const RawReducerDefV10& reducer) { auto it = std::find_if(reducers_.begin(), reducers_.end(), [&](const auto& existing) { return existing.source_name == reducer.source_name; }); @@ -131,7 +131,7 @@ void V10Builder::UpsertReducer(const RawReducerDefV11& reducer) { } } -void V10Builder::UpsertProcedure(const RawProcedureDefV11& procedure) { +void V10Builder::UpsertProcedure(const RawProcedureDefV10& procedure) { auto it = std::find_if(procedures_.begin(), procedures_.end(), [&](const auto& existing) { return existing.source_name == procedure.source_name; }); @@ -210,107 +210,107 @@ RawConstraintDefV10 V10Builder::CreateUniqueConstraint(const std::string& table_ } void V10Builder::SetFunctionVisibility(const std::string& name, ::SpacetimeDB::FunctionVisibility visibility) { - FunctionVisibilityV11 declared; + FunctionVisibility declared; switch (visibility) { - case ::SpacetimeDB::FunctionVisibility::Public: declared = FunctionVisibilityV11::ClientCallable; break; - case ::SpacetimeDB::FunctionVisibility::Private: declared = FunctionVisibilityV11::Private; break; - case ::SpacetimeDB::FunctionVisibility::Internal: declared = FunctionVisibilityV11::Internal; break; + case ::SpacetimeDB::FunctionVisibility::Public: declared = FunctionVisibility::ExplicitClientCallable; break; + case ::SpacetimeDB::FunctionVisibility::Private: declared = FunctionVisibility::Private; break; + case ::SpacetimeDB::FunctionVisibility::Internal: declared = FunctionVisibility::Internal; break; default: SetConstraintRegistrationError("INVALID_FUNCTION_VISIBILITY", "function='" + name + "'"); return; } for (const auto& lifecycle : lifecycle_reducers_) { - if (lifecycle.function_name == name && declared != FunctionVisibilityV11::Internal) { + if (lifecycle.function_name == name && declared != FunctionVisibility::Internal) { SetConstraintRegistrationError("INVALID_LIFECYCLE_VISIBILITY", "function='" + name + "' must be Internal"); return; } } for (auto& reducer : reducers_) { - if (reducer.source_name == name) { reducer.declared_visibility = declared; return; } + if (reducer.source_name == name) { reducer.visibility = declared; return; } } for (auto& procedure : procedures_) { - if (procedure.source_name == name) { procedure.declared_visibility = declared; return; } + if (procedure.source_name == name) { procedure.visibility = declared; return; } } SetConstraintRegistrationError("UNKNOWN_FUNCTION_VISIBILITY", "function='" + name + "' is not a reducer or procedure"); } -RawModuleDefV11 V10Builder::BuildModuleDef() const { - RawModuleDefV11 v11_module; +RawModuleDefV10 V10Builder::BuildModuleDef() const { + RawModuleDefV10 v10_module; std::vector types = types_; - std::vector reducers = reducers_; - std::vector procedures = procedures_; + std::vector reducers = reducers_; + std::vector procedures = procedures_; - RawModuleDefV11Section section_typespace; + RawModuleDefV10Section section_typespace; section_typespace.set<0>(typespace_); - v11_module.sections.push_back(section_typespace); - RawModuleDefV11Section capabilities; + v10_module.sections.push_back(section_typespace); + RawModuleDefV10Section capabilities; capabilities.set<13>(std::vector{"hosted_auth_v1"}); - v11_module.sections.push_back(std::move(capabilities)); + v10_module.sections.push_back(std::move(capabilities)); if (!types.empty()) { - RawModuleDefV11Section section_types; + RawModuleDefV10Section section_types; section_types.set<1>(std::move(types)); - v11_module.sections.push_back(std::move(section_types)); + v10_module.sections.push_back(std::move(section_types)); } if (!tables_.empty()) { - RawModuleDefV11Section section_tables; + RawModuleDefV10Section section_tables; section_tables.set<2>(tables_); - v11_module.sections.push_back(std::move(section_tables)); + v10_module.sections.push_back(std::move(section_tables)); } if (!reducers.empty()) { - RawModuleDefV11Section section_reducers; + RawModuleDefV10Section section_reducers; section_reducers.set<3>(std::move(reducers)); - v11_module.sections.push_back(std::move(section_reducers)); + v10_module.sections.push_back(std::move(section_reducers)); } if (!procedures.empty()) { - RawModuleDefV11Section section_procedures; + RawModuleDefV10Section section_procedures; section_procedures.set<4>(std::move(procedures)); - v11_module.sections.push_back(std::move(section_procedures)); + v10_module.sections.push_back(std::move(section_procedures)); } if (!views_.empty()) { - RawModuleDefV11Section section_views; + RawModuleDefV10Section section_views; section_views.set<5>(views_); - v11_module.sections.push_back(std::move(section_views)); + v10_module.sections.push_back(std::move(section_views)); } if (!schedules_.empty()) { - RawModuleDefV11Section section_schedules; + RawModuleDefV10Section section_schedules; section_schedules.set<6>(schedules_); - v11_module.sections.push_back(std::move(section_schedules)); + v10_module.sections.push_back(std::move(section_schedules)); } if (!lifecycle_reducers_.empty()) { - RawModuleDefV11Section section_lifecycle; + RawModuleDefV10Section section_lifecycle; section_lifecycle.set<7>(lifecycle_reducers_); - v11_module.sections.push_back(std::move(section_lifecycle)); + v10_module.sections.push_back(std::move(section_lifecycle)); } if (case_conversion_policy_.has_value()) { - RawModuleDefV11Section section_case_policy; + RawModuleDefV10Section section_case_policy; section_case_policy.set<9>(case_conversion_policy_.value()); - v11_module.sections.push_back(std::move(section_case_policy)); + v10_module.sections.push_back(std::move(section_case_policy)); } if (!explicit_names_.empty()) { - RawModuleDefV11Section section_explicit_names; + RawModuleDefV10Section section_explicit_names; section_explicit_names.set<10>(ExplicitNames{explicit_names_}); - v11_module.sections.push_back(std::move(section_explicit_names)); + v10_module.sections.push_back(std::move(section_explicit_names)); } if (!http_handlers_.empty()) { - RawModuleDefV11Section section_http_handlers; + RawModuleDefV10Section section_http_handlers; section_http_handlers.set<11>(http_handlers_); - v11_module.sections.push_back(std::move(section_http_handlers)); + v10_module.sections.push_back(std::move(section_http_handlers)); } if (!http_routes_.empty()) { - RawModuleDefV11Section section_http_routes; + RawModuleDefV10Section section_http_routes; section_http_routes.set<12>(http_routes_); - v11_module.sections.push_back(std::move(section_http_routes)); + v10_module.sections.push_back(std::move(section_http_routes)); } if (!row_level_security_.empty()) { - RawModuleDefV11Section section_rls; + RawModuleDefV10Section section_rls; section_rls.set<8>(row_level_security_); - v11_module.sections.push_back(std::move(section_rls)); + v10_module.sections.push_back(std::move(section_rls)); } - return v11_module; + return v10_module; } } // namespace Internal diff --git a/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp index 7365c046baa..23cff3a0482 100644 --- a/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp +++ b/crates/bindings-cpp/tests/unit/function_visibility_unit_tests.cpp @@ -24,14 +24,14 @@ TEST_CASE(visibility_macro_applies_after_function_registration) { if (section.get_tag() != 3) continue; for (const auto& reducer : section.get<3>()) { if (reducer.source_name != "visibility_macro_target") continue; - ASSERT_EQ(FunctionVisibilityV11::Internal, *reducer.declared_visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, reducer.visibility); found = true; } } ASSERT_TRUE(found); } -TEST_CASE(v11_retains_explicit_visibility_and_schedule_default_omission) { +TEST_CASE(v10_retains_explicit_visibility_and_schedule_default) { V10Builder builder; builder.RegisterReducer("omitted", &noop, {}); builder.RegisterReducer("public", &noop, {}); @@ -46,24 +46,24 @@ TEST_CASE(v11_retains_explicit_visibility_and_schedule_default_omission) { builder.SetFunctionVisibility("procedure", SpacetimeDB::FunctionVisibility::Internal); RawModuleDef versioned; - versioned.set<3>(builder.BuildModuleDef()); + versioned.set<2>(builder.BuildModuleDef()); std::vector bytes; bsatn::Writer writer(bytes); bsatn::serialize(writer, versioned); - ASSERT_EQ(uint8_t{3}, bytes.at(0)); - ASSERT_EQ(uint8_t{3}, versioned.get_tag()); + ASSERT_EQ(uint8_t{2}, bytes.at(0)); + ASSERT_EQ(uint8_t{2}, versioned.get_tag()); bool saw_reducers = false, saw_procedure = false, saw_capability = false; - for (const auto& section : versioned.get<3>().sections) { + for (const auto& section : versioned.get<2>().sections) { if (section.get_tag() == 3) { const auto& reducers = section.get<3>(); ASSERT_EQ(size_t{4}, reducers.size()); - ASSERT_TRUE(!reducers[0].declared_visibility.has_value()); - ASSERT_EQ(FunctionVisibilityV11::ClientCallable, *reducers[1].declared_visibility); - ASSERT_EQ(FunctionVisibilityV11::Private, *reducers[2].declared_visibility); - ASSERT_EQ(FunctionVisibilityV11::Internal, *reducers[3].declared_visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::ClientCallable, reducers[0].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::ExplicitClientCallable, reducers[1].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Private, reducers[2].visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, reducers[3].visibility); saw_reducers = true; } else if (section.get_tag() == 4) { - ASSERT_EQ(FunctionVisibilityV11::Internal, *section.get<4>().at(0).declared_visibility); + ASSERT_EQ(SpacetimeDB::Internal::FunctionVisibility::Internal, section.get<4>().at(0).visibility); saw_procedure = true; } else if (section.get_tag() == 13) { ASSERT_EQ(std::vector{"hosted_auth_v1"}, section.get<13>()); @@ -72,3 +72,27 @@ TEST_CASE(v11_retains_explicit_visibility_and_schedule_default_omission) { } ASSERT_TRUE(saw_reducers && saw_procedure && saw_capability); } + +TEST_CASE(v10_visibility_extends_enum_without_changing_reducer_field_layout) { + V10Builder builder; + builder.RegisterReducer("r", &noop, {}); + auto reducer = builder.GetReducers().at(0); + for (uint8_t tag = 0; tag <= 3; ++tag) { + reducer.visibility = static_cast(tag); + std::vector bytes; + bsatn::Writer writer(bytes); + bsatn::serialize(writer, reducer); + const std::vector expected{1, 0, 0, 0, 'r', 0, 0, 0, 0, tag, 2, 0, 0, 0, 0, 4}; + ASSERT_EQ(expected, bytes); + const RawProcedureDefV10 procedure_def{ + "p", ProductType{}, reducer.ok_return_type, reducer.visibility, + }; + std::vector procedure_bytes; + bsatn::Writer procedure_writer(procedure_bytes); + bsatn::serialize(procedure_writer, procedure_def); + const std::vector expected_procedure{ + 1, 0, 0, 0, 'p', 0, 0, 0, 0, 2, 0, 0, 0, 0, tag, + }; + ASSERT_EQ(expected_procedure, procedure_bytes); + } +} diff --git a/crates/bindings-csharp/Codegen.Tests/Tests.cs b/crates/bindings-csharp/Codegen.Tests/Tests.cs index 40e8c051ff4..849f40b0b43 100644 --- a/crates/bindings-csharp/Codegen.Tests/Tests.cs +++ b/crates/bindings-csharp/Codegen.Tests/Tests.cs @@ -380,17 +380,11 @@ public static void InternalJob(ReducerContext ctx) {} Assert.Empty(GetCompilationErrors(compilation.AddSyntaxTrees(result.GeneratedTrees))); var generated = string.Join("\n", result.GeneratedTrees.Select(t => t.ToString())); Assert.Contains( - "DeclaredVisibility: SpacetimeDB.Internal.FunctionVisibilityV11.ClientCallable", - generated - ); - Assert.Contains( - "DeclaredVisibility: SpacetimeDB.Internal.FunctionVisibilityV11.Private", - generated - ); - Assert.Contains( - "DeclaredVisibility: SpacetimeDB.Internal.FunctionVisibilityV11.Internal", + "Visibility: SpacetimeDB.Internal.FunctionVisibility.ExplicitClientCallable", generated ); + Assert.Contains("Visibility: SpacetimeDB.Internal.FunctionVisibility.Private", generated); + Assert.Contains("Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal", generated); var invalid = CSharpSyntaxTree.ParseText( """ diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs index 667077f7b6e..9762f360c78 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/diag/snapshots/Module#FFI.verified.cs @@ -3026,13 +3026,13 @@ static class ModuleRegistration { class __ReducerWithReservedPrefix : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(__ReducerWithReservedPrefix), Params: [], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3049,13 +3049,13 @@ class DummyScheduledReducer : SpacetimeDB.Internal.IReducer { private static readonly TestScheduleIssues.BSATN tableRW = new(); - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(DummyScheduledReducer), Params: [new("table", tableRW.GetAlgebraicType(registrar))], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3073,13 +3073,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class OnReducerWithReservedPrefix : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(OnReducerWithReservedPrefix), Params: [], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3094,13 +3094,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestDuplicateReducerKind1 : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestDuplicateReducerKind1), Params: [], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3115,13 +3115,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestDuplicateReducerKind2 : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestDuplicateReducerKind2), Params: [], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3136,13 +3136,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestDuplicateReducerName : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestDuplicateReducerName), Params: [], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3157,13 +3157,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestReducerReturnType : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestReducerReturnType), Params: [], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -3178,13 +3178,13 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class TestReducerWithoutContext : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(TestReducerWithoutContext), Params: [], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs index 1b4f68886d4..8773f431bde 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/explicitnames/snapshots/Module#FFI.verified.cs @@ -522,13 +522,13 @@ class DemoReducer : SpacetimeDB.Internal.IReducer { private static readonly SpacetimeDB.BSATN.I32 valueRW = new(); - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(DemoReducer), Params: [new("value", valueRW.GetAlgebraicType(registrar))], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -543,14 +543,14 @@ public void Invoke(BinaryReader reader, SpacetimeDB.Internal.IReducerContext ctx class DemoProcedure : SpacetimeDB.Internal.IProcedure { - public SpacetimeDB.Internal.RawProcedureDefV11 MakeProcedureDef( + public SpacetimeDB.Internal.RawProcedureDefV10 MakeProcedureDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(DemoProcedure), Params: [], ReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, - DeclaredVisibility: null + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable ); public byte[] Invoke(BinaryReader reader, SpacetimeDB.Internal.IProcedureContext ctx) diff --git a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs index 346631b0583..8c0072cae5a 100644 --- a/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs +++ b/crates/bindings-csharp/Codegen.Tests/fixtures/server/snapshots/Module#FFI.verified.cs @@ -2331,13 +2331,13 @@ static class ModuleRegistration { class Init : SpacetimeDB.Internal.IReducer { - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(Init), Params: [], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.Internal, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2354,13 +2354,13 @@ class InsertData : SpacetimeDB.Internal.IReducer { private static readonly PublicTable.BSATN dataRW = new(); - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(InsertData), Params: [new("data", dataRW.GetAlgebraicType(registrar))], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2377,13 +2377,13 @@ class InsertData2 : SpacetimeDB.Internal.IReducer { private static readonly PublicTable.BSATN dataRW = new(); - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(InsertData2), Params: [new("data", dataRW.GetAlgebraicType(registrar))], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2403,13 +2403,13 @@ class InsertMultiData : SpacetimeDB.Internal.IReducer { private static readonly MultiTableRow.BSATN dataRW = new(); - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(InsertMultiData), Params: [new("data", dataRW.GetAlgebraicType(registrar))], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2426,13 +2426,13 @@ class ScheduleImmediate : SpacetimeDB.Internal.IReducer { private static readonly PublicTable.BSATN dataRW = new(); - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(ScheduleImmediate), Params: [new("data", dataRW.GetAlgebraicType(registrar))], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -2449,13 +2449,13 @@ class SendScheduledMessage : SpacetimeDB.Internal.IReducer { private static readonly Timers.SendMessageTimer.BSATN argRW = new(); - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef( SpacetimeDB.BSATN.ITypeRegistrar registrar ) => new( SourceName: nameof(SendScheduledMessage), Params: [new("arg", argRW.GetAlgebraicType(registrar))], - DeclaredVisibility: null, + Visibility: SpacetimeDB.Internal.FunctionVisibility.ClientCallable, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); diff --git a/crates/bindings-csharp/Codegen/Module.cs b/crates/bindings-csharp/Codegen/Module.cs index 097918f3df5..c6268371d59 100644 --- a/crates/bindings-csharp/Codegen/Module.cs +++ b/crates/bindings-csharp/Codegen/Module.cs @@ -1405,7 +1405,7 @@ public byte[] Invoke( } /// -/// Validates a declared function visibility and maps it to the V11 schema. +/// Validates a declared function visibility and maps it to the V10 schema. /// static class FunctionVisibilityDeclaration { @@ -1424,15 +1424,17 @@ DiagReporter diag ) { diag.Report(ErrorDescriptor.InvalidFunctionVisibility, method); - return "null"; + return "SpacetimeDB.Internal.FunctionVisibility.Internal"; } return visibility switch { FunctionVisibility.Public => - "SpacetimeDB.Internal.FunctionVisibilityV11.ClientCallable", - FunctionVisibility.Private => "SpacetimeDB.Internal.FunctionVisibilityV11.Private", - FunctionVisibility.Internal => "SpacetimeDB.Internal.FunctionVisibilityV11.Internal", - _ => "null", + "SpacetimeDB.Internal.FunctionVisibility.ExplicitClientCallable", + FunctionVisibility.Private => "SpacetimeDB.Internal.FunctionVisibility.Private", + FunctionVisibility.Internal => "SpacetimeDB.Internal.FunctionVisibility.Internal", + _ => lifecycle + ? "SpacetimeDB.Internal.FunctionVisibility.Internal" + : "SpacetimeDB.Internal.FunctionVisibility.ClientCallable", }; } } @@ -1442,7 +1444,7 @@ record ReducerDeclaration public readonly string Name; public readonly string? CanonicalName; public readonly ReducerKind Kind; - public readonly string DeclaredVisibility; + public readonly string Visibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1481,7 +1483,7 @@ public ReducerDeclaration(GeneratorAttributeSyntaxContext context, DiagReporter } Kind = attr.Kind; - DeclaredVisibility = FunctionVisibilityDeclaration.Resolve( + Visibility = FunctionVisibilityDeclaration.Resolve( attr.Visibility, Kind != ReducerKind.UserDefined, methodSyntax, @@ -1512,10 +1514,10 @@ public string GenerateClass() class {{Identifier}}: SpacetimeDB.Internal.IReducer { {{MemberDeclaration.GenerateBsatnFields(Accessibility.Private, Args)}} - public SpacetimeDB.Internal.RawReducerDefV11 MakeReducerDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new ( + public SpacetimeDB.Internal.RawReducerDefV10 MakeReducerDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new ( SourceName: nameof({{Identifier}}), Params: [{{MemberDeclaration.GenerateDefs(Args)}}], - DeclaredVisibility: {{DeclaredVisibility}}, + Visibility: {{Visibility}}, OkReturnType: SpacetimeDB.BSATN.AlgebraicType.Unit, ErrReturnType: new SpacetimeDB.BSATN.AlgebraicType.String(default) ); @@ -1572,7 +1574,7 @@ record ProcedureDeclaration { public readonly string Name; public readonly string? CanonicalName; - public readonly string DeclaredVisibility; + public readonly string Visibility; public readonly string FullName; public readonly EquatableArray Args; public readonly Scope Scope; @@ -1589,7 +1591,7 @@ public ProcedureDeclaration(GeneratorAttributeSyntaxContext context, DiagReporte var methodSyntax = (MethodDeclarationSyntax)context.TargetNode; var method = (IMethodSymbol)context.TargetSymbol; var attr = context.Attributes.Single().ParseAs(); - DeclaredVisibility = FunctionVisibilityDeclaration.Resolve( + Visibility = FunctionVisibilityDeclaration.Resolve( attr.Visibility, false, methodSyntax, @@ -1750,11 +1752,11 @@ public string GenerateClass() class {{{Identifier}}} : SpacetimeDB.Internal.IProcedure { {{{classFields}}} - public SpacetimeDB.Internal.RawProcedureDefV11 MakeProcedureDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new( + public SpacetimeDB.Internal.RawProcedureDefV10 MakeProcedureDef(SpacetimeDB.BSATN.ITypeRegistrar registrar) => new( SourceName: nameof({{{Identifier}}}), Params: [{{{MemberDeclaration.GenerateDefs(Args)}}}], ReturnType: {{{returnTypeExpr}}}, - DeclaredVisibility: {{{DeclaredVisibility}}} + Visibility: {{{Visibility}}} ); public byte[] Invoke(BinaryReader reader, SpacetimeDB.Internal.IProcedureContext ctx) { diff --git a/crates/bindings-csharp/README.md b/crates/bindings-csharp/README.md index d75b17677b1..94158b66fb5 100644 --- a/crates/bindings-csharp/README.md +++ b/crates/bindings-csharp/README.md @@ -25,7 +25,7 @@ public static void ProcessJobs(ReducerContext ctx) { } `ctx.SenderAuth.IsInternal` comes from the host's invocation authority. It is independent of connection and JWT presence, so an internal call can have a JWT. JWT identity is the verified sender supplied by the host. Newly compiled modules -emit schema V11 and advertise `hosted_auth_v1`, requiring a compatible host. +emit schema V10 and advertise `hosted_auth_v1`, requiring a compatible host. These projects contain the SpacetimeDB SATS typesystem, codegen and runtime bindings for SpacetimeDB WebAssembly modules. It also contains serialization code for SpacetimeDB C# clients. diff --git a/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs index 544391af3da..d50e6cff7f4 100644 --- a/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs +++ b/crates/bindings-csharp/Runtime.Tests/FunctionVisibilityTests.cs @@ -6,14 +6,28 @@ namespace Runtime.Tests; public class FunctionVisibilityTests { [Theory] - [InlineData(null)] - [InlineData(FunctionVisibilityV11.ClientCallable)] - [InlineData(FunctionVisibilityV11.Private)] - [InlineData(FunctionVisibilityV11.Internal)] - public void SchedulingPreservesDeclaredVisibility(FunctionVisibilityV11? visibility) + [InlineData(FunctionVisibility.Private, 0)] + [InlineData(FunctionVisibility.ClientCallable, 1)] + [InlineData(FunctionVisibility.Internal, 2)] + [InlineData(FunctionVisibility.ExplicitClientCallable, 3)] + public void V10RetainsVisibilityEnumEncoding(FunctionVisibility visibility, byte tag) { - var module = new RawModuleDefV11(); - var reducer = new RawReducerDefV11( + var bytes = IStructuralReadWrite.ToBytes( + new SpacetimeDB.BSATN.Enum(), + visibility + ); + Assert.Equal(new byte[] { tag }, bytes); + } + + [Theory] + [InlineData(FunctionVisibility.ExplicitClientCallable)] + [InlineData(FunctionVisibility.ClientCallable)] + [InlineData(FunctionVisibility.Private)] + [InlineData(FunctionVisibility.Internal)] + public void SchedulingPreservesVisibility(FunctionVisibility visibility) + { + var module = new RawModuleDefV10(); + var reducer = new RawReducerDefV10( "run_job", [], visibility, @@ -26,21 +40,21 @@ public void SchedulingPreservesDeclaredVisibility(FunctionVisibilityV11? visibil new RawScheduleDefV10(null, "jobs", 0, "run_job") ); var raw = module.BuildModuleDefinition(); - var reducers = Assert.Single(raw.Sections.OfType()); - Assert.Equal(visibility, Assert.Single(reducers.Reducers_).DeclaredVisibility); + var reducers = Assert.Single(raw.Sections.OfType()); + Assert.Equal(visibility, Assert.Single(reducers.Reducers_).Visibility); var capabilities = Assert.Single( - raw.Sections.OfType() + raw.Sections.OfType() ); Assert.Contains("hosted_auth_v1", capabilities.Capabilities_); } [Theory] - [InlineData(FunctionVisibilityV11.ClientCallable)] - [InlineData(FunctionVisibilityV11.Private)] - public void LifecycleRejectsExternalVisibility(FunctionVisibilityV11 visibility) + [InlineData(FunctionVisibility.ClientCallable)] + [InlineData(FunctionVisibility.ExplicitClientCallable)] + public void LifecycleRejectsExternalVisibility(FunctionVisibility visibility) { - var module = new RawModuleDefV11(); - var reducer = new RawReducerDefV11( + var module = new RawModuleDefV10(); + var reducer = new RawReducerDefV10( "initialize", [], visibility, diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs index 2f9772dd591..29adc856f78 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibility.g.cs @@ -12,5 +12,7 @@ public enum FunctionVisibility { Private, ClientCallable, + Internal, + ExplicitClientCallable, } } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibilityV11.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibilityV11.g.cs deleted file mode 100644 index fcfe2b32368..00000000000 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/FunctionVisibilityV11.g.cs +++ /dev/null @@ -1,17 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -#nullable enable - -using System; - -namespace SpacetimeDB.Internal -{ - [SpacetimeDB.Type] - public enum FunctionVisibilityV11 - { - Private, - ClientCallable, - Internal, - } -} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDef.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDef.g.cs index 8995d4d3b86..d6d5d9f52a4 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDef.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDef.g.cs @@ -11,7 +11,6 @@ namespace SpacetimeDB.Internal public partial record RawModuleDef : SpacetimeDB.TaggedEnum<( RawModuleDefV8 V8BackCompat, RawModuleDefV9 V9, - RawModuleDefV10 V10, - RawModuleDefV11 V11 + RawModuleDefV10 V10 )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 6706fb278f0..98301786227 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -21,6 +21,7 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( SpacetimeDB.CaseConversionPolicy CaseConversionPolicy, ExplicitNames ExplicitNames, System.Collections.Generic.List HttpHandlers, - System.Collections.Generic.List HttpRoutes + System.Collections.Generic.List HttpRoutes, + System.Collections.Generic.List Capabilities )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11.g.cs deleted file mode 100644 index db3c58f7bb8..00000000000 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11.g.cs +++ /dev/null @@ -1,29 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Runtime.Serialization; - -namespace SpacetimeDB.Internal -{ - [SpacetimeDB.Type] - [DataContract] - public sealed partial class RawModuleDefV11 - { - [DataMember(Name = "sections")] - public System.Collections.Generic.List Sections; - - public RawModuleDefV11(System.Collections.Generic.List Sections) - { - this.Sections = Sections; - } - - public RawModuleDefV11() - { - this.Sections = new(); - } - } -} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11Section.g.cs deleted file mode 100644 index 7b5d51e2d98..00000000000 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV11Section.g.cs +++ /dev/null @@ -1,27 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -#nullable enable - -using System; - -namespace SpacetimeDB.Internal -{ - [SpacetimeDB.Type] - public partial record RawModuleDefV11Section : SpacetimeDB.TaggedEnum<( - Typespace Typespace, - System.Collections.Generic.List Types, - System.Collections.Generic.List Tables, - System.Collections.Generic.List Reducers, - System.Collections.Generic.List Procedures, - System.Collections.Generic.List Views, - System.Collections.Generic.List Schedules, - System.Collections.Generic.List LifeCycleReducers, - System.Collections.Generic.List RowLevelSecurity, - SpacetimeDB.CaseConversionPolicy CaseConversionPolicy, - ExplicitNames ExplicitNames, - System.Collections.Generic.List HttpHandlers, - System.Collections.Generic.List HttpRoutes, - System.Collections.Generic.List Capabilities - )>; -} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawProcedureDefV11.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawProcedureDefV11.g.cs deleted file mode 100644 index ca4de3f0c5d..00000000000 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawProcedureDefV11.g.cs +++ /dev/null @@ -1,45 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Runtime.Serialization; - -namespace SpacetimeDB.Internal -{ - [SpacetimeDB.Type] - [DataContract] - public sealed partial class RawProcedureDefV11 - { - [DataMember(Name = "source_name")] - public string SourceName; - [DataMember(Name = "params")] - public List Params; - [DataMember(Name = "declared_visibility")] - public FunctionVisibilityV11? DeclaredVisibility; - [DataMember(Name = "return_type")] - public SpacetimeDB.BSATN.AlgebraicType ReturnType; - - public RawProcedureDefV11( - string SourceName, - List Params, - FunctionVisibilityV11? DeclaredVisibility, - SpacetimeDB.BSATN.AlgebraicType ReturnType - ) - { - this.SourceName = SourceName; - this.Params = Params; - this.DeclaredVisibility = DeclaredVisibility; - this.ReturnType = ReturnType; - } - - public RawProcedureDefV11() - { - this.SourceName = ""; - this.Params = new(); - this.ReturnType = null!; - } - } -} diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawReducerDefV11.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawReducerDefV11.g.cs deleted file mode 100644 index 86990c74376..00000000000 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawReducerDefV11.g.cs +++ /dev/null @@ -1,50 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE -// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Runtime.Serialization; - -namespace SpacetimeDB.Internal -{ - [SpacetimeDB.Type] - [DataContract] - public sealed partial class RawReducerDefV11 - { - [DataMember(Name = "source_name")] - public string SourceName; - [DataMember(Name = "params")] - public List Params; - [DataMember(Name = "declared_visibility")] - public FunctionVisibilityV11? DeclaredVisibility; - [DataMember(Name = "ok_return_type")] - public SpacetimeDB.BSATN.AlgebraicType OkReturnType; - [DataMember(Name = "err_return_type")] - public SpacetimeDB.BSATN.AlgebraicType ErrReturnType; - - public RawReducerDefV11( - string SourceName, - List Params, - FunctionVisibilityV11? DeclaredVisibility, - SpacetimeDB.BSATN.AlgebraicType OkReturnType, - SpacetimeDB.BSATN.AlgebraicType ErrReturnType - ) - { - this.SourceName = SourceName; - this.Params = Params; - this.DeclaredVisibility = DeclaredVisibility; - this.OkReturnType = OkReturnType; - this.ErrReturnType = ErrReturnType; - } - - public RawReducerDefV11() - { - this.SourceName = ""; - this.Params = new(); - this.OkReturnType = null!; - this.ErrReturnType = null!; - } - } -} diff --git a/crates/bindings-csharp/Runtime/Internal/IReducer.cs b/crates/bindings-csharp/Runtime/Internal/IReducer.cs index af068b38253..878c98a2a2e 100644 --- a/crates/bindings-csharp/Runtime/Internal/IReducer.cs +++ b/crates/bindings-csharp/Runtime/Internal/IReducer.cs @@ -18,7 +18,7 @@ public static Identity GetDatabaseIdentity() public interface IReducer { - RawReducerDefV11 MakeReducerDef(ITypeRegistrar registrar); + RawReducerDefV10 MakeReducerDef(ITypeRegistrar registrar); Lifecycle? Lifecycle { get; } diff --git a/crates/bindings-csharp/Runtime/Internal/Module.cs b/crates/bindings-csharp/Runtime/Internal/Module.cs index 1b72234b70e..e89de0c0172 100644 --- a/crates/bindings-csharp/Runtime/Internal/Module.cs +++ b/crates/bindings-csharp/Runtime/Internal/Module.cs @@ -7,15 +7,15 @@ namespace SpacetimeDB.Internal; using SpacetimeDB; using SpacetimeDB.BSATN; -partial class RawModuleDefV11 +partial class RawModuleDefV10 { private readonly Typespace typespace = new(); private readonly List typeDefs = []; private readonly List tableDefs = []; private readonly List scheduleDefs = []; - private readonly List reducerDefs = []; + private readonly List reducerDefs = []; private readonly List lifecycleReducerDefs = []; - private readonly List procedureDefs = []; + private readonly List procedureDefs = []; private readonly List httpHandlerDefs = []; private readonly List httpRouteDefs = []; private readonly List viewDefs = []; @@ -53,11 +53,13 @@ internal AlgebraicType.Ref RegisterType(Func procedureDefs.Add(procedure); + internal void RegisterProcedure(RawProcedureDefV10 procedure) => procedureDefs.Add(procedure); internal void RegisterHttpHandler(RawHttpHandlerDefV10 handler) => httpHandlerDefs.Add(handler); @@ -117,7 +119,7 @@ internal void RegisterExplicitFunctionName(string sourceName, string canonicalNa internal void RegisterExplicitIndexName(string sourceName, string canonicalName) => explicitNames.Add(new ExplicitNameEntry.Index(new NameMapping(sourceName, canonicalName))); - internal RawModuleDefV11 BuildModuleDefinition() + internal RawModuleDefV10 BuildModuleDefinition() { var builtTables = new List(tableDefs.Count); foreach (var table in tableDefs) @@ -141,64 +143,64 @@ internal RawModuleDefV11 BuildModuleDefinition() ); } - var sections = new List + var sections = new List { - new RawModuleDefV11Section.Typespace(typespace), - new RawModuleDefV11Section.Capabilities(["hosted_auth_v1"]), + new RawModuleDefV10Section.Typespace(typespace), + new RawModuleDefV10Section.Capabilities(["hosted_auth_v1"]), }; if (typeDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.Types(typeDefs)); + sections.Add(new RawModuleDefV10Section.Types(typeDefs)); } if (builtTables.Count > 0) { - sections.Add(new RawModuleDefV11Section.Tables(builtTables)); + sections.Add(new RawModuleDefV10Section.Tables(builtTables)); } if (reducerDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.Reducers(reducerDefs)); + sections.Add(new RawModuleDefV10Section.Reducers(reducerDefs)); } if (procedureDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.Procedures(procedureDefs)); + sections.Add(new RawModuleDefV10Section.Procedures(procedureDefs)); } if (httpHandlerDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.HttpHandlers(httpHandlerDefs)); + sections.Add(new RawModuleDefV10Section.HttpHandlers(httpHandlerDefs)); } if (httpRouteDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.HttpRoutes(httpRouteDefs)); + sections.Add(new RawModuleDefV10Section.HttpRoutes(httpRouteDefs)); } if (viewDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.Views(viewDefs)); + sections.Add(new RawModuleDefV10Section.Views(viewDefs)); } if (scheduleDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.Schedules(scheduleDefs)); + sections.Add(new RawModuleDefV10Section.Schedules(scheduleDefs)); } if (lifecycleReducerDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.LifeCycleReducers(lifecycleReducerDefs)); + sections.Add(new RawModuleDefV10Section.LifeCycleReducers(lifecycleReducerDefs)); } // TODO: Add sections for Event tables and Case conversion policy (mirrors Rust `raw_def/v10.rs` TODO). if (caseConversionPolicy is { } policy) { - sections.Add(new RawModuleDefV11Section.CaseConversionPolicy(policy)); + sections.Add(new RawModuleDefV10Section.CaseConversionPolicy(policy)); } if (explicitNames.Count > 0) { sections.Add( - new RawModuleDefV11Section.ExplicitNames( + new RawModuleDefV10Section.ExplicitNames( new ExplicitNames(new List(explicitNames)) ) ); } if (rowLevelSecurityDefs.Count > 0) { - sections.Add(new RawModuleDefV11Section.RowLevelSecurity(rowLevelSecurityDefs)); + sections.Add(new RawModuleDefV10Section.RowLevelSecurity(rowLevelSecurityDefs)); } Sections = sections; @@ -228,8 +230,8 @@ private static void EnsureNativeAotTypeRoots() { _ = new RawIndexAlgorithm.BTree(null!); _ = new RawConstraintDataV9.Unique(null!); - _ = new RawModuleDef.V11(null!); - _ = new RawModuleDefV11Section.Typespace(null!); + _ = new RawModuleDef.V10(null!); + _ = new RawModuleDefV10Section.Typespace(null!); _ = new ExplicitNameEntry.Table(null!); _ = new MiscModuleExport.TypeAlias(null!); _ = new RawMiscModuleExportV9.ColumnDefaultValue(null!); @@ -238,7 +240,7 @@ private static void EnsureNativeAotTypeRoots() } } - private static readonly RawModuleDefV11 moduleDef = new(); + private static readonly RawModuleDefV10 moduleDef = new(); private static readonly List reducers = []; private static readonly List procedures = []; @@ -496,7 +498,7 @@ public static void __describe_module__(BytesSink description) try { var module = moduleDef.BuildModuleDefinition(); - RawModuleDef versioned = new RawModuleDef.V11(module); + RawModuleDef versioned = new RawModuleDef.V10(module); var moduleBytes = IStructuralReadWrite.ToBytes(new RawModuleDef.BSATN(), versioned); description.Write(moduleBytes); } diff --git a/crates/bindings-csharp/Runtime/Internal/Procedure.cs b/crates/bindings-csharp/Runtime/Internal/Procedure.cs index 010c8d9e192..81d2be9a749 100644 --- a/crates/bindings-csharp/Runtime/Internal/Procedure.cs +++ b/crates/bindings-csharp/Runtime/Internal/Procedure.cs @@ -13,7 +13,7 @@ public interface IProcedure /// /// Creates a procedure definition for registration with the module system. /// - RawProcedureDefV11 MakeProcedureDef(ITypeRegistrar registrar); + RawProcedureDefV10 MakeProcedureDef(ITypeRegistrar registrar); /// /// Invokes the procedure with the given arguments and context. diff --git a/crates/bindings-typescript/README.md b/crates/bindings-typescript/README.md index d4f2d609599..9a37a85f582 100644 --- a/crates/bindings-typescript/README.md +++ b/crates/bindings-typescript/README.md @@ -32,8 +32,9 @@ admit the owner, and public functions admit any client. `ctx.senderAuth.isIntern captures the host's invocation authority independently of connection and JWT presence, so an internal call can have a JWT. `ctx.senderAuth.jwt.identity` is the verified sender supplied by the host. Procedure transactions preserve this -authentication. Newly compiled modules emit schema V11 and advertise -`hosted_auth_v1`, requiring a compatible host. +authentication. Newly compiled modules retain schema V10 and advertise +`hosted_auth_v1`. The extended visibility values and capability section require +a compatible host; older V10 definitions retain their existing defaults. #### Client SDK diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index f8a25bf55bf..f4bea6d4a72 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -76,16 +76,10 @@ export type ExplicitNames = __Infer; export const FunctionVisibility = __t.enum('FunctionVisibility', { Private: __t.unit(), ClientCallable: __t.unit(), -}); -export type FunctionVisibility = __Infer; - -// The tagged union or sum type for the algebraic type `FunctionVisibilityV11`. -export const FunctionVisibilityV11 = __t.enum('FunctionVisibilityV11', { - Private: __t.unit(), - ClientCallable: __t.unit(), Internal: __t.unit(), + ExplicitClientCallable: __t.unit(), }); -export type FunctionVisibilityV11 = __Infer; +export type FunctionVisibility = __Infer; export const HttpHeaderPair = __t.object('HttpHeaderPair', { name: __t.string(), @@ -344,9 +338,6 @@ export const RawModuleDef = __t.enum('RawModuleDef', { get V10() { return RawModuleDefV10; }, - get V11() { - return RawModuleDefV11; - }, }); export type RawModuleDef = __Infer; @@ -398,60 +389,9 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { get HttpRoutes() { return __t.array(RawHttpRouteDefV10); }, -}); -export type RawModuleDefV10Section = __Infer; - -export const RawModuleDefV11 = __t.object('RawModuleDefV11', { - get sections() { - return __t.array(RawModuleDefV11Section); - }, -}); -export type RawModuleDefV11 = __Infer; - -// The tagged union or sum type for the algebraic type `RawModuleDefV11Section`. -export const RawModuleDefV11Section = __t.enum('RawModuleDefV11Section', { - get Typespace() { - return Typespace; - }, - get Types() { - return __t.array(RawTypeDefV10); - }, - get Tables() { - return __t.array(RawTableDefV10); - }, - get Reducers() { - return __t.array(RawReducerDefV11); - }, - get Procedures() { - return __t.array(RawProcedureDefV11); - }, - get Views() { - return __t.array(RawViewDefV10); - }, - get Schedules() { - return __t.array(RawScheduleDefV10); - }, - get LifeCycleReducers() { - return __t.array(RawLifeCycleReducerDefV10); - }, - get RowLevelSecurity() { - return __t.array(RawRowLevelSecurityDefV9); - }, - get CaseConversionPolicy() { - return CaseConversionPolicy; - }, - get ExplicitNames() { - return ExplicitNames; - }, - get HttpHandlers() { - return __t.array(RawHttpHandlerDefV10); - }, - get HttpRoutes() { - return __t.array(RawHttpRouteDefV10); - }, Capabilities: __t.array(__t.string()), }); -export type RawModuleDefV11Section = __Infer; +export type RawModuleDefV10Section = __Infer; export const RawModuleDefV8 = __t.object('RawModuleDefV8', { get typespace() { @@ -505,20 +445,6 @@ export const RawProcedureDefV10 = __t.object('RawProcedureDefV10', { }); export type RawProcedureDefV10 = __Infer; -export const RawProcedureDefV11 = __t.object('RawProcedureDefV11', { - sourceName: __t.string(), - get params() { - return ProductType; - }, - get declaredVisibility() { - return __t.option(FunctionVisibilityV11); - }, - get returnType() { - return AlgebraicType; - }, -}); -export type RawProcedureDefV11 = __Infer; - export const RawProcedureDefV9 = __t.object('RawProcedureDefV9', { name: __t.string(), get params() { @@ -547,23 +473,6 @@ export const RawReducerDefV10 = __t.object('RawReducerDefV10', { }); export type RawReducerDefV10 = __Infer; -export const RawReducerDefV11 = __t.object('RawReducerDefV11', { - sourceName: __t.string(), - get params() { - return ProductType; - }, - get declaredVisibility() { - return __t.option(FunctionVisibilityV11); - }, - get okReturnType() { - return AlgebraicType; - }, - get errReturnType() { - return AlgebraicType; - }, -}); -export type RawReducerDefV11 = __Infer; - export const RawReducerDefV9 = __t.object('RawReducerDefV9', { name: __t.string(), get params() { diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index 229fc25b1ad..f5526b271d5 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -7,8 +7,8 @@ import { } from './algebraic_type'; import type { CaseConversionPolicy, - RawModuleDefV11, - RawModuleDefV11Section, + RawModuleDefV10, + RawModuleDefV10Section, RawScopedTypeNameV10, RawTableDefV10, } from './autogen/types'; @@ -174,10 +174,10 @@ type CompoundTypeCache = Map< >; export type ModuleDef = { - [S in RawModuleDefV11Section as Uncapitalize]: S['value']; + [S in RawModuleDefV10Section as Uncapitalize]: S['value']; }; -type Section = RawModuleDefV11Section; +type Section = RawModuleDefV10Section; export class ModuleContext { #compoundTypes: CompoundTypeCache = new Map(); @@ -208,7 +208,7 @@ export class ModuleContext { return this.#moduleDef; } - rawModuleDefV11(): RawModuleDefV11 { + rawModuleDefV10(): RawModuleDefV10 { const sections: Section[] = []; const push = (s: T | undefined) => { diff --git a/crates/bindings-typescript/src/server/function_visibility.ts b/crates/bindings-typescript/src/server/function_visibility.ts index d12d9692e74..658fb1dad0d 100644 --- a/crates/bindings-typescript/src/server/function_visibility.ts +++ b/crates/bindings-typescript/src/server/function_visibility.ts @@ -1,21 +1,23 @@ -import { FunctionVisibilityV11 } from '../lib/autogen/types'; +import { FunctionVisibility as RawFunctionVisibility } from '../lib/autogen/types'; /** Internal functions require verified internal authority. Private functions also * admit the owner. Public functions admit any authenticated client. */ export type FunctionVisibility = 'public' | 'private' | 'internal'; -export function declaredVisibility( +export function rawVisibility( visibility: FunctionVisibility | undefined -): FunctionVisibilityV11 | undefined { +): RawFunctionVisibility { switch (visibility) { case undefined: - return undefined; + // Preserve V10's existing context-dependent default, including scheduled + // private functions, without changing the raw definition's field layout. + return RawFunctionVisibility.ClientCallable; case 'public': - return FunctionVisibilityV11.ClientCallable; + return RawFunctionVisibility.ExplicitClientCallable; case 'private': - return FunctionVisibilityV11.Private; + return RawFunctionVisibility.Private; case 'internal': - return FunctionVisibilityV11.Internal; + return RawFunctionVisibility.Internal; default: throw new TypeError('Invalid function visibility'); } diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index 0b056b8d660..ea932e6efcd 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -5,10 +5,7 @@ import { type Deserializer, type Serializer, } from '../lib/algebraic_type'; -import { - declaredVisibility, - type FunctionVisibility, -} from './function_visibility'; +import { rawVisibility, type FunctionVisibility } from './function_visibility'; import BinaryReader from '../lib/binary_reader'; import BinaryWriter from '../lib/binary_writer'; import type { ConnectionId } from '../lib/connection_id'; @@ -130,7 +127,7 @@ function registerProcedure< sourceName: exportName, params: paramsType, returnType, - declaredVisibility: declaredVisibility(opts?.visibility), + visibility: rawVisibility(opts?.visibility), }); if (opts?.name != null) { diff --git a/crates/bindings-typescript/src/server/reducers.ts b/crates/bindings-typescript/src/server/reducers.ts index a43f1821955..05001d9c476 100644 --- a/crates/bindings-typescript/src/server/reducers.ts +++ b/crates/bindings-typescript/src/server/reducers.ts @@ -1,9 +1,6 @@ import { AlgebraicType } from '../lib/algebraic_type'; import { type Lifecycle } from '../lib/autogen/types'; -import { - declaredVisibility, - type FunctionVisibility, -} from './function_visibility'; +import { rawVisibility, type FunctionVisibility } from './function_visibility'; import type { ParamsObj, Reducer } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; import { RowBuilder, type RowObj } from '../lib/type_builders'; @@ -90,8 +87,8 @@ export function registerReducer( ctx.moduleDef.reducers.push({ sourceName: exportName, params: paramsType, - // Preserve omission so the host can apply the scheduled private default. - declaredVisibility: declaredVisibility(opts?.visibility), + // Keep the legacy default distinct from an explicit public declaration. + visibility: rawVisibility(opts?.visibility), //Hardcoded for now - reducers do not return values yet okReturnType: AlgebraicType.Product({ elements: [] }), errReturnType: AlgebraicType.String, diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index bbf17b81c9d..e4b1a28cf5f 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -383,7 +383,7 @@ class ModuleHooksImpl implements ModuleHooks { const writer = new BinaryWriter(128); RawModuleDef.serialize( writer, - RawModuleDef.V11(this.#schema.rawModuleDefV11()) + RawModuleDef.V10(this.#schema.rawModuleDefV10()) ); return writer.getBuffer(); } diff --git a/crates/bindings-typescript/tests/hosted_auth.test.ts b/crates/bindings-typescript/tests/hosted_auth.test.ts index a2c2a3b6774..c3aa52e3b53 100644 --- a/crates/bindings-typescript/tests/hosted_auth.test.ts +++ b/crates/bindings-typescript/tests/hosted_auth.test.ts @@ -34,7 +34,14 @@ import { Timestamp } from '../src/lib/timestamp'; import { schema, exportContext, registerExport } from '../src/server/schema'; import { callProcedure } from '../src/server/procedures'; import { t } from '../src/lib/type_builders'; -import { RawModuleDef } from '../src/lib/autogen/types'; +import { + AlgebraicType, + FunctionVisibility, + ProductType, + RawModuleDef, + RawModuleDefV10Section, + RawReducerDefV10, +} from '../src/lib/autogen/types'; import BinaryReader from '../src/lib/binary_reader'; import BinaryWriter from '../src/lib/binary_writer'; @@ -136,7 +143,52 @@ describe('verified invocation authentication', () => { }); }); -describe('V11 explicit function visibility', () => { +describe('V10 explicit function visibility', () => { + it('preserves existing visibility tags and appends the new variants and capability section', () => { + const legacyVisibility = t.enum('LegacyFunctionVisibility', { + Private: t.unit(), + ClientCallable: t.unit(), + }); + const variants = [ + FunctionVisibility.Private, + FunctionVisibility.ClientCallable, + FunctionVisibility.Internal, + FunctionVisibility.ExplicitClientCallable, + ]; + for (const [tag, visibility] of variants.entries()) { + const writer = new BinaryWriter(8); + FunctionVisibility.serialize(writer, visibility); + expect([...writer.getBuffer()]).toEqual([tag]); + const reader = new BinaryReader(writer.getBuffer()); + if (tag < 2) { + expect(legacyVisibility.deserialize(reader).tag).toBe(visibility.tag); + } + } + const writer = new BinaryWriter(8); + RawModuleDefV10Section.serialize(writer, { + tag: 'Capabilities', + value: [], + }); + expect([...writer.getBuffer()]).toEqual([13, 0, 0, 0, 0]); + }); + + it('retains the V10 reducer field layout without an optional visibility wrapper', () => { + const module = schema({}); + const reducer = module.reducer({ visibility: 'public' }, () => {}); + const inner = reducer[exportContext]!; + reducer[registerExport](inner, 'public_reducer'); + const definition = inner.moduleDef.reducers[0]; + const writer = new BinaryWriter(128); + RawReducerDefV10.serialize(writer, definition); + const expected = new BinaryWriter(128); + expected.writeString(definition.sourceName); + ProductType.serialize(expected, definition.params); + expected.writeByte(3); + AlgebraicType.serialize(expected, definition.okReturnType); + AlgebraicType.serialize(expected, definition.errReturnType); + expect(writer.getBuffer()).toEqual(expected.getBuffer()); + }); + it('serializes omission separately from explicit visibility and advertises hosted auth', () => { const module = schema({}); const omitted = module.reducer(() => {}); @@ -157,33 +209,48 @@ describe('V11 explicit function visibility', () => { } // Being scheduled must not erase a public choice or manufacture an explicit // choice for the default. The host resolves the latter to Private. - inner.moduleDef.schedules.push({ - sourceName: undefined, - tableName: 'jobs', - scheduleAtCol: 0, - functionName: 'explicitlyPublic', - }); - const raw = RawModuleDef.V11(inner.rawModuleDefV11()); + for (const name of [ + 'omitted', + 'explicitlyPublic', + 'privateReducer', + 'internalReducer', + ]) { + inner.moduleDef.schedules.push({ + sourceName: undefined, + tableName: `jobs_${name}`, + scheduleAtCol: 0, + functionName: name, + }); + } + const raw = RawModuleDef.V10(inner.rawModuleDefV10()); const writer = new BinaryWriter(128); RawModuleDef.serialize(writer, raw); - expect(writer.getBuffer()[0]).toBe(3); + expect(writer.getBuffer()[0]).toBe(2); const decoded = RawModuleDef.deserialize( new BinaryReader(writer.getBuffer()) ); const roundTrip = new BinaryWriter(128); RawModuleDef.serialize(roundTrip, decoded); expect(roundTrip.getBuffer()).toEqual(writer.getBuffer()); - expect(decoded.tag).toBe('V11'); - if (decoded.tag !== 'V11') throw new Error('Expected V11'); + expect(decoded.tag).toBe('V10'); + if (decoded.tag !== 'V10') throw new Error('Expected V10'); const reducers = decoded.value.sections.find( section => section.tag === 'Reducers' ); + expect(reducers?.value.map(reducer => reducer.visibility.tag)).toEqual([ + 'ClientCallable', + 'ExplicitClientCallable', + 'Private', + 'Internal', + ]); expect( - reducers?.value.map(reducer => reducer.declaredVisibility?.tag) - ).toEqual([undefined, 'ClientCallable', 'Private', 'Internal']); - expect( - inner.moduleDef.reducers.map(reducer => reducer.declaredVisibility?.tag) - ).toEqual([undefined, 'ClientCallable', 'Private', 'Internal']); + inner.moduleDef.reducers.map(reducer => reducer.visibility.tag) + ).toEqual([ + 'ClientCallable', + 'ExplicitClientCallable', + 'Private', + 'Internal', + ]); expect(inner.moduleDef.capabilities).toEqual(['hosted_auth_v1']); }); @@ -199,9 +266,7 @@ describe('V11 explicit function visibility', () => { proc[registerExport](inner, 'source_name'); reducer[registerExport](inner, 'accept_visibility'); expect(inner.moduleDef.procedures[0].sourceName).toBe('source_name'); - expect(inner.moduleDef.procedures[0].declaredVisibility?.tag).toBe( - 'Internal' - ); + expect(inner.moduleDef.procedures[0].visibility.tag).toBe('Internal'); expect(inner.moduleDef.explicitNames.entries).toContainEqual({ tag: 'Function', value: { sourceName: 'source_name', canonicalName: 'public_name' }, @@ -211,16 +276,30 @@ describe('V11 explicit function visibility', () => { ); }); - it('rejects externally callable lifecycle declarations', () => { - const module = schema({}); - const invalid = module.init({ visibility: 'private' }, () => {}); - expect(() => - invalid[registerExport](invalid[exportContext]!, 'invalid_init') - ).toThrow('Lifecycle reducers only support internal visibility'); - const valid = module.init({ visibility: 'internal' }, () => {}); - valid[registerExport](valid[exportContext]!, 'valid_init'); - expect( - valid[exportContext]!.moduleDef.reducers[0].declaredVisibility?.tag - ).toBe('Internal'); - }); + it.each(['private', 'public'] as const)( + 'rejects explicit %s lifecycle declarations', + visibility => { + const module = schema({}); + const invalid = module.init({ visibility }, () => {}); + expect(() => + invalid[registerExport](invalid[exportContext]!, 'invalid_init') + ).toThrow('Lifecycle reducers only support internal visibility'); + } + ); + + it.each([undefined, 'internal'] as const)( + 'preserves permitted lifecycle declaration %s for host event dispatch', + visibility => { + const module = schema({}); + const valid = module.init({ visibility }, () => {}); + valid[registerExport](valid[exportContext]!, 'valid_init'); + const inner = valid[exportContext]!; + expect(inner.moduleDef.reducers[0].visibility.tag).toBe( + visibility === undefined ? 'ClientCallable' : 'Internal' + ); + expect(inner.moduleDef.lifeCycleReducers).toEqual([ + { lifecycleSpec: { tag: 'Init' }, functionName: 'valid_init' }, + ]); + } + ); }); diff --git a/crates/bindings/src/rt.rs b/crates/bindings/src/rt.rs index e73c1340a7e..93ac08983eb 100644 --- a/crates/bindings/src/rt.rs +++ b/crates/bindings/src/rt.rs @@ -1,13 +1,14 @@ #![deny(unsafe_op_in_unsafe_fn)] -pub use spacetimedb_lib::db::raw_def::v11::FunctionVisibility; -use spacetimedb_lib::db::raw_def::v11::RawModuleDefV11Builder; +pub use spacetimedb_lib::db::raw_def::v10::FunctionVisibility; use crate::query_builder::{FromWhere, HasCols, LeftSemiJoin, RawQuery, RightSemiJoin, Table as QbTable}; use crate::table::IndexAlgo; use crate::{sys, AnonymousViewContext, IterBuf, ReducerContext, ReducerResult, SpacetimeType, Table, ViewContext}; use spacetimedb_lib::bsatn::EncodeError; -use spacetimedb_lib::db::raw_def::v10::{CaseConversionPolicy, ExplicitNames as RawExplicitNames}; +use spacetimedb_lib::db::raw_def::v10::{ + CaseConversionPolicy, ExplicitNames as RawExplicitNames, RawModuleDefV10Builder, +}; pub use spacetimedb_lib::db::raw_def::v9::Lifecycle as LifecycleReducer; use spacetimedb_lib::db::raw_def::v9::{RawIndexAlgorithm, TableType, ViewResultHeader}; use spacetimedb_lib::de::{self, Deserialize, DeserializeOwned, Error as _, SeqProductAccess}; @@ -941,7 +942,7 @@ pub fn register_case_conversion_policy(policy: CaseConversionPolicy) { #[derive(Default)] pub struct ModuleBuilder { /// The module definition. - inner: RawModuleDefV11Builder, + inner: RawModuleDefV10Builder, /// The reducers of the module. reducers: Vec, /// The procedures of the module. @@ -1010,7 +1011,7 @@ extern "C" fn __describe_module__(description: BytesSink) { // Serialize the module to bsatn. let module_def = module.inner.finish(); - let module_def = RawModuleDef::V11(module_def); + let module_def = RawModuleDef::V10(module_def); let bytes = bsatn::to_vec(&module_def).expect("unable to serialize typespace"); // Write the sets of reducers, procedures and views. diff --git a/crates/bindings/tests/pass/function_visibility.rs b/crates/bindings/tests/pass/function_visibility.rs index 6fe9cfa03f3..7f53af5717f 100644 --- a/crates/bindings/tests/pass/function_visibility.rs +++ b/crates/bindings/tests/pass/function_visibility.rs @@ -31,26 +31,32 @@ fn public_procedure(_ctx: &mut ProcedureContext) -> u64 { } fn main() { - assert_eq!( + assert!(matches!( internal_reducer::DECLARED_VISIBILITY, Some(FunctionVisibility::Internal) - ); - assert_eq!(private_reducer::DECLARED_VISIBILITY, Some(FunctionVisibility::Private)); - assert_eq!( + )); + assert!(matches!( + private_reducer::DECLARED_VISIBILITY, + Some(FunctionVisibility::Private) + )); + assert!(matches!( public_reducer::DECLARED_VISIBILITY, Some(FunctionVisibility::ClientCallable) - ); - assert_eq!(initialize::DECLARED_VISIBILITY, Some(FunctionVisibility::Internal)); - assert_eq!( + )); + assert!(matches!( + initialize::DECLARED_VISIBILITY, + Some(FunctionVisibility::Internal) + )); + assert!(matches!( internal_procedure::DECLARED_VISIBILITY, Some(FunctionVisibility::Internal) - ); - assert_eq!( + )); + assert!(matches!( private_procedure::DECLARED_VISIBILITY, Some(FunctionVisibility::Private) - ); - assert_eq!( + )); + assert!(matches!( public_procedure::DECLARED_VISIBILITY, Some(FunctionVisibility::ClientCallable) - ); + )); } diff --git a/crates/bindings/tests/ui/tables.stderr b/crates/bindings/tests/ui/tables.stderr index e6f65b174bb..18b61f49224 100644 --- a/crates/bindings/tests/ui/tables.stderr +++ b/crates/bindings/tests/ui/tables.stderr @@ -210,7 +210,7 @@ error[E0277]: `&'a Alpha` cannot appear as an argument to an index filtering ope = help: the following other types implement trait `FilterableValue`: &ConnectionId &ContainerMode - &FunctionVisibilityV11 + &FunctionVisibility &Identity &Lifecycle &PortExposure @@ -242,7 +242,7 @@ help: the trait `FilterableValue` is not implemented for `Alpha` = help: the following other types implement trait `FilterableValue`: &ConnectionId &ContainerMode - &FunctionVisibilityV11 + &FunctionVisibility &Identity &Lifecycle &PortExposure diff --git a/crates/cli/src/api.rs b/crates/cli/src/api.rs index 53fdc28a39a..a608d220335 100644 --- a/crates/cli/src/api.rs +++ b/crates/cli/src/api.rs @@ -4,7 +4,7 @@ use std::ops::Add; use reqwest::{header, Client, RequestBuilder}; use serde::Deserialize; -use spacetimedb_lib::db::raw_def::v11::RawModuleDefV11; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::de::serde::DeserializeWrapper; use spacetimedb_lib::Identity; @@ -61,11 +61,11 @@ impl ClientApi { } /// Reads the `ModuleDef` from the `schema` endpoint. - pub async fn module_def(&self) -> anyhow::Result { + pub async fn module_def(&self) -> anyhow::Result { let res = self .client .get(self.con.db_uri("schema")) - .query(&[("version", "11")]) + .query(&[("version", "10")]) .send() .await?; let DeserializeWrapper(module_def) = res.json_or_error().await?; diff --git a/crates/cli/src/subcommands/describe.rs b/crates/cli/src/subcommands/describe.rs index 735968a818d..da2046bc19d 100644 --- a/crates/cli/src/subcommands/describe.rs +++ b/crates/cli/src/subcommands/describe.rs @@ -112,6 +112,8 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error .accessor_name; let reducer = module_def .reducers() + .into_iter() + .flatten() .find(|r| *r.source_name == **source_name) .context("no such reducer")?; sats_to_json(reducer)? @@ -120,6 +122,8 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error let source_name = &canonical.table(table_name).context("no such table")?.accessor_name; let table = module_def .tables() + .into_iter() + .flatten() .find(|t| *t.source_name == **source_name) .context("no such table")?; sats_to_json(table)? diff --git a/crates/client-api/src/routes/database.rs b/crates/client-api/src/routes/database.rs index e57830bbbc0..1d6ce136dd7 100644 --- a/crates/client-api/src/routes/database.rs +++ b/crates/client-api/src/routes/database.rs @@ -42,7 +42,6 @@ use spacetimedb_client_api_messages::name::{ PrePublishResult, PrettyPrintStyle, PublishOp, PublishResult, }; use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; -use spacetimedb_lib::db::raw_def::v11::RawModuleDefV11; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::http as st_http; use spacetimedb_lib::{sats, AlgebraicValue, Hash, ProductValue, Timestamp}; @@ -530,8 +529,6 @@ enum SchemaVersion { V9, #[serde(rename = "10")] V10, - #[serde(rename = "11")] - V11, } pub async fn schema( @@ -560,12 +557,7 @@ where axum::Json(sats::serde::SerdeWrapper(raw)).into_response() } SchemaVersion::V10 => { - let raw = RawModuleDefV10::try_from(module_def.as_ref().clone()) - .map_err(|err| bad_request(err.to_string().into()))?; - axum::Json(sats::serde::SerdeWrapper(raw)).into_response() - } - SchemaVersion::V11 => { - let raw = RawModuleDefV11::from(module_def.as_ref().clone()); + let raw = RawModuleDefV10::from(module_def.as_ref().clone()); axum::Json(sats::serde::SerdeWrapper(raw)).into_response() } }; diff --git a/crates/codegen/src/util.rs b/crates/codegen/src/util.rs index ac2c40a4d96..f8c590af07d 100644 --- a/crates/codegen/src/util.rs +++ b/crates/codegen/src/util.rs @@ -216,14 +216,14 @@ pub fn iter_types(module: &ModuleDef) -> impl Iterator { mod visibility_tests { use super::*; use spacetimedb_lib::db::raw_def::{ - v11::{FunctionVisibility, RawModuleDefV11Builder}, + v10::{FunctionVisibility, RawModuleDefV10Builder}, v9::Lifecycle, }; use spacetimedb_lib::{AlgebraicType, ProductType}; #[test] fn public_codegen_excludes_internal_private_and_every_lifecycle() { - let mut builder = RawModuleDefV11Builder::new(); + let mut builder = RawModuleDefV10Builder::new(); builder.add_reducer("ordinary", ProductType::unit()); for (name, visibility) in [ ("public_function", FunctionVisibility::ClientCallable), diff --git a/crates/core/src/auth/invocation/tests.rs b/crates/core/src/auth/invocation/tests.rs index 9453da7d03c..5d1ed1f1537 100644 --- a/crates/core/src/auth/invocation/tests.rs +++ b/crates/core/src/auth/invocation/tests.rs @@ -6,12 +6,12 @@ use crate::db::relational_db::tests_utils::TestDB; use spacetimedb_datastore::execution_context::Workload; use spacetimedb_datastore::system_tables::StContainerFenceRow; use spacetimedb_lib::db::auth::StAccess; -use spacetimedb_lib::db::raw_def::v11::RawModuleDefV11Builder; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; use spacetimedb_lib::identity::AuthCtx; use std::time::Duration; fn module(hosted_auth: bool) -> ModuleDef { - let mut builder = RawModuleDefV11Builder::new(); + let mut builder = RawModuleDefV10Builder::new(); if hosted_auth { builder.add_capability("hosted_auth_v1"); } diff --git a/crates/core/src/host/empty_module.rs b/crates/core/src/host/empty_module.rs index cfd89fbe29d..fff546cf83c 100644 --- a/crates/core/src/host/empty_module.rs +++ b/crates/core/src/host/empty_module.rs @@ -1,7 +1,7 @@ //! Versioned built-in module for a database published with only a container. //! //! This is a real Wasm program, not an empty byte string. It declares an empty -//! V11 user schema and hosted_auth_v1, so normal database initialization, system +//! V10 user schema and hosted_auth_v1, so normal database initialization, system //! tables, subscriptions, and later migration use the existing host machinery. //! Its required reducer ABI entry point traps because no reducer is declared. //! diff --git a/crates/core/src/host/empty_module/README.md b/crates/core/src/host/empty_module/README.md index 4b9b15057d7..4db85e2fdd1 100644 --- a/crates/core/src/host/empty_module/README.md +++ b/crates/core/src/host/empty_module/README.md @@ -1,6 +1,6 @@ # System empty module, version 1 -The host uses this real Wasm module for a database published with a container and no user module. It declares V11 with an empty typespace and `hosted_auth_v1`; there are no user tables, reducers, procedures, views, schedules, or HTTP handlers. +The host uses this real Wasm module for a database published with a container and no user module. It declares V10 with an empty typespace and `hosted_auth_v1`; there are no user tables, reducers, procedures, views, schedules, or HTTP handlers. `generate.py` is the complete source. It uses only the Python standard library and explicitly encodes the Wasm sections and BSATN description. It produces a 250-byte Wasm binary with one fixed memory page, a schema describer, and the required reducer ABI entry point. The reducer entry point always traps because no reducer is declared. The `spacetime_10.6::get_call_auth_flags` import requires the host ABI corresponding to the advertised capability; the empty module has no invocation contexts to construct. @@ -15,9 +15,9 @@ Omit `--check` to regenerate the three files. This does not require a Rust modul The SHA-256 checksum of `v1.wasm` is recorded in `v1.sha256`. SpacetimeDB's separate Keccak-256 program identity is: ```text -9b6cf2db3644c1d321d97ae0fcd4ab3fc065fc94f54a49f61c7a1dfa40a02612 +08365097d1ef202653615f02cc46be16088044c6b9cd9660e3a0f336e47918f0 ``` Version 1 is immutable. A change to its schema or Wasm requires a new version and new files. Keep version 1 available for existing deployments and replay. The Rust helper checks the known program hash as well as the exact bytes, kind, and version, so a different publisher-supplied module with an empty-looking schema cannot qualify as the system empty module. `Program::empty` is also rejected. -Core tests compare the BSATN fixture against the current V11 Rust wire types, load the Wasm through the actual host, and initialize a real database through `HostController`. They verify that initialization stores the bundled program and the database's metadata without invoking a user reducer. +Core tests compare the BSATN fixture against the current V10 Rust wire types, load the Wasm through the actual host, and initialize a real database through `HostController`. They verify that initialization stores the bundled program and the database's metadata without invoking a user reducer. diff --git a/crates/core/src/host/empty_module/generate.py b/crates/core/src/host/empty_module/generate.py index f7eb227023a..6400d46db9d 100644 --- a/crates/core/src/host/empty_module/generate.py +++ b/crates/core/src/host/empty_module/generate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Reproduce the version-1 system empty module using only Python's standard library. -The binary has one page of memory, a V11 schema describer, and a reducer entry +The binary has one page of memory, a V10 schema describer, and a reducer entry point which always traps. There are no declared user tables or functions. Equivalent code (the data payload below is BSATN, not WebAssembly encoding): @@ -53,10 +53,10 @@ def function_type(params, results): def generate(): u32 = lambda value: struct.pack(";t diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index d0f702301eb..d29614d94a6 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -1324,10 +1324,7 @@ impl Host { old_module.module_def.raw_module_def_version(), module_def.raw_module_def_version() ), - ( - RawModuleDefVersion::V9OrEarlier, - RawModuleDefVersion::V10 | RawModuleDefVersion::V11 - ) + (RawModuleDefVersion::V9OrEarlier, RawModuleDefVersion::V10) ); let res = match ponder_migrate(&old_module.module_def, &module_def) { diff --git a/crates/core/src/subscription/module_subscription_actor.rs b/crates/core/src/subscription/module_subscription_actor.rs index 1603a04c77b..ac1c34fc2f4 100644 --- a/crates/core/src/subscription/module_subscription_actor.rs +++ b/crates/core/src/subscription/module_subscription_actor.rs @@ -378,14 +378,12 @@ impl ModuleSubscriptions { match version { RawModuleDefVersion::V9OrEarlier => 0, RawModuleDefVersion::V10 => 1, - RawModuleDefVersion::V11 => 2, } } fn decode_module_def_version(version: u8) -> RawModuleDefVersion { match version { 1 => RawModuleDefVersion::V10, - 2 => RawModuleDefVersion::V11, 0 => RawModuleDefVersion::V9OrEarlier, _ => unreachable!("invalid stored module definition version"), } diff --git a/crates/lib/src/db/raw_def.rs b/crates/lib/src/db/raw_def.rs index c67f22dacc8..a29161403a5 100644 --- a/crates/lib/src/db/raw_def.rs +++ b/crates/lib/src/db/raw_def.rs @@ -16,5 +16,3 @@ pub use v8::*; pub mod v9; pub mod v10; - -pub mod v11; diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 8e5a9ecb1e6..17f3e377c74 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -95,6 +95,10 @@ pub enum RawModuleDefV10Section { /// HTTP route definitions. HttpRoutes(Vec), + + /// Module bindings capabilities, independent of function visibility. + /// Older hosts reject this section instead of silently ignoring its requirements. + Capabilities(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -317,6 +321,9 @@ pub struct RawReducerDefV10 { } /// The visibility of a function (reducer or procedure). +/// +/// New variants MUST be appended to preserve existing BSATN tags. Older hosts +/// reject unknown tags, so new restrictions cannot be silently discarded. #[derive(Debug, Copy, Clone, SpacetimeType)] #[sats(crate = crate)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] @@ -326,11 +333,31 @@ pub enum FunctionVisibility { /// Still callable by the module owner, collaborators, /// and internal module code. /// - /// Enabled for lifecycle reducers and scheduled functions by default. + /// The default for scheduled functions. Older lifecycle definitions also use + /// this tag; lifecycle assignments always enforce host-event-only invocation. Private, - /// Callable from client code. + /// Callable from client code, with the historical contextual defaults. + /// Scheduled functions become Private; lifecycle reducers remain host event handlers. ClientCallable, + + /// Callable only by a host-verified internal invocation. + Internal, + + /// Explicitly callable from client code, including when scheduled. + /// This separate tag preserves the meaning of existing ClientCallable definitions. + ExplicitClientCallable, +} + +impl FunctionVisibility { + /// Encode a source declaration without changing historical contextual defaults. + pub fn from_declaration(declared: Option, default: Self) -> Self { + match declared { + Some(Self::ClientCallable | Self::ExplicitClientCallable) => Self::ExplicitClientCallable, + Some(visibility) => visibility, + None => default, + } + } } /// A schedule definition. @@ -671,7 +698,7 @@ impl RawModuleDefV10Builder { } /// Get mutable access to the typespace section, creating it if missing. - pub(super) fn typespace_mut(&mut self) -> &mut Typespace { + fn typespace_mut(&mut self) -> &mut Typespace { let idx = self .module .sections @@ -785,7 +812,7 @@ impl RawModuleDefV10Builder { } /// Get mutable access to the types section, creating it if missing. - pub(super) fn types_mut(&mut self) -> &mut Vec { + fn types_mut(&mut self) -> &mut Vec { let idx = self .module .sections @@ -1027,10 +1054,20 @@ impl RawModuleDefV10Builder { /// This is because `SpacetimeType` is not implemented for `ReducerContext`, /// so it can never act like an ordinary argument.) pub fn add_reducer(&mut self, source_name: impl Into, params: ProductType) { + self.add_reducer_with_visibility(source_name, params, None); + } + + /// Add a reducer with an optional explicit visibility declaration. + pub fn add_reducer_with_visibility( + &mut self, + source_name: impl Into, + params: ProductType, + visibility: Option, + ) { self.reducers_mut().push(RawReducerDefV10 { source_name: source_name.into(), params, - visibility: FunctionVisibility::ClientCallable, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::ClientCallable), ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }); @@ -1051,12 +1088,23 @@ impl RawModuleDefV10Builder { source_name: impl Into, params: ProductType, return_type: AlgebraicType, + ) { + self.add_procedure_with_visibility(source_name, params, return_type, None); + } + + /// Add a procedure with an optional explicit visibility declaration. + pub fn add_procedure_with_visibility( + &mut self, + source_name: impl Into, + params: ProductType, + return_type: AlgebraicType, + visibility: Option, ) { self.procedures_mut().push(RawProcedureDefV10 { source_name: source_name.into(), params, return_type, - visibility: FunctionVisibility::ClientCallable, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::ClientCallable), }) } @@ -1088,6 +1136,19 @@ impl RawModuleDefV10Builder { lifecycle_spec: Lifecycle, function_name: impl Into, params: ProductType, + ) { + self.add_lifecycle_reducer_with_visibility(lifecycle_spec, function_name, params, None); + } + + /// Add a lifecycle reducer with an optional visibility declaration. + /// Source bindings must reject explicit Private or public lifecycle annotations. + /// The raw Private tag remains accepted for compatibility with existing modules. + pub fn add_lifecycle_reducer_with_visibility( + &mut self, + lifecycle_spec: Lifecycle, + function_name: impl Into, + params: ProductType, + visibility: Option, ) { let function_name = function_name.into(); self.lifecycle_reducers_mut().push(RawLifeCycleReducerDefV10 { @@ -1098,7 +1159,7 @@ impl RawModuleDefV10Builder { self.reducers_mut().push(RawReducerDefV10 { source_name: function_name, params, - visibility: FunctionVisibility::Private, + visibility: FunctionVisibility::from_declaration(visibility, FunctionVisibility::Private), ok_return_type: reducer_default_ok_return_type(), err_return_type: reducer_default_err_return_type(), }); @@ -1124,6 +1185,22 @@ impl RawModuleDefV10Builder { }); } + /// Declare a module bindings capability. + pub fn add_capability(&mut self, capability: impl Into) { + if let Some(RawModuleDefV10Section::Capabilities(names)) = self + .module + .sections + .iter_mut() + .find(|section| matches!(section, RawModuleDefV10Section::Capabilities(_))) + { + names.push(capability.into()); + } else { + self.module + .sections + .push(RawModuleDefV10Section::Capabilities(vec![capability.into()])); + } + } + /// Add a row-level security policy to the module. /// /// The `sql` expression should be a valid SQL expression that will be used to filter rows. @@ -1406,3 +1483,141 @@ impl RawTableDefBuilderV10<'_> { .map(|i| ColId(i as u16)) } } + +#[cfg(test)] +mod compatibility_tests { + use super::*; + use crate::{bsatn, RawModuleDef}; + + // Frozen pre-extension wire types. Do not replace the visibility, function, + // or section definitions below with their current counterparts. + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacyVisibility { + Private, + ClientCallable, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyReducer { + source_name: RawIdentifier, + params: ProductType, + visibility: LegacyVisibility, + ok_return_type: AlgebraicType, + err_return_type: AlgebraicType, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyProcedure { + source_name: RawIdentifier, + params: ProductType, + return_type: AlgebraicType, + visibility: LegacyVisibility, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacySection { + Typespace(Typespace), + Types(Vec), + Tables(Vec), + Reducers(Vec), + Procedures(Vec), + Views(Vec), + Schedules(Vec), + LifeCycleReducers(Vec), + RowLevelSecurity(Vec), + CaseConversionPolicy(CaseConversionPolicy), + ExplicitNames(ExplicitNames), + HttpHandlers(Vec), + HttpRoutes(Vec), + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + struct LegacyV10 { + sections: Vec, + } + + #[derive(SpacetimeType)] + #[sats(crate = crate)] + enum LegacyModule { + V8BackCompat(crate::RawModuleDefV8), + V9(super::super::v9::RawModuleDefV9), + V10(LegacyV10), + } + + #[test] + fn existing_v10_wire_tags_and_function_products_are_unchanged() { + for (visibility, expected) in [ + (FunctionVisibility::Private, 0), + (FunctionVisibility::ClientCallable, 1), + (FunctionVisibility::Internal, 2), + (FunctionVisibility::ExplicitClientCallable, 3), + ] { + assert_eq!(bsatn::to_vec(&visibility).unwrap(), [expected]); + } + let legacy = LegacyModule::V10(LegacyV10 { + sections: vec![ + LegacySection::Reducers(vec![LegacyReducer { + source_name: "run".into(), + params: ProductType::unit(), + visibility: LegacyVisibility::ClientCallable, + ok_return_type: reducer_default_ok_return_type(), + err_return_type: reducer_default_err_return_type(), + }]), + LegacySection::Procedures(vec![LegacyProcedure { + source_name: "read".into(), + params: ProductType::unit(), + return_type: AlgebraicType::U64, + visibility: LegacyVisibility::Private, + }]), + ], + }); + let bytes = bsatn::to_vec(&legacy).unwrap(); + assert_eq!(bytes[0], 2); + let current: RawModuleDef = bsatn::from_slice(&bytes).unwrap(); + assert_eq!(bsatn::to_vec(¤t).unwrap(), bytes); + let frozen: LegacyModule = bsatn::from_slice(&bsatn::to_vec(¤t).unwrap()).unwrap(); + assert_eq!(bsatn::to_vec(&frozen).unwrap(), bytes); + + assert_eq!( + bsatn::to_vec(&RawModuleDefV10Section::HttpRoutes(vec![])).unwrap(), + [12, 0, 0, 0, 0] + ); + assert_eq!( + bsatn::to_vec(&RawModuleDefV10Section::Capabilities(vec![])).unwrap(), + [13, 0, 0, 0, 0] + ); + } + + #[test] + fn older_hosts_reject_new_visibility_and_capabilities() { + for visibility in [FunctionVisibility::Internal, FunctionVisibility::ExplicitClientCallable] { + for procedure in [false, true] { + let mut builder = RawModuleDefV10Builder::new(); + if procedure { + builder.add_procedure_with_visibility( + "run", + ProductType::unit(), + AlgebraicType::unit(), + Some(visibility), + ); + } else { + builder.add_reducer_with_visibility("run", ProductType::unit(), Some(visibility)); + } + let bytes = bsatn::to_vec(&RawModuleDef::V10(builder.finish())).unwrap(); + assert_eq!(bytes[0], 2); + assert!(bsatn::from_slice::(&bytes).is_err()); + assert!(bsatn::from_slice::(&bytes).is_ok()); + } + } + let mut builder = RawModuleDefV10Builder::new(); + builder.add_capability("hosted_auth_v1"); + let bytes = bsatn::to_vec(&RawModuleDef::V10(builder.finish())).unwrap(); + assert!(bsatn::from_slice::(&bytes).is_err()); + assert!(bsatn::from_slice::(&bytes).is_ok()); + } +} diff --git a/crates/lib/src/db/raw_def/v11.rs b/crates/lib/src/db/raw_def/v11.rs deleted file mode 100644 index ac6cc593291..00000000000 --- a/crates/lib/src/db/raw_def/v11.rs +++ /dev/null @@ -1,269 +0,0 @@ -//! Version 11 module definitions: explicit function visibility with contextual defaults. -//! -//! Non-function sections retain their V10 wire shapes. V11 is a distinct top-level -//! variant, so hosts unaware of Internal visibility reject the entire definition. - -use super::v10; -use super::v10::*; -use super::v9::Lifecycle; -use spacetimedb_sats::raw_identifier::RawIdentifier; -use spacetimedb_sats::typespace::TypespaceBuilder; -use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductType, SpacetimeType, Typespace}; -use std::{ - any::TypeId, - collections::BTreeMap, - ops::{Deref, DerefMut}, -}; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, SpacetimeType)] -#[sats(crate = crate)] -pub enum FunctionVisibilityV11 { - Private, - ClientCallable, - Internal, -} - -pub use FunctionVisibilityV11 as FunctionVisibility; - -#[derive(Default, Debug, Clone, SpacetimeType)] -#[sats(crate = crate)] -#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] -pub struct RawModuleDefV11 { - pub sections: Vec, -} - -#[derive(Debug, Clone, SpacetimeType)] -#[sats(crate = crate)] -#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] -#[non_exhaustive] -pub enum RawModuleDefV11Section { - Typespace(Typespace), - Types(Vec), - Tables(Vec), - Reducers(Vec), - Procedures(Vec), - Views(Vec), - Schedules(Vec), - LifeCycleReducers(Vec), - RowLevelSecurity(Vec), - CaseConversionPolicy(CaseConversionPolicy), - ExplicitNames(ExplicitNames), - HttpHandlers(Vec), - HttpRoutes(Vec), - /// Module bindings capabilities, independent of function visibility. - Capabilities(Vec), -} - -#[derive(Debug, Clone, SpacetimeType)] -#[sats(crate = crate)] -#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] -pub struct RawReducerDefV11 { - pub source_name: RawIdentifier, - pub params: ProductType, - /// None selects the context default; Some preserves the author's selection. - pub declared_visibility: Option, - pub ok_return_type: AlgebraicType, - pub err_return_type: AlgebraicType, -} - -#[derive(Debug, Clone, SpacetimeType)] -#[sats(crate = crate)] -#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] -pub struct RawProcedureDefV11 { - pub source_name: RawIdentifier, - pub params: ProductType, - /// None selects the context default; Some preserves the author's selection. - pub declared_visibility: Option, - pub return_type: AlgebraicType, -} - -/// Shares the unchanged V10 table/type builders while emitting only V11. -#[derive(Default)] -pub struct RawModuleDefV11Builder { - inner: RawModuleDefV10Builder, - type_map: BTreeMap, - declared_visibility: BTreeMap, - capabilities: Vec, -} - -impl Deref for RawModuleDefV11Builder { - type Target = RawModuleDefV10Builder; - fn deref(&self) -> &Self::Target { - &self.inner - } -} -impl DerefMut for RawModuleDefV11Builder { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} -impl RawModuleDefV11Builder { - pub fn new() -> Self { - Self::default() - } - pub fn add_type(&mut self) -> AlgebraicType { - TypespaceBuilder::add_type::(self) - } - - pub fn add_reducer_with_visibility( - &mut self, - name: impl Into, - params: ProductType, - visibility: Option, - ) { - let name = name.into(); - self.inner.add_reducer(name.clone(), params); - if let Some(visibility) = visibility { - self.declared_visibility.insert(name, visibility); - } - } - - pub fn add_lifecycle_reducer_with_visibility( - &mut self, - lifecycle: Lifecycle, - name: impl Into, - params: ProductType, - visibility: Option, - ) { - let name = name.into(); - self.inner.add_lifecycle_reducer(lifecycle, name.clone(), params); - if let Some(visibility) = visibility { - self.declared_visibility.insert(name, visibility); - } - } - - pub fn add_procedure_with_visibility( - &mut self, - name: impl Into, - params: ProductType, - return_type: AlgebraicType, - visibility: Option, - ) { - let name = name.into(); - self.inner.add_procedure(name.clone(), params, return_type); - if let Some(visibility) = visibility { - self.declared_visibility.insert(name, visibility); - } - } - - pub fn add_capability(&mut self, capability: impl Into) { - self.capabilities.push(capability.into()); - } - - pub fn finish(self) -> RawModuleDefV11 { - let declared = self.declared_visibility; - let mut sections: Vec<_> = self - .inner - .finish() - .sections - .into_iter() - .map(|section| match section { - RawModuleDefV10Section::Typespace(value) => RawModuleDefV11Section::Typespace(value), - RawModuleDefV10Section::Types(value) => RawModuleDefV11Section::Types(value), - RawModuleDefV10Section::Tables(value) => RawModuleDefV11Section::Tables(value), - RawModuleDefV10Section::Reducers(rows) => RawModuleDefV11Section::Reducers( - rows.into_iter() - .map(|row| RawReducerDefV11 { - declared_visibility: declared.get(&row.source_name).copied(), - source_name: row.source_name, - params: row.params, - ok_return_type: row.ok_return_type, - err_return_type: row.err_return_type, - }) - .collect(), - ), - RawModuleDefV10Section::Procedures(rows) => RawModuleDefV11Section::Procedures( - rows.into_iter() - .map(|row| RawProcedureDefV11 { - declared_visibility: declared.get(&row.source_name).copied(), - source_name: row.source_name, - params: row.params, - return_type: row.return_type, - }) - .collect(), - ), - RawModuleDefV10Section::Views(value) => RawModuleDefV11Section::Views(value), - RawModuleDefV10Section::Schedules(value) => RawModuleDefV11Section::Schedules(value), - RawModuleDefV10Section::LifeCycleReducers(value) => RawModuleDefV11Section::LifeCycleReducers(value), - RawModuleDefV10Section::RowLevelSecurity(value) => RawModuleDefV11Section::RowLevelSecurity(value), - RawModuleDefV10Section::CaseConversionPolicy(value) => { - RawModuleDefV11Section::CaseConversionPolicy(value) - } - RawModuleDefV10Section::ExplicitNames(value) => RawModuleDefV11Section::ExplicitNames(value), - RawModuleDefV10Section::HttpHandlers(value) => RawModuleDefV11Section::HttpHandlers(value), - RawModuleDefV10Section::HttpRoutes(value) => RawModuleDefV11Section::HttpRoutes(value), - }) - .collect(); - if !self.capabilities.is_empty() { - sections.push(RawModuleDefV11Section::Capabilities(self.capabilities)); - } - RawModuleDefV11 { sections } - } -} - -impl TypespaceBuilder for RawModuleDefV11Builder { - fn add( - &mut self, - typeid: TypeId, - source_name: Option<&'static str>, - make_ty: impl FnOnce(&mut Self) -> AlgebraicType, - ) -> AlgebraicType { - if let Some(reference) = self.type_map.get(&typeid) { - return AlgebraicType::Ref(*reference); - } - let reference = self.inner.typespace_mut().add(AlgebraicType::unit()); - self.type_map.insert(typeid, reference); - if let Some(name) = source_name { - self.inner.types_mut().push(RawTypeDefV10 { - source_name: v10::sats_name_to_scoped_name_v10(name), - ty: reference, - custom_ordering: true, - }); - } - let ty = make_ty(self); - self.inner.typespace_mut()[reference] = ty; - AlgebraicType::Ref(reference) - } -} - -impl RawModuleDefV11 { - pub fn reducers(&self) -> impl Iterator { - self.sections - .iter() - .filter_map(|section| match section { - RawModuleDefV11Section::Reducers(rows) => Some(rows), - _ => None, - }) - .flatten() - } - pub fn tables(&self) -> impl Iterator { - self.sections - .iter() - .filter_map(|section| match section { - RawModuleDefV11Section::Tables(rows) => Some(rows), - _ => None, - }) - .flatten() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{RawModuleDef, SpacetimeType}; - - #[derive(SpacetimeType)] - #[sats(crate = crate)] - enum LegacyRawModuleDef { - V8BackCompat(crate::RawModuleDefV8), - V9(super::super::v9::RawModuleDefV9), - V10(RawModuleDefV10), - } - - #[test] - fn legacy_decoder_rejects_v11_instead_of_ignoring_visibility() { - let bytes = crate::bsatn::to_vec(&RawModuleDef::V11(RawModuleDefV11::default())).unwrap(); - assert_eq!(bytes[0], 3); - assert!(crate::bsatn::from_slice::(&bytes).is_err()); - } -} diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs index a81b2b19637..43435a9e1c8 100644 --- a/crates/lib/src/deployment.rs +++ b/crates/lib/src/deployment.rs @@ -11,8 +11,8 @@ pub const SYSTEM_EMPTY_MODULE_VERSION: u32 = 1; /// Immutable Keccak-256 program identity of the version-1 bundled empty Wasm /// module. Control can verify initial program bytes without linking the host. pub const SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH: Hash = Hash::from_byte_array([ - 0x9b, 0x6c, 0xf2, 0xdb, 0x36, 0x44, 0xc1, 0xd3, 0x21, 0xd9, 0x7a, 0xe0, 0xfc, 0xd4, 0xab, 0x3f, 0xc0, 0x65, 0xfc, - 0x94, 0xf5, 0x4a, 0x49, 0xf6, 0x1c, 0x7a, 0x1d, 0xfa, 0x40, 0xa0, 0x26, 0x12, + 0x08, 0x36, 0x50, 0x97, 0xd1, 0xef, 0x20, 0x26, 0x53, 0x61, 0x5f, 0x02, 0xcc, 0x46, 0xbe, 0x16, 0x08, 0x80, 0x44, + 0xc6, 0xb9, 0xcd, 0x96, 0x60, 0xe3, 0xa0, 0xf3, 0x36, 0xe4, 0x79, 0x18, 0xf0, ]); pub const MAX_DEPLOYMENT_BYTES: usize = 256 * 1024; pub const PUBLISH_RETRY_WINDOW_MS: u64 = 7 * 24 * 60 * 60 * 1000; diff --git a/crates/lib/src/deployment/tests.rs b/crates/lib/src/deployment/tests.rs index d8f37993615..e720783d8aa 100644 --- a/crates/lib/src/deployment/tests.rs +++ b/crates/lib/src/deployment/tests.rs @@ -45,15 +45,15 @@ fn concrete_module_actions_preserve_wire_tags_and_export_distinct_names() { expected.extend(bsatn::to_vec(&module).unwrap()); assert_eq!(bsatn::to_vec(&ModuleAction::Set(module)).unwrap(), expected); - use crate::db::raw_def::v11::{RawModuleDefV11Builder, RawModuleDefV11Section}; - let mut builder = RawModuleDefV11Builder::new(); + use crate::db::raw_def::v10::{RawModuleDefV10Builder, RawModuleDefV10Section}; + let mut builder = RawModuleDefV10Builder::new(); builder.add_type::(); let raw = builder.finish(); let names: Vec<_> = raw .sections .iter() .filter_map(|section| match section { - RawModuleDefV11Section::Types(types) => Some(types), + RawModuleDefV10Section::Types(types) => Some(types), _ => None, }) .flatten() diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index a8febe31d9b..df7485ee282 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -169,7 +169,6 @@ pub enum RawModuleDef { V8BackCompat(RawModuleDefV8), V9(db::raw_def::v9::RawModuleDefV9), V10(db::raw_def::v10::RawModuleDefV10), - V11(db::raw_def::v11::RawModuleDefV11), // TODO(jgilles): It would be nice to have a custom error message if this fails with an unknown variant, // but I'm not sure if that can be done via the Deserialize trait. } diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 4a3f44a6ede..6f0417c4585 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -37,9 +37,6 @@ use spacetimedb_lib::db::raw_def::v10::{ RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, }; -use spacetimedb_lib::db::raw_def::v11::{ - RawModuleDefV11, RawModuleDefV11Section, RawProcedureDefV11, RawReducerDefV11, -}; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, RawMiscModuleExportV9, RawModuleDefV9, RawProcedureDefV9, RawReducerDefV9, RawRowLevelSecurityDefV9, @@ -177,8 +174,6 @@ pub enum RawModuleDefVersion { V9OrEarlier, /// Represents [`RawModuleDefV10`]. V10, - /// Explicit function visibility and contextual defaults. - V11, } impl ModuleDef { @@ -211,8 +206,7 @@ impl ModuleDef { pub fn into_raw(self) -> RawModuleDef { match self.raw_module_def_version { RawModuleDefVersion::V9OrEarlier => RawModuleDef::V9(self.try_into().expect("same-version conversion")), - RawModuleDefVersion::V10 => RawModuleDef::V10(self.try_into().expect("same-version conversion")), - RawModuleDefVersion::V11 => RawModuleDef::V11(self.into()), + RawModuleDefVersion::V10 => RawModuleDef::V10(self.into()), } } @@ -501,7 +495,6 @@ impl TryFrom for ModuleDef { RawModuleDef::V8BackCompat(v8_mod) => Self::try_from(v8_mod), RawModuleDef::V9(v9_mod) => Self::try_from(v9_mod), RawModuleDef::V10(v10_mod) => Self::try_from(v10_mod), - RawModuleDef::V11(v11_mod) => Self::try_from(v11_mod), _ => Err(crate::error::ValidationError::UnsupportedModuleVersion.into()), } } @@ -595,14 +588,8 @@ impl TryFrom for ModuleDef { } } -impl TryFrom for RawModuleDefV10 { - type Error = SchemaConversionError; - fn try_from(val: ModuleDef) -> Result { - if val.raw_module_def_version != RawModuleDefVersion::V10 { - return Err(SchemaConversionError { - target: RawModuleDefVersion::V10, - }); - } +impl From for RawModuleDefV10 { + fn from(val: ModuleDef) -> Self { let ModuleDef { tables, views, @@ -618,7 +605,7 @@ impl TryFrom for RawModuleDefV10 { http_handlers, http_routes, raw_module_def_version: _, - capabilities: _, + capabilities, } = val; let mut sections = Vec::new(); @@ -678,9 +665,17 @@ impl TryFrom for RawModuleDefV10 { RawIdentifier::from(rd.accessor_name.clone()), RawIdentifier::from(rd.name.clone()), ); - rd.try_into() + let public_scheduled = rd.visibility.is_client_callable() + && schedules + .iter() + .any(|schedule| schedule.function_name == RawIdentifier::from(rd.name.clone())); + let mut raw: RawReducerDefV10 = rd.into(); + if public_scheduled { + raw.visibility = RawFunctionVisibility::ExplicitClientCallable; + } + raw }) - .collect::>()?; + .collect(); if !raw_reducers.is_empty() { sections.push(RawModuleDefV10Section::Reducers(raw_reducers)); } @@ -693,9 +688,17 @@ impl TryFrom for RawModuleDefV10 { RawIdentifier::from(pd.accessor_name.clone()), RawIdentifier::from(pd.name.clone()), ); - pd.try_into() + let public_scheduled = pd.visibility.is_client_callable() + && schedules + .iter() + .any(|schedule| schedule.function_name == RawIdentifier::from(pd.name.clone())); + let mut raw: RawProcedureDefV10 = pd.into(); + if public_scheduled { + raw.visibility = RawFunctionVisibility::ExplicitClientCallable; + } + raw }) - .collect::>()?; + .collect(); if !raw_procedures.is_empty() { sections.push(RawModuleDefV10Section::Procedures(raw_procedures)); } @@ -753,173 +756,10 @@ impl TryFrom for RawModuleDefV10 { // Always emit ExplicitNames so canonical names survive the round-trip. sections.push(RawModuleDefV10Section::ExplicitNames(explicit_names)); - Ok(RawModuleDefV10 { sections }) - } -} - -impl TryFrom for ModuleDef { - type Error = ValidationErrors; - fn try_from(value: RawModuleDefV11) -> Result { - validate::v11::validate(value) - } -} - -impl From for RawModuleDefV11 { - fn from(val: ModuleDef) -> Self { - let ModuleDef { - tables, - views, - reducers, - lifecycle_reducers, - types, - typespace, - stored_in_table_def: _, - typespace_for_generate: _, - refmap: _, - row_level_security_raw, - procedures, - http_handlers, - http_routes, - raw_module_def_version: _, - capabilities, - } = val; - - let mut sections = Vec::new(); - let mut explicit_names = ExplicitNames::default(); - - sections.push(RawModuleDefV11Section::Typespace(typespace)); - - // Extract lifecycle reducer names before consuming reducers. - let raw_lifecycle: Vec = lifecycle_reducers - .into_iter() - .filter_map(|(lifecycle, reducer_id)| { - let id = reducer_id?; - let (name, _) = reducers.get_index(id.idx())?; - Some(RawLifeCycleReducerDefV10 { - lifecycle_spec: lifecycle, - function_name: name.clone().into(), - }) - }) - .collect(); - - let raw_types: Vec = types.into_values().map(Into::into).collect(); - if !raw_types.is_empty() { - sections.push(RawModuleDefV11Section::Types(raw_types)); - } - - // Collect schedules from tables (V10 stores them in a separate section). - // Also collect ExplicitNames for tables: accessor_name → source_name, name → canonical_name. - let mut schedules = Vec::new(); - let raw_tables: Vec = tables - .into_values() - .map(|td| { - // Always emit name as ExplicitNames canonical_name. - explicit_names.insert_table( - RawIdentifier::from(td.accessor_name.clone()), - RawIdentifier::from(td.name.clone()), - ); - if let Some(sched) = td.schedule.clone() { - schedules.push(RawScheduleDefV10 { - source_name: Some(sched.name.into()), - table_name: td.name.clone().into(), - schedule_at_col: sched.at_column, - function_name: sched.function_name.into(), - }); - } - td.into() - }) - .collect(); - if !raw_tables.is_empty() { - sections.push(RawModuleDefV11Section::Tables(raw_tables)); - } - - // Collect ExplicitNames for reducers: accessor_name → source_name, name → canonical_name. - let raw_reducers: Vec = reducers - .into_values() - .map(|rd| { - explicit_names.insert_function( - RawIdentifier::from(rd.accessor_name.clone()), - RawIdentifier::from(rd.name.clone()), - ); - rd.into() - }) - .collect(); - if !raw_reducers.is_empty() { - sections.push(RawModuleDefV11Section::Reducers(raw_reducers)); - } - - // Collect ExplicitNames for procedures: accessor_name → source_name, name → canonical_name. - let raw_procedures: Vec = procedures - .into_values() - .map(|pd| { - explicit_names.insert_function( - RawIdentifier::from(pd.accessor_name.clone()), - RawIdentifier::from(pd.name.clone()), - ); - pd.into() - }) - .collect(); - if !raw_procedures.is_empty() { - sections.push(RawModuleDefV11Section::Procedures(raw_procedures)); - } - - let raw_http_handlers: Vec = http_handlers - .into_values() - .map(|hd| RawHttpHandlerDefV10 { - source_name: hd.accessor_name.into(), - }) - .collect(); - if !raw_http_handlers.is_empty() { - sections.push(RawModuleDefV11Section::HttpHandlers(raw_http_handlers)); - } - - if !http_routes.is_empty() { - let raw_http_routes: Vec = http_routes - .into_iter() - .map(|route| RawHttpRouteDefV10 { - handler_function: route.handler_name.into(), - method: route.method, - path: RawIdentifier::new(route.path.as_ref()), - }) - .collect(); - sections.push(RawModuleDefV11Section::HttpRoutes(raw_http_routes)); - } - - // Collect ExplicitNames for views: accessor_name → source_name, name → canonical_name. - let raw_views: Vec = views - .into_values() - .map(|vd| { - explicit_names.insert_function( - RawIdentifier::from(vd.accessor_name.clone()), - RawIdentifier::from(vd.name.clone()), - ); - vd.into() - }) - .collect(); - if !raw_views.is_empty() { - sections.push(RawModuleDefV11Section::Views(raw_views)); - } - - if !schedules.is_empty() { - sections.push(RawModuleDefV11Section::Schedules(schedules)); - } - - if !raw_lifecycle.is_empty() { - sections.push(RawModuleDefV11Section::LifeCycleReducers(raw_lifecycle)); - } - - let raw_rls: Vec = row_level_security_raw.into_values().collect(); - if !raw_rls.is_empty() { - sections.push(RawModuleDefV11Section::RowLevelSecurity(raw_rls)); - } - - // Always emit ExplicitNames so canonical names survive the round-trip. - sections.push(RawModuleDefV11Section::ExplicitNames(explicit_names)); - if !capabilities.is_empty() { - sections.push(RawModuleDefV11Section::Capabilities(capabilities.into_iter().collect())); + sections.push(RawModuleDefV10Section::Capabilities(capabilities.into_iter().collect())); } - RawModuleDefV11 { sections } + RawModuleDefV10 { sections } } } @@ -1971,42 +1811,23 @@ impl From for FunctionVisibility { fn from(val: RawFunctionVisibility) -> Self { match val { RawFunctionVisibility::Private => FunctionVisibility::Private, - RawFunctionVisibility::ClientCallable => FunctionVisibility::ClientCallable, + RawFunctionVisibility::ClientCallable | RawFunctionVisibility::ExplicitClientCallable => { + FunctionVisibility::ClientCallable + } + RawFunctionVisibility::Internal => FunctionVisibility::Internal, } } } #[derive(Debug, Clone, thiserror::Error)] -#[error("schema cannot be represented as {target:?} without losing function visibility or source-version semantics; request schema version 11")] +#[error("schema cannot be represented as {target:?} without losing function visibility or source-version semantics; request schema version 10")] pub struct SchemaConversionError { pub target: RawModuleDefVersion, } -impl TryFrom for RawFunctionVisibility { - type Error = SchemaConversionError; - fn try_from(val: FunctionVisibility) -> Result { +impl From for RawFunctionVisibility { + fn from(val: FunctionVisibility) -> Self { match val { - FunctionVisibility::Private => Ok(Self::Private), - FunctionVisibility::ClientCallable => Ok(Self::ClientCallable), - FunctionVisibility::Internal => Err(SchemaConversionError { - target: RawModuleDefVersion::V10, - }), - } - } -} - -impl From for FunctionVisibility { - fn from(value: raw_def::v11::FunctionVisibility) -> Self { - match value { - raw_def::v11::FunctionVisibility::Private => Self::Private, - raw_def::v11::FunctionVisibility::ClientCallable => Self::ClientCallable, - raw_def::v11::FunctionVisibility::Internal => Self::Internal, - } - } -} -impl From for raw_def::v11::FunctionVisibility { - fn from(value: FunctionVisibility) -> Self { - match value { FunctionVisibility::Private => Self::Private, FunctionVisibility::ClientCallable => Self::ClientCallable, FunctionVisibility::Internal => Self::Internal, @@ -2068,21 +1889,20 @@ impl TryFrom for RawReducerDefV9 { } } -impl TryFrom for RawReducerDefV10 { - type Error = SchemaConversionError; - fn try_from(val: ReducerDef) -> Result { +impl From for RawReducerDefV10 { + fn from(val: ReducerDef) -> Self { let visibility = if val.lifecycle.is_some() { RawFunctionVisibility::Private } else { - val.visibility.try_into()? + val.visibility.into() }; - Ok(RawReducerDefV10 { + RawReducerDefV10 { source_name: val.accessor_name.into(), params: val.params, visibility, ok_return_type: val.ok_return_type, err_return_type: val.err_return_type, - }) + } } } @@ -2158,15 +1978,14 @@ impl TryFrom for RawProcedureDefV9 { } } -impl TryFrom for RawProcedureDefV10 { - type Error = SchemaConversionError; - fn try_from(val: ProcedureDef) -> Result { - Ok(RawProcedureDefV10 { +impl From for RawProcedureDefV10 { + fn from(val: ProcedureDef) -> Self { + RawProcedureDefV10 { source_name: val.accessor_name.into(), params: val.params, return_type: val.return_type, - visibility: val.visibility.try_into()?, - }) + visibility: val.visibility.into(), + } } } @@ -2441,25 +2260,3 @@ mod tests { == 2)) } } - -impl From for RawReducerDefV11 { - fn from(value: ReducerDef) -> Self { - Self { - source_name: value.accessor_name.into(), - params: value.params, - declared_visibility: Some(value.visibility.into()), - ok_return_type: value.ok_return_type, - err_return_type: value.err_return_type, - } - } -} -impl From for RawProcedureDefV11 { - fn from(value: ProcedureDef) -> Self { - Self { - source_name: value.accessor_name.into(), - params: value.params, - declared_visibility: Some(value.visibility.into()), - return_type: value.return_type, - } - } -} diff --git a/crates/schema/src/def/validate.rs b/crates/schema/src/def/validate.rs index 0b7c5d28c69..44829a0e8b8 100644 --- a/crates/schema/src/def/validate.rs +++ b/crates/schema/src/def/validate.rs @@ -3,7 +3,6 @@ use crate::error::ValidationErrors; pub mod v10; -pub mod v11; pub mod v8; pub mod v9; diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index 7ed1b846c63..7bf35c42898 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -71,9 +71,50 @@ impl From for ValidationCase { } } } -/// Validate a `RawModuleDefV9` and convert it into a `ModuleDef`, +/// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { + let mut seen_capabilities = false; + let mut capabilities = std::collections::BTreeSet::new(); + for section in &def.sections { + if let RawModuleDefV10Section::Capabilities(names) = section { + if seen_capabilities { + return Err(ValidationError::DuplicateModuleSection { + section: "Capabilities".into(), + } + .into()); + } + seen_capabilities = true; + if names.len() > 32 { + return Err(ValidationError::InvalidModuleCapabilities.into()); + } + for name in names { + if name.is_empty() + || name.len() > 64 + || !name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + || !capabilities.insert(name.clone()) + { + return Err(ValidationError::InvalidModuleCapabilities.into()); + } + } + } + } + // Retain the raw distinction until schedules are attached. Tag 1 has the + // historical contextual default; tag 3 is an explicit public declaration. + let raw_visibility: HashMap<_, _> = def + .reducers() + .into_iter() + .flatten() + .map(|function| (function.source_name.clone(), function.visibility)) + .chain( + def.procedures() + .into_iter() + .flatten() + .map(|function| (function.source_name.clone(), function.visibility)), + ) + .collect(); let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); let known_type_definitions = def.types().into_iter().flatten().map(|def| def.ty); let case_policy = def.case_conversion_policy().into(); @@ -270,7 +311,12 @@ pub fn validate(def: RawModuleDefV10) -> Result { attach_schedules_to_tables(&mut tables, schedules)?; check_scheduled_functions_exist(&mut tables, &reducers, &procedures)?; - change_scheduled_functions_and_lifetimes_visibility(&tables, &mut reducers, &mut procedures)?; + change_scheduled_functions_and_lifetimes_visibility( + &tables, + &mut reducers, + &mut procedures, + &raw_visibility, + )?; assign_query_view_primary_keys(&tables, &mut views); Ok((tables, types, reducers, procedures, views, http_handlers_and_routes)) @@ -315,17 +361,18 @@ pub fn validate(def: RawModuleDefV10) -> Result { procedures, http_handlers, http_routes, - capabilities: Default::default(), + capabilities, raw_module_def_version: RawModuleDefVersion::V10, }) } -/// Change the visibility of scheduled functions and lifecycle reducers to Internal. -/// +/// Apply historical schedule defaults only to the original ClientCallable tag. +/// Lifecycle reducers always retain their separate host-event restriction. fn change_scheduled_functions_and_lifetimes_visibility( tables: &HashMap, reducers: &mut IndexMap, procedures: &mut IndexMap, + raw_visibility: &HashMap, ) -> Result<()> { for sched_def in tables.iter().filter_map(|(_, t)| t.schedule.as_ref()) { match sched_def.function_kind { @@ -337,7 +384,12 @@ fn change_scheduled_functions_and_lifetimes_visibility( } })?; - def.visibility = crate::def::FunctionVisibility::Private; + if matches!( + raw_visibility.get(&RawIdentifier::from(def.accessor_name.clone())), + Some(RawFunctionVisibility::ClientCallable) + ) { + def.visibility = crate::def::FunctionVisibility::Private; + } } FunctionKind::Procedure => { @@ -348,7 +400,12 @@ fn change_scheduled_functions_and_lifetimes_visibility( } })?; - def.visibility = crate::def::FunctionVisibility::Private; + if matches!( + raw_visibility.get(&RawIdentifier::from(def.accessor_name.clone())), + Some(RawFunctionVisibility::ClientCallable) + ) { + def.visibility = crate::def::FunctionVisibility::Private; + } } FunctionKind::Unknown => {} @@ -357,6 +414,15 @@ fn change_scheduled_functions_and_lifetimes_visibility( for red_def in reducers.iter_mut().map(|(_, r)| r) { if red_def.lifecycle.is_some() { + if matches!( + raw_visibility.get(&RawIdentifier::from(red_def.accessor_name.clone())), + Some(RawFunctionVisibility::ExplicitClientCallable) + ) { + return Err(ValidationError::InvalidLifecycleVisibility { + function: red_def.accessor_name.clone().into(), + } + .into()); + } red_def.visibility = crate::def::FunctionVisibility::Internal; } } @@ -2428,3 +2494,277 @@ mod tests { assert_eq!(view.param_columns[0].view_name, id("Level2Person")); } } + +#[cfg(test)] +mod visibility_tests { + use super::*; + use crate::def::FunctionVisibility; + use spacetimedb_lib::db::raw_def::v10; + use spacetimedb_lib::{db::raw_def::v9, RawModuleDef, ScheduleAt}; + use spacetimedb_sats::{AlgebraicType, ProductType}; + use v10::{FunctionVisibility as Declared, RawModuleDefV10Builder}; + + fn scheduled_module(visibility: Option, procedure: bool) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "Jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") + .finish(); + let params = ProductType::from([("job", AlgebraicType::Ref(row))]); + if procedure { + builder.add_procedure_with_visibility("run_job", params, AlgebraicType::unit(), visibility); + } else { + builder.add_reducer_with_visibility("run_job", params, visibility); + } + builder.add_schedule("Jobs", 1, "run_job"); + builder.finish().try_into().unwrap() + } + + #[test] + fn explicit_scheduled_visibility_overrides_the_private_default() { + for procedure in [false, true] { + for (selection, expected) in [ + (None, FunctionVisibility::Private), + (Some(Declared::Private), FunctionVisibility::Private), + (Some(Declared::Internal), FunctionVisibility::Internal), + (Some(Declared::ClientCallable), FunctionVisibility::ClientCallable), + ] { + let module = scheduled_module(selection, procedure); + let visibility = if procedure { + &module.procedure("run_job").unwrap().visibility + } else { + &module.reducer("run_job").unwrap().visibility + }; + assert_eq!(visibility, &expected); + assert_eq!(module.raw_module_def_version(), RawModuleDefVersion::V10); + } + } + } + + #[test] + fn ordinary_defaults_and_lifecycle_restrictions() { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer("ordinary", ProductType::unit()); + builder.add_procedure("ordinary_procedure", ProductType::unit(), AlgebraicType::unit()); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "initialize", ProductType::unit()); + let module: ModuleDef = builder.finish().try_into().unwrap(); + assert!(module.reducer("ordinary").unwrap().visibility.is_client_callable()); + assert!(module + .procedure("ordinary_procedure") + .unwrap() + .visibility + .is_client_callable()); + assert!(module.reducer("initialize").unwrap().visibility.is_internal()); + let exported: RawModuleDefV10 = module.into(); + assert!(exported + .reducers() + .into_iter() + .flatten() + .all(|function| matches!(function.visibility, Declared::ClientCallable | Declared::Private))); + assert!(exported + .procedures() + .into_iter() + .flatten() + .all(|function| matches!(function.visibility, Declared::ClientCallable))); + for selection in [Declared::ClientCallable, Declared::ExplicitClientCallable] { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer_with_visibility( + v9::Lifecycle::Init, + "initialize", + ProductType::unit(), + Some(selection), + ); + assert!(ModuleDef::try_from(builder.finish()) + .unwrap_err() + .to_string() + .contains("must have Internal visibility")); + } + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer_with_visibility( + v9::Lifecycle::Init, + "initialize", + ProductType::unit(), + Some(Declared::Internal), + ); + assert!(ModuleDef::try_from(builder.finish()).is_ok()); + } + + #[test] + fn duplicate_definitions_sections_and_lifecycles_are_rejected() { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer("same", ProductType::unit()); + builder.add_procedure("same", ProductType::unit(), AlgebraicType::unit()); + assert!(ModuleDef::try_from(builder.finish()).is_err()); + let raw = v10::RawModuleDefV10 { + sections: vec![ + v10::RawModuleDefV10Section::Capabilities(vec![]), + v10::RawModuleDefV10Section::Capabilities(vec![]), + ], + }; + assert!(ModuleDef::try_from(raw) + .unwrap_err() + .to_string() + .contains("repeated V10 section")); + let mut builder = RawModuleDefV10Builder::new(); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "a", ProductType::unit()); + builder.add_lifecycle_reducer(v9::Lifecycle::Init, "b", ProductType::unit()); + assert!(ModuleDef::try_from(builder.finish()).is_err()); + } + + #[test] + fn resolved_v10_roundtrips_without_reapplying_defaults_and_rejects_v9_exports() { + for procedure in [false, true] { + for selection in [ + None, + Some(Declared::Private), + Some(Declared::Internal), + Some(Declared::ClientCallable), + ] { + let module = scheduled_module(selection, procedure); + assert!(v9::RawModuleDefV9::try_from(module.clone()).is_err()); + let RawModuleDef::V10(raw) = module.clone().into_raw() else { + panic!("lost source version") + }; + if matches!(selection, Some(Declared::ClientCallable)) { + assert!(raw + .reducers() + .into_iter() + .flatten() + .map(|function| &function.visibility) + .chain( + raw.procedures() + .into_iter() + .flatten() + .map(|function| &function.visibility) + ) + .all(|visibility| matches!(visibility, Declared::ExplicitClientCallable))); + } + let bytes = spacetimedb_lib::bsatn::to_vec(&RawModuleDef::V10(raw)).unwrap(); + let roundtrip: RawModuleDef = spacetimedb_lib::bsatn::from_slice(&bytes).unwrap(); + let roundtrip: ModuleDef = roundtrip.try_into().unwrap(); + if procedure { + assert_eq!( + roundtrip.procedure("run_job").unwrap().visibility, + module.procedure("run_job").unwrap().visibility + ); + } else { + assert_eq!( + roundtrip.reducer("run_job").unwrap().visibility, + module.reducer("run_job").unwrap().visibility + ); + } + assert_eq!(roundtrip.raw_module_def_version(), RawModuleDefVersion::V10); + } + } + } + + #[test] + fn legacy_v9_schedules_stay_public_and_v10_schedules_stay_private() { + let mut builder = v9::RawModuleDefV9Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index(v9::btree(0), "jobs_id_idx") + .with_schedule("run_job", 1) + .finish(); + builder.add_reducer("run_job", ProductType::from([("job", row.into())]), None); + let v9: ModuleDef = builder.finish().try_into().unwrap(); + assert!(v9.reducer("run_job").unwrap().visibility.is_client_callable()); + let upgraded: RawModuleDefV10 = v9.clone().into(); + assert!(matches!( + upgraded.reducers().unwrap()[0].visibility, + Declared::ExplicitClientCallable + )); + let upgraded: ModuleDef = upgraded.try_into().unwrap(); + assert!(upgraded.reducer("run_job").unwrap().visibility.is_client_callable()); + assert!(matches!(v9.into_raw(), RawModuleDef::V9(_))); + + let mut builder = v10::RawModuleDefV10Builder::new(); + let at = builder.add_type::(); + let row = builder + .build_table_with_new_type( + "jobs", + ProductType::from([("id", AlgebraicType::U64), ("at", at)]), + true, + ) + .with_auto_inc_primary_key(0) + .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") + .finish(); + builder.add_reducer("run_job", ProductType::from([("job", row.into())])); + builder.add_schedule("jobs", 1, "run_job"); + let v10: ModuleDef = builder.finish().try_into().unwrap(); + assert!(v10.reducer("run_job").unwrap().visibility.is_private()); + assert!(matches!(v10.into_raw(), RawModuleDef::V10(_))); + } + + #[test] + fn capabilities_are_explicit_bounded_and_preserved() { + let bare: ModuleDef = RawModuleDefV10Builder::new().finish().try_into().unwrap(); + assert!(!bare.supports_hosted_auth_v1()); + let mut builder = RawModuleDefV10Builder::new(); + builder.add_capability("hosted_auth_v1"); + let module: ModuleDef = builder.finish().try_into().unwrap(); + assert!(module.supports_hosted_auth_v1()); + let reloaded: ModuleDef = module.into_raw().try_into().unwrap(); + assert!(reloaded.supports_hosted_auth_v1()); + for names in [ + vec!["".to_string()], + vec!["Uppercase".to_string()], + vec!["with-dash".to_string()], + vec!["a".repeat(65)], + vec!["duplicate".to_string(); 2], + (0..33).map(|i| format!("cap_{i}")).collect(), + ] { + let mut builder = RawModuleDefV10Builder::new(); + for name in names { + builder.add_capability(name); + } + assert!(ModuleDef::try_from(builder.finish()).is_err()); + } + } + + #[test] + fn narrowing_function_visibility_is_a_reported_client_break() { + let module = |visibility| { + let mut builder = RawModuleDefV10Builder::new(); + builder.add_reducer_with_visibility("run_now", ProductType::unit(), Some(visibility)); + ModuleDef::try_from(builder.finish()).unwrap() + }; + let public = module(Declared::ClientCallable); + let internal = module(Declared::Internal); + let plan = crate::auto_migrate::ponder_migrate(&public, &internal).unwrap(); + assert!(plan.breaks_client()); + let display = plan + .pretty_print(crate::auto_migrate::PrettyPrintStyle::NoColor) + .unwrap(); + assert!(display.contains("run_now")); + assert!(display.contains("Internal")); + assert!(!crate::auto_migrate::ponder_migrate(&internal, &public) + .unwrap() + .breaks_client()); + } + + #[test] + fn visibility_authority_is_cumulative_without_elevating_the_owner() { + for (visibility, external, owner, internal) in [ + (FunctionVisibility::Internal, false, false, true), + (FunctionVisibility::Private, false, true, true), + (FunctionVisibility::ClientCallable, true, true, true), + ] { + assert_eq!(visibility.allows_invocation(false, false), external); + assert_eq!(visibility.allows_invocation(false, true), owner); + assert_eq!(visibility.allows_invocation(true, false), internal); + } + } +} diff --git a/crates/schema/src/def/validate/v11.rs b/crates/schema/src/def/validate/v11.rs deleted file mode 100644 index 2bd91e23da1..00000000000 --- a/crates/schema/src/def/validate/v11.rs +++ /dev/null @@ -1,353 +0,0 @@ -//! V11 reuses V10 structural validation, then resolves declarations after schedules -//! and lifecycle assignments exist. No V11 metadata is decoded as a V10 module. -use super::Result; -use crate::{ - def::{FunctionVisibility, ModuleDef, RawModuleDefVersion}, - error::ValidationError, -}; -use spacetimedb_lib::db::raw_def::{v10, v11}; -use spacetimedb_sats::raw_identifier::RawIdentifier; -use std::collections::{BTreeMap, BTreeSet, HashSet}; - -pub fn validate(def: v11::RawModuleDefV11) -> Result { - let mut seen_sections = HashSet::new(); - let mut declared = BTreeMap::new(); - let mut sections = Vec::new(); - let mut capabilities = BTreeSet::new(); - for section in def.sections { - if !seen_sections.insert(std::mem::discriminant(§ion)) { - return Err(ValidationError::DuplicateModuleSection { - section: format!("{:?}", std::mem::discriminant(§ion)), - } - .into()); - } - if let v11::RawModuleDefV11Section::Capabilities(names) = section { - if names.len() > 32 { - return Err(ValidationError::InvalidModuleCapabilities.into()); - } - for name in names { - if name.is_empty() - || name.len() > 64 - || !name - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') - || !capabilities.insert(name) - { - return Err(ValidationError::InvalidModuleCapabilities.into()); - } - } - continue; - } - sections.push(match section { - v11::RawModuleDefV11Section::Reducers(rows) => v10::RawModuleDefV10Section::Reducers( - rows.into_iter() - .map(|row| { - insert_declaration(&mut declared, &row.source_name, row.declared_visibility)?; - Ok(v10::RawReducerDefV10 { - source_name: row.source_name, - params: row.params, - visibility: v10::FunctionVisibility::ClientCallable, - ok_return_type: row.ok_return_type, - err_return_type: row.err_return_type, - }) - }) - .collect::>()?, - ), - v11::RawModuleDefV11Section::Procedures(rows) => v10::RawModuleDefV10Section::Procedures( - rows.into_iter() - .map(|row| { - insert_declaration(&mut declared, &row.source_name, row.declared_visibility)?; - Ok(v10::RawProcedureDefV10 { - source_name: row.source_name, - params: row.params, - visibility: v10::FunctionVisibility::ClientCallable, - return_type: row.return_type, - }) - }) - .collect::>()?, - ), - v11::RawModuleDefV11Section::Typespace(value) => v10::RawModuleDefV10Section::Typespace(value), - v11::RawModuleDefV11Section::Types(value) => v10::RawModuleDefV10Section::Types(value), - v11::RawModuleDefV11Section::Tables(value) => v10::RawModuleDefV10Section::Tables(value), - v11::RawModuleDefV11Section::Views(value) => v10::RawModuleDefV10Section::Views(value), - v11::RawModuleDefV11Section::Schedules(value) => v10::RawModuleDefV10Section::Schedules(value), - v11::RawModuleDefV11Section::LifeCycleReducers(value) => { - v10::RawModuleDefV10Section::LifeCycleReducers(value) - } - v11::RawModuleDefV11Section::RowLevelSecurity(value) => { - v10::RawModuleDefV10Section::RowLevelSecurity(value) - } - v11::RawModuleDefV11Section::CaseConversionPolicy(value) => { - v10::RawModuleDefV10Section::CaseConversionPolicy(value) - } - v11::RawModuleDefV11Section::ExplicitNames(value) => v10::RawModuleDefV10Section::ExplicitNames(value), - v11::RawModuleDefV11Section::HttpHandlers(value) => v10::RawModuleDefV10Section::HttpHandlers(value), - v11::RawModuleDefV11Section::HttpRoutes(value) => v10::RawModuleDefV10Section::HttpRoutes(value), - _ => unreachable!("all V11 sections are handled"), - }); - } - let mut module = super::v10::validate(v10::RawModuleDefV10 { sections })?; - for reducer in module.reducers.values_mut() { - let source_name = RawIdentifier::from(reducer.accessor_name.clone()); - let declaration = declared.get(&source_name).copied().flatten(); - if reducer.lifecycle.is_some() { - if declaration.is_some_and(|visibility| visibility != v11::FunctionVisibility::Internal) { - return Err(ValidationError::InvalidLifecycleVisibility { function: source_name }.into()); - } - reducer.visibility = FunctionVisibility::Internal; - } else if let Some(visibility) = declaration { - reducer.visibility = visibility.into(); - } - } - for procedure in module.procedures.values_mut() { - if let Some(visibility) = declared - .get(&RawIdentifier::from(procedure.accessor_name.clone())) - .copied() - .flatten() - { - procedure.visibility = visibility.into(); - } - } - module.raw_module_def_version = RawModuleDefVersion::V11; - module.capabilities = capabilities; - Ok(module) -} - -fn insert_declaration( - declared: &mut BTreeMap>, - name: &RawIdentifier, - visibility: Option, -) -> Result<()> { - if declared.insert(name.clone(), visibility).is_some() { - return Err(ValidationError::DuplicateName { name: name.clone() }.into()); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use spacetimedb_lib::{db::raw_def::v9, RawModuleDef, ScheduleAt}; - use spacetimedb_sats::{AlgebraicType, ProductType}; - use v11::{FunctionVisibility as Declared, RawModuleDefV11Builder}; - - fn scheduled_module(visibility: Option, procedure: bool) -> ModuleDef { - let mut builder = RawModuleDefV11Builder::new(); - let at = builder.add_type::(); - let row = builder - .build_table_with_new_type( - "Jobs", - ProductType::from([("id", AlgebraicType::U64), ("at", at)]), - true, - ) - .with_auto_inc_primary_key(0) - .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") - .finish(); - let params = ProductType::from([("job", AlgebraicType::Ref(row))]); - if procedure { - builder.add_procedure_with_visibility("run_job", params, AlgebraicType::unit(), visibility); - } else { - builder.add_reducer_with_visibility("run_job", params, visibility); - } - builder.add_schedule("Jobs", 1, "run_job"); - builder.finish().try_into().unwrap() - } - - #[test] - fn explicit_scheduled_visibility_overrides_the_private_default() { - for procedure in [false, true] { - for (selection, expected) in [ - (None, FunctionVisibility::Private), - (Some(Declared::Private), FunctionVisibility::Private), - (Some(Declared::Internal), FunctionVisibility::Internal), - (Some(Declared::ClientCallable), FunctionVisibility::ClientCallable), - ] { - let module = scheduled_module(selection, procedure); - let visibility = if procedure { - &module.procedure("run_job").unwrap().visibility - } else { - &module.reducer("run_job").unwrap().visibility - }; - assert_eq!(visibility, &expected); - assert_eq!(module.raw_module_def_version(), RawModuleDefVersion::V11); - } - } - } - - #[test] - fn ordinary_defaults_and_lifecycle_restrictions() { - let mut builder = RawModuleDefV11Builder::new(); - builder.add_reducer("ordinary", ProductType::unit()); - builder.add_procedure("ordinary_procedure", ProductType::unit(), AlgebraicType::unit()); - builder.add_lifecycle_reducer(v9::Lifecycle::Init, "initialize", ProductType::unit()); - let module: ModuleDef = builder.finish().try_into().unwrap(); - assert!(module.reducer("ordinary").unwrap().visibility.is_client_callable()); - assert!(module - .procedure("ordinary_procedure") - .unwrap() - .visibility - .is_client_callable()); - assert!(module.reducer("initialize").unwrap().visibility.is_internal()); - for selection in [Declared::Private, Declared::ClientCallable] { - let mut builder = RawModuleDefV11Builder::new(); - builder.add_lifecycle_reducer_with_visibility( - v9::Lifecycle::Init, - "initialize", - ProductType::unit(), - Some(selection), - ); - assert!(ModuleDef::try_from(builder.finish()) - .unwrap_err() - .to_string() - .contains("must have Internal visibility")); - } - let mut builder = RawModuleDefV11Builder::new(); - builder.add_lifecycle_reducer_with_visibility( - v9::Lifecycle::Init, - "initialize", - ProductType::unit(), - Some(Declared::Internal), - ); - assert!(ModuleDef::try_from(builder.finish()).is_ok()); - } - - #[test] - fn duplicate_definitions_sections_and_lifecycles_are_rejected() { - let mut builder = RawModuleDefV11Builder::new(); - builder.add_reducer("same", ProductType::unit()); - builder.add_procedure("same", ProductType::unit(), AlgebraicType::unit()); - assert!(ModuleDef::try_from(builder.finish()).is_err()); - let raw = v11::RawModuleDefV11 { - sections: vec![ - v11::RawModuleDefV11Section::Reducers(vec![]), - v11::RawModuleDefV11Section::Reducers(vec![]), - ], - }; - assert!(ModuleDef::try_from(raw) - .unwrap_err() - .to_string() - .contains("repeated V11 section")); - let mut builder = RawModuleDefV11Builder::new(); - builder.add_lifecycle_reducer(v9::Lifecycle::Init, "a", ProductType::unit()); - builder.add_lifecycle_reducer(v9::Lifecycle::Init, "b", ProductType::unit()); - assert!(ModuleDef::try_from(builder.finish()).is_err()); - } - - #[test] - fn resolved_v11_roundtrips_without_reapplying_defaults_and_rejects_legacy_exports() { - for selection in [None, Some(Declared::Internal), Some(Declared::ClientCallable)] { - let module = scheduled_module(selection, false); - assert!(v9::RawModuleDefV9::try_from(module.clone()).is_err()); - assert!(v10::RawModuleDefV10::try_from(module.clone()).is_err()); - let RawModuleDef::V11(raw) = module.clone().into_raw() else { - panic!("lost source version") - }; - assert!(raw.reducers().all(|reducer| reducer.declared_visibility.is_some())); - let bytes = spacetimedb_lib::bsatn::to_vec(&RawModuleDef::V11(raw)).unwrap(); - let roundtrip: RawModuleDef = spacetimedb_lib::bsatn::from_slice(&bytes).unwrap(); - let roundtrip: ModuleDef = roundtrip.try_into().unwrap(); - assert_eq!( - roundtrip.reducer("run_job").unwrap().visibility, - module.reducer("run_job").unwrap().visibility - ); - assert_eq!(roundtrip.raw_module_def_version(), RawModuleDefVersion::V11); - } - } - - #[test] - fn legacy_v9_schedules_stay_public_and_v10_schedules_stay_private() { - let mut builder = v9::RawModuleDefV9Builder::new(); - let at = builder.add_type::(); - let row = builder - .build_table_with_new_type( - "jobs", - ProductType::from([("id", AlgebraicType::U64), ("at", at)]), - true, - ) - .with_auto_inc_primary_key(0) - .with_index(v9::btree(0), "jobs_id_idx") - .with_schedule("run_job", 1) - .finish(); - builder.add_reducer("run_job", ProductType::from([("job", row.into())]), None); - let v9: ModuleDef = builder.finish().try_into().unwrap(); - assert!(v9.reducer("run_job").unwrap().visibility.is_client_callable()); - assert!(matches!(v9.into_raw(), RawModuleDef::V9(_))); - - let mut builder = v10::RawModuleDefV10Builder::new(); - let at = builder.add_type::(); - let row = builder - .build_table_with_new_type( - "jobs", - ProductType::from([("id", AlgebraicType::U64), ("at", at)]), - true, - ) - .with_auto_inc_primary_key(0) - .with_index_no_accessor_name(v9::btree(0), "jobs_id_idx") - .finish(); - builder.add_reducer("run_job", ProductType::from([("job", row.into())])); - builder.add_schedule("jobs", 1, "run_job"); - let v10: ModuleDef = builder.finish().try_into().unwrap(); - assert!(v10.reducer("run_job").unwrap().visibility.is_private()); - assert!(matches!(v10.into_raw(), RawModuleDef::V10(_))); - } - - #[test] - fn capabilities_are_explicit_bounded_and_preserved() { - let bare: ModuleDef = RawModuleDefV11Builder::new().finish().try_into().unwrap(); - assert!(!bare.supports_hosted_auth_v1()); - let mut builder = RawModuleDefV11Builder::new(); - builder.add_capability("hosted_auth_v1"); - let module: ModuleDef = builder.finish().try_into().unwrap(); - assert!(module.supports_hosted_auth_v1()); - let reloaded: ModuleDef = module.into_raw().try_into().unwrap(); - assert!(reloaded.supports_hosted_auth_v1()); - for names in [ - vec!["".to_string()], - vec!["Uppercase".to_string()], - vec!["with-dash".to_string()], - vec!["a".repeat(65)], - vec!["duplicate".to_string(); 2], - (0..33).map(|i| format!("cap_{i}")).collect(), - ] { - let mut builder = RawModuleDefV11Builder::new(); - for name in names { - builder.add_capability(name); - } - assert!(ModuleDef::try_from(builder.finish()).is_err()); - } - } - - #[test] - fn narrowing_function_visibility_is_a_reported_client_break() { - let module = |visibility| { - let mut builder = RawModuleDefV11Builder::new(); - builder.add_reducer_with_visibility("run_now", ProductType::unit(), Some(visibility)); - ModuleDef::try_from(builder.finish()).unwrap() - }; - let public = module(Declared::ClientCallable); - let internal = module(Declared::Internal); - let plan = crate::auto_migrate::ponder_migrate(&public, &internal).unwrap(); - assert!(plan.breaks_client()); - let display = plan - .pretty_print(crate::auto_migrate::PrettyPrintStyle::NoColor) - .unwrap(); - assert!(display.contains("run_now")); - assert!(display.contains("Internal")); - assert!(!crate::auto_migrate::ponder_migrate(&internal, &public) - .unwrap() - .breaks_client()); - } - - #[test] - fn visibility_authority_is_cumulative_without_elevating_the_owner() { - for (visibility, external, owner, internal) in [ - (FunctionVisibility::Internal, false, false, true), - (FunctionVisibility::Private, false, true, true), - (FunctionVisibility::ClientCallable, true, true, true), - ] { - assert_eq!(visibility.allows_invocation(false, false), external); - assert_eq!(visibility.allows_invocation(false, true), owner); - assert_eq!(visibility.allows_invocation(true, false), internal); - } - } -} diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 1ffe44196c9..5dfdcbe7f7e 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -28,7 +28,7 @@ pub enum ValidationError { InvalidModuleCapabilities, #[error("lifecycle reducer `{function}` must have Internal visibility")] InvalidLifecycleVisibility { function: RawIdentifier }, - #[error("module contains repeated V11 section `{section}`")] + #[error("module contains repeated V10 section `{section}`")] DuplicateModuleSection { section: String }, #[error("name `{name}` is used for multiple entities")] DuplicateName { name: RawIdentifier }, diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/smoketests/http_routes.rs index 62263da685c..567c3684ac8 100644 --- a/crates/smoketests/tests/smoketests/http_routes.rs +++ b/crates/smoketests/tests/smoketests/http_routes.rs @@ -1172,7 +1172,7 @@ fn assert_http_routes_end_to_end(server_url: &str, identity: &str) { assert_eq!(resp.text().expect("missing route body"), NO_SUCH_ROUTE_BODY); let resp = client - .get(format!("{server_url}/v1/database/{identity}/schema?version=11")) + .get(format!("{server_url}/v1/database/{identity}/schema?version=10")) .header("authorization", "Bearer not-a-jwt") .send() .expect("schema request failed"); diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index a361112b266..4388f8a41f1 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -510,7 +510,7 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; try { const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=11` + `http://localhost:3000/v1/database/${module_identity}/schema?version=10` ); return response.text(); } catch (e) { diff --git a/modules/module-test/src/lib.rs b/modules/module-test/src/lib.rs index 0932377b59d..3be87f37593 100644 --- a/modules/module-test/src/lib.rs +++ b/modules/module-test/src/lib.rs @@ -545,7 +545,7 @@ fn with_tx(ctx: &mut ProcedureContext) { fn get_my_schema_via_http(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=11" + "http://localhost:3000/v1/database/{module_identity}/schema?version=10" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => format!("{e}"), diff --git a/modules/sdk-test-procedure-ts/src/index.ts b/modules/sdk-test-procedure-ts/src/index.ts index 5ab74792193..e9e42163f17 100644 --- a/modules/sdk-test-procedure-ts/src/index.ts +++ b/modules/sdk-test-procedure-ts/src/index.ts @@ -93,7 +93,7 @@ export const will_panic = spacetimedb.procedure(t.unit(), _ctx => { export const read_my_schema = spacetimedb.procedure(t.string(), ctx => { const module_identity = ctx.databaseIdentity; const response = ctx.http.fetch( - `http://localhost:3000/v1/database/${module_identity}/schema?version=11` + `http://localhost:3000/v1/database/${module_identity}/schema?version=10` ); return response.text(); }); diff --git a/modules/sdk-test-procedure/src/lib.rs b/modules/sdk-test-procedure/src/lib.rs index a1c5a38efc3..71b6bd4fedd 100644 --- a/modules/sdk-test-procedure/src/lib.rs +++ b/modules/sdk-test-procedure/src/lib.rs @@ -44,7 +44,7 @@ fn will_panic(_ctx: &mut ProcedureContext) { fn read_my_schema(ctx: &mut ProcedureContext) -> String { let module_identity = ctx.identity(); match ctx.http.get(format!( - "http://localhost:3000/v1/database/{module_identity}/schema?version=11" + "http://localhost:3000/v1/database/{module_identity}/schema?version=10" )) { Ok(result) => result.into_body().into_string_lossy(), Err(e) => panic!("{e}"), diff --git a/sdks/rust/tests/procedure-client/src/test_handlers.rs b/sdks/rust/tests/procedure-client/src/test_handlers.rs index e280ddc2d90..2dda622bf21 100644 --- a/sdks/rust/tests/procedure-client/src/test_handlers.rs +++ b/sdks/rust/tests/procedure-client/src/test_handlers.rs @@ -1,7 +1,7 @@ use crate::module_bindings::*; use anyhow::Context; use core::time::Duration; -use spacetimedb_lib::db::raw_def::v11::{RawModuleDefV11Section, RawModuleDefV11}; +use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10, RawModuleDefV10Section}; use spacetimedb_sdk::{DbConnectionBuilder, DbContext, Table}; use test_counter::TestCounter; @@ -249,7 +249,7 @@ async fn exec_insert_with_tx_rollback(db_name: &str) { /// Test that a procedure can perform an HTTP request and return a string derived from the response. /// /// Invoke the procedure `read_my_schema`, -/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV11`], +/// which does an HTTP request to the `/database/schema` route and returns a JSON-ified [`RawModuleDefV10`], /// then (in the client) deserialize the response and assert that it contains a description of that procedure. async fn exec_procedure_http_ok(db_name: &str) { let test_counter = TestCounter::new(); @@ -263,12 +263,14 @@ async fn exec_procedure_http_ok(db_name: &str) { #[allow(clippy::redundant_closure_call)] (|| { anyhow::ensure!(res.is_ok(), "Expected Ok result but got {res:?}"); - let module_def: RawModuleDefV11 = spacetimedb_lib::de::serde::deserialize_from( + let module_def: RawModuleDefV10 = spacetimedb_lib::de::serde::deserialize_from( &mut serde_json::Deserializer::from_str(&res.unwrap()), )?; anyhow::ensure!(module_def.sections.iter().any(|section| { - if let RawModuleDefV11Section::Procedures(procedures) = section { - procedures.iter().any(|procedure| &*procedure.source_name == "read_my_schema") + if let RawModuleDefV10Section::Procedures(procedures) = section { + procedures + .iter() + .any(|procedure| &*procedure.source_name == "read_my_schema") } else { false } From b6410a7c0586efdafbf7237ffb2ee1e2d74a6df5 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Mon, 7 Sep 2026 23:22:27 -0400 Subject: [PATCH 04/23] Add transactional SQL management for database environment variables --- crates/client-api/src/lib.rs | 8 ++- crates/core/src/sql/execute.rs | 106 ++++++++++++++++++++++++++-- crates/expr/src/errors.rs | 4 ++ crates/expr/src/statement.rs | 21 ++++++ crates/query/src/lib.rs | 6 +- crates/sql-parser/src/ast/sql.rs | 9 +++ crates/sql-parser/src/parser/sql.rs | 35 ++++++++- crates/testing/tests/environment.rs | 54 ++++++++++---- 8 files changed, 215 insertions(+), 28 deletions(-) diff --git a/crates/client-api/src/lib.rs b/crates/client-api/src/lib.rs index 2495d01bc06..5a2793e0d17 100644 --- a/crates/client-api/src/lib.rs +++ b/crates/client-api/src/lib.rs @@ -155,7 +155,9 @@ impl Host { .await .map_err(|_| (StatusCode::NOT_FOUND, "module not found".to_string()))?; - tracing::info!(sql = body); + // Administrative SQL may contain environment values. The commitlog + // retains the table mutation, but routine request logs must not copy it. + tracing::info!(sql_bytes = body.len(), "executing SQL"); let mut header = vec![]; let sql_start = std::time::Instant::now(); let sql_span = tracing::trace_span!("execute_sql", total_duration = tracing::field::Empty,); @@ -173,7 +175,9 @@ impl Host { ) .await .map_err(|e| { - log::warn!("{e}"); + // Parser diagnostics can quote an environment value. Return the + // diagnostic to its caller without duplicating it in server logs. + log::warn!("SQL request rejected"); (StatusCode::BAD_REQUEST, e.to_string()) })?; diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 1e5fac1d8c8..6c89e6adc3e 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -96,6 +96,21 @@ fn run_inner( let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { check_hosted_admission(tx, db.database_identity(), auth.hosted.as_deref())?; let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; + // Check mutation authority while the automatic rollback guard owns + // the transaction, including rejected administrative statements. + if matches!(&stmt, Statement::DML(_) | Statement::Environment(_)) && !auth.has_write_access() { + return Err(anyhow!( + "Caller {} is not authorized to run SQL mutations", + auth.caller() + )); + } + if let Statement::DML(dml) = &stmt + && dml.table_id() == spacetimedb_datastore::system_tables::ST_ENV_ID + { + return Err(anyhow!( + "Use SET env.KEY or DELETE env.KEY to modify database environment variables" + )); + } if let Statement::DML(dml) = &stmt && spacetimedb_datastore::system_tables::is_host_managed_deployment_table(dml.table_id()) { @@ -156,14 +171,21 @@ fn run_inner( trapped, )) } - Statement::DML(stmt) => { - // An extra layer of auth is required for DML - if !auth.has_write_access() { - return Err(anyhow!("Caller {} is not authorized to run SQL DML statements", auth.caller()).into()); - } - + stmt @ (Statement::DML(_) | Statement::Environment(_)) => { // Evaluate the mutation - let (mut tx, _) = db.with_auto_rollback(tx, |tx| execute_dml_stmt(&auth, stmt, tx, &mut metrics))?; + let (mut tx, _) = db.with_auto_rollback(tx, |tx| -> anyhow::Result<()> { + match stmt { + Statement::DML(stmt) => execute_dml_stmt(&auth, stmt, tx, &mut metrics)?, + Statement::Environment(environment) => match environment.value { + Some(value) => crate::db::environment::set(&db, tx, &environment.key, &value)?, + None => { + crate::db::environment::delete(&db, tx, &environment.key)?; + } + }, + Statement::Select(_) => unreachable!(), + } + Ok(()) + })?; // Update transaction metrics tx.metrics.merge(metrics); @@ -258,6 +280,76 @@ pub(crate) mod tests { use spacetimedb_schema::schema::{ColumnSchema, TableSchema}; use spacetimedb_schema::table_name::TableName; + #[test] + fn environment_sql_enforces_permissions_escaping_limits_and_rollback() { + use spacetimedb_lib::identity::SqlPermission; + let db = TestDB::in_memory().unwrap(); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let owner = AuthCtx::for_current(Identity::ZERO); + let viewer = AuthCtx::with_permissions( + Identity::ONE, + Arc::new(|permission| matches!(permission, SqlPermission::Read(_))), + ); + let outsider = AuthCtx::new(Identity::ZERO, Identity::ONE); + let execute = |statement: &str, auth: AuthCtx| { + runtime.block_on(run(db.clone(), statement.to_string(), auth, None, None, &mut vec![])) + }; + let value = "it's \\quoted;\nUTF-8 é\0tail"; + execute( + &format!( + "/* prefix */ SET env.Mixed_Key = '{}'; -- suffix", + value.replace('\'', "''") + ), + owner.clone(), + ) + .unwrap(); + execute("SET env.EMPTY TO ''", owner.clone()).unwrap(); + let rows = execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", viewer.clone()) + .unwrap() + .rows; + assert_eq!(rows, vec![product![value]]); + assert!(execute("SELECT * FROM st_env", outsider.clone()).is_err()); + for auth in [viewer, outsider] { + assert!(execute("SET env.Mixed_Key = 'forbidden'", auth.clone()).is_err()); + assert!(execute("DELETE env.Mixed_Key", auth).is_err()); + } + for statement in [ + "SET env.EMPTY = 5".to_string(), + "SET env.\"BAD-KEY\" = 'value'".to_string(), + format!("SET env.EMPTY = '{}'", "x".repeat(8193)), + "SET env.EMPTY = 'changed'; DELETE env.Mixed_Key".to_string(), + "DELETE env.Mixed_Key WHERE true".to_string(), + "INSERT INTO st_env (key, value) VALUES ('BYPASS', 'value')".to_string(), + "UPDATE st_env SET value = 'bypass'".to_string(), + "DELETE FROM st_env".to_string(), + ] { + assert!( + execute(&statement, owner.clone()).is_err(), + "unexpectedly accepted {statement}" + ); + } + // Every rejected path released its transaction and preserved old data. + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner.clone()) + .unwrap() + .rows, + vec![product![""]] + ); + assert_eq!( + execute("SELECT value FROM st_env WHERE key = 'Mixed_Key'", owner.clone()) + .unwrap() + .rows, + vec![product![value]] + ); + execute("SET env.EMPTY = 'updated'", owner.clone()).unwrap(); + execute("DELETE env.EMPTY", owner.clone()).unwrap(); + execute("DELETE env.EMPTY", owner.clone()).unwrap(); + assert!(execute("SELECT value FROM st_env WHERE key = 'EMPTY'", owner) + .unwrap() + .rows + .is_empty()); + } + /// Short-cut for simplify test execution pub(crate) fn run_for_testing(db: &Arc, sql_text: &str) -> Result, DBError> { let (subs, runtime) = ModuleSubscriptions::for_test_new_runtime(db.clone()); diff --git a/crates/expr/src/errors.rs b/crates/expr/src/errors.rs index 489882ce8c8..9148ed64b7b 100644 --- a/crates/expr/src/errors.rs +++ b/crates/expr/src/errors.rs @@ -132,6 +132,10 @@ pub struct DmlOnView { #[derive(Error, Debug)] pub enum TypingError { + #[error(transparent)] + Environment(#[from] spacetimedb_lib::environment::EnvironmentValidationError), + #[error("environment values must be SQL string literals")] + EnvironmentValueType, #[error(transparent)] Unsupported(#[from] Unsupported), #[error(transparent)] diff --git a/crates/expr/src/statement.rs b/crates/expr/src/statement.rs index b7422dd031c..797f2c8fc7c 100644 --- a/crates/expr/src/statement.rs +++ b/crates/expr/src/statement.rs @@ -31,6 +31,12 @@ use super::{ pub enum Statement { Select(ProjectList), DML(DML), + Environment(EnvironmentWrite), +} + +pub struct EnvironmentWrite { + pub key: Box, + pub value: Option>, } pub enum DML { @@ -458,6 +464,21 @@ pub fn parse_and_type_sql(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Ty SqlAst::Update(update) => Ok(Statement::DML(DML::Update(type_update(update, tx)?))), SqlAst::Set(set) => Ok(Statement::DML(DML::Insert(type_and_rewrite_set(set, tx)?))), SqlAst::Show(show) => Ok(Statement::Select(type_and_rewrite_show(show, tx)?)), + SqlAst::Environment(environment) => { + // Resolve through the normal private-table visibility check. + tx.schema("st_env").ok_or_else(|| Unresolved::table("st_env"))?; + let key = &*environment.key.0; + spacetimedb_lib::environment::validate_key(key)?; + let value = match environment.value { + None => None, + Some(SqlLiteral::Str(value)) => { + spacetimedb_lib::environment::validate_value(&value)?; + Some(value) + } + Some(_) => return Err(TypingError::EnvironmentValueType), + }; + Ok(Statement::Environment(EnvironmentWrite { key: key.into(), value })) + } } } diff --git a/crates/query/src/lib.rs b/crates/query/src/lib.rs index 4a1bea08c9c..23ec20c0701 100644 --- a/crates/query/src/lib.rs +++ b/crates/query/src/lib.rs @@ -29,7 +29,7 @@ pub fn compile_subscription( auth: &AuthCtx, ) -> Result<(Vec, TableId, TableName, bool)> { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") + bail!("SQL query exceeds maximum allowed length") } let (plan, mut has_param) = parse_and_type_sub(sql, tx, auth)?; @@ -59,11 +59,11 @@ pub fn compile_subscription( /// A utility for parsing and type checking a sql statement pub fn compile_sql_stmt(sql: &str, tx: &impl SchemaView, auth: &AuthCtx) -> Result { if sql.len() > MAX_SQL_LENGTH { - bail!("SQL query exceeds maximum allowed length: \"{sql:.120}...\"") + bail!("SQL query exceeds maximum allowed length") } match parse_and_type_sql(sql, tx, auth)? { - stmt @ Statement::DML(_) => Ok(stmt), + stmt @ (Statement::DML(_) | Statement::Environment(_)) => Ok(stmt), Statement::Select(expr) => Ok(Statement::Select(resolve_views_for_sql(tx, expr, auth)?)), } } diff --git a/crates/sql-parser/src/ast/sql.rs b/crates/sql-parser/src/ast/sql.rs index be7b753f395..a7593cc4582 100644 --- a/crates/sql-parser/src/ast/sql.rs +++ b/crates/sql-parser/src/ast/sql.rs @@ -19,6 +19,15 @@ pub enum SqlAst { Set(SqlSet), /// SHOW var Show(SqlShow), + /// Administrative environment mutation, distinct from generic table DML. + Environment(SqlEnvironment), +} + +#[derive(Debug)] +pub struct SqlEnvironment { + pub key: SqlIdent, + /// None is DELETE; a string literal, including empty, is SET. + pub value: Option, } impl SqlAst { diff --git a/crates/sql-parser/src/parser/sql.rs b/crates/sql-parser/src/parser/sql.rs index e689817cc4c..762a56e3b80 100644 --- a/crates/sql-parser/src/parser/sql.rs +++ b/crates/sql-parser/src/parser/sql.rs @@ -133,11 +133,13 @@ use sqlparser::{ Value, Values, }, dialect::PostgreSqlDialect, + keywords::Keyword, parser::Parser, + tokenizer::Token, }; use crate::ast::{ - sql::{SqlAst, SqlDelete, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, + sql::{SqlAst, SqlDelete, SqlEnvironment, SqlInsert, SqlSelect, SqlSet, SqlShow, SqlUpdate, SqlValues}, SqlIdent, }; @@ -148,7 +150,36 @@ use super::{ /// Parse a SQL string pub fn parse_sql(sql: &str) -> SqlParseResult { - let mut stmts = Parser::parse_sql(&PostgreSqlDialect {}, sql)?; + // DELETE env.KEY is a SpacetimeDB administrative statement, not the + // PostgreSQL DELETE FROM grammar. Use the same tokenizer and expression + // parser, including comments and SQL string escaping, for this extension. + let mut parser = Parser::new(&PostgreSqlDialect {}).try_with_sql(sql)?; + let verb = parser.peek_token().token; + let environment_prefix = matches!(parser.peek_nth_token(1).token, + Token::Word(word) if word.quote_style.is_none() && word.value.eq_ignore_ascii_case("env")) + && parser.peek_nth_token(2).token == Token::Period; + if environment_prefix + && matches!(&verb, Token::Word(word) if matches!(word.keyword, Keyword::SET | Keyword::DELETE)) + { + parser.next_token(); + parser.next_token(); + parser.next_token(); + let key = SqlIdent(parser.parse_identifier()?.value.into()); + let value = if matches!(verb, Token::Word(word) if word.keyword == Keyword::SET) { + if !parser.parse_keyword(Keyword::TO) { + parser.expect_token(&Token::Eq)?; + } + Some(parse_literal_expr(parser.parse_expr()?, SqlUnsupported::Assignment)?) + } else { + None + }; + let _ = parser.consume_token(&Token::SemiColon); + if parser.peek_token().token != Token::EOF { + return Err(SqlUnsupported::MultiStatement.into()); + } + return Ok(SqlAst::Environment(SqlEnvironment { key, value })); + } + let mut stmts = parser.parse_statements()?; if stmts.len() > 1 { return Err(SqlUnsupported::MultiStatement.into()); } diff --git a/crates/testing/tests/environment.rs b/crates/testing/tests/environment.rs index cb0086bdfaf..afa5c8f6f34 100644 --- a/crates/testing/tests/environment.rs +++ b/crates/testing/tests/environment.rs @@ -1,15 +1,31 @@ //! Actual module calls exercise environment ABI, bindings, and snapshot semantics. use serial_test::serial; -use spacetimedb::db::environment; -use spacetimedb::host::FunctionArgs; -use spacetimedb_datastore::execution_context::Workload; +use spacetimedb::host::{FunctionArgs, ModuleHost}; +use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::{bsatn, sats::product, AlgebraicValue, Identity}; use spacetimedb_testing::modules::{CompilationMode, CompiledModule, DEFAULT_CONFIG}; +async fn sql(module: &ModuleHost, statement: String) -> Vec { + spacetimedb::sql::execute::run( + module.relational_db().clone(), + statement, + AuthCtx::for_current(Identity::ZERO), + Some(module.info.subscriptions.clone()), + Some(module.clone()), + &mut vec![], + ) + .await + .unwrap() + .rows +} + +async fn set_environment(module: &ModuleHost, key: &str, value: &str) { + sql(module, format!("SET env.{key} = '{}'", value.replace('\'', "''"))).await; +} + fn exercise_fixture(name: &str) { CompiledModule::compile(name, CompilationMode::Debug).with_module_async(DEFAULT_CONFIG, |handle| async move { let module = handle.client.module(); - let db = module.relational_db(); for (key, expected) in [ ("MISSING", None), ("EMPTY", Some("".to_string())), @@ -18,8 +34,7 @@ fn exercise_fixture(name: &str) { ("MAXIMUM", Some("é".repeat(4096))), ] { if let Some(value) = &expected { - db.with_auto_commit(Workload::Internal, |tx| environment::set(db, tx, key, value)) - .unwrap(); + set_environment(&module, key, value).await; } let args = product![key, expected.clone()]; let result = module @@ -50,8 +65,7 @@ fn exercise_fixture(name: &str) { .return_val; assert_eq!(result, AlgebraicValue::from(expected.clone())); if expected.is_some() { - db.with_auto_commit(Workload::Internal, |tx| environment::set(db, tx, key, "updated")) - .unwrap(); + set_environment(&module, key, "updated").await; let result = module .call_procedure(Identity::ZERO, None, None, "read_environment", read()) .await @@ -59,8 +73,7 @@ fn exercise_fixture(name: &str) { .unwrap() .return_val; assert_eq!(result, AlgebraicValue::from(Some("updated".to_string()))); - db.with_auto_commit(Workload::Internal, |tx| environment::delete(db, tx, key)) - .unwrap(); + sql(&module, format!("DELETE env.{key}")).await; let result = module .call_procedure(Identity::ZERO, None, None, "read_environment", read()) .await @@ -71,10 +84,23 @@ fn exercise_fixture(name: &str) { } } if name == "environment-test" { - db.with_auto_commit(Workload::Internal, |tx| { - environment::set(db, tx, "LIMIT", &"x".repeat(8192)) - }) - .unwrap(); + // This view first reads a missing key. Its dependency must survive + // absence, and normal SQL mutations must invalidate its cached row. + let read_view = || "SELECT * FROM environment_value".to_string(); + assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); + set_environment(&module, "WATCHED", "first").await; + assert_eq!( + sql(&module, read_view()).await, + vec![product![Some("first".to_string())]] + ); + set_environment(&module, "WATCHED", "second").await; + assert_eq!( + sql(&module, read_view()).await, + vec![product![Some("second".to_string())]] + ); + sql(&module, "DELETE env.WATCHED".into()).await; + assert_eq!(sql(&module, read_view()).await, vec![product![None::]]); + set_environment(&module, "LIMIT", &"x".repeat(8192)).await; for _ in 0..2 { module .call_reducer( From 1ebeeb9e057cc40158e42a3c7e661078de0eabc6 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 00:33:00 -0400 Subject: [PATCH 05/23] Make container publication recovery preserve durable outcomes --- crates/core/src/db/deployment.rs | 133 +++++++++- crates/core/src/db/deployment/tests.rs | 248 ++++++++++++++++++ crates/core/src/error.rs | 8 +- crates/core/src/host/module_common.rs | 2 +- .../src/host/wasm_common/module_host_actor.rs | 2 +- .../module_subscription_manager.rs | 8 +- crates/lib/src/deployment.rs | 2 + crates/lib/src/deployment/manifest.rs | 172 ++++++++++++ crates/oci/src/layers.rs | 31 ++- crates/oci/src/lib.rs | 16 +- 10 files changed, 593 insertions(+), 29 deletions(-) create mode 100644 crates/lib/src/deployment/manifest.rs diff --git a/crates/core/src/db/deployment.rs b/crates/core/src/db/deployment.rs index 5b6285577b8..53d0e800b77 100644 --- a/crates/core/src/db/deployment.rs +++ b/crates/core/src/db/deployment.rs @@ -92,6 +92,14 @@ pub enum CommitAdmission { Ready, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AbortResult { + /// The commit won the race. Recovery must converge on this deployment. + AlreadyCommitted(PublishResult), + /// This epoch can no longer admit a commit, and the prior revision remains. + Aborted { previous_revision: Option }, +} + /// Check the actual program selected by the host, not a caller-provided /// capability bit. The program bytes are hashed here because `Program` also /// has a public constructor which accepts a previously computed hash. @@ -176,7 +184,7 @@ pub fn install_publication_fence( publication_epoch: u64, operation_id: Uuid, ) -> Result<(), DeploymentError> { - if publication_epoch == 0 { + if publication_epoch == 0 || operation_id == Uuid::NIL { return Err(DeploymentError::PublicationFenced); } let next = StPublishFenceRow { @@ -200,12 +208,75 @@ pub fn install_publication_fence( Ok(()) } +/// Close an admitted publication before reporting an abort to control. Run in +/// one serializable transaction and await its durability before releasing the +/// control operation or resuming the previous container under a fresh generation. +/// A delayed commit and this transaction serialize on the same database fence. +/// +/// The nil operation ID is a closed-epoch marker, never a publish operation. +/// Keeping the epoch makes closure irreversible at that epoch while allowing +/// the next control-allocated epoch to install its own operation normally. +pub fn abort_deployment_commit(tx: &mut MutTx, request: &DeploymentCommit) -> Result { + if let Some(result) = committed_deployment_operation(tx, request)? { + return Ok(AbortResult::AlreadyCommitted(result)); + } + // Recovery must be able to close an expired operation too. Its expiry + // prevents new commits, but cannot substitute for a durable abort fence. + // Control owns the immutable epoch-to-operation mapping; authenticate and + // resolve that recorded operation before calling this function. + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + let fence = fence + .filter(|row| { + row.publication_epoch == request.publication_epoch + && (row.operation_id == request.operation_id.as_u128() || row.operation_id == 0) + }) + .ok_or(DeploymentError::PublicationFenced)?; + if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + if fence.operation_id == 0 { + return Ok(AbortResult::Aborted { + previous_revision: request.expected_revision, + }); + } + tx.clear_table(ST_PUBLISH_FENCE_ID)?; + tx.insert_via_serialize_bsatn( + ST_PUBLISH_FENCE_ID, + &StPublishFenceRow { + key: 0, + publication_epoch: request.publication_epoch, + operation_id: 0, + }, + )?; + Ok(AbortResult::Aborted { + previous_revision: request.expected_revision, + }) +} + fn normalized_request( request: &DeploymentCommit, limits: &ContainerSpecLimits, ) -> Result<(DeploymentSpec, Hash, Hash), DeploymentError> { let spec = request.deployment.clone().normalize(limits)?; - let revision = spec.revision()?; + if spec != request.deployment { + return Err(DeploymentValidationError::InvalidEncoding.into()); + } + let (revision, request_hash) = request_identity(request)?; + Ok((spec, revision, request_hash)) +} + +fn request_identity(request: &DeploymentCommit) -> Result<(Hash, Hash), DeploymentError> { + if request.operation_id.get_version() != Some(spacetimedb_sats::uuid::Version::V7) { + return Err(DeploymentValidationError::InvalidOperationId.into()); + } + if request.publication_epoch == 0 { + return Err(DeploymentError::PublicationFenced); + } + // These bytes were normalized at admission. Do not re-apply today's + // resource eligibility when inspecting yesterday's committed outcome. + let revision = request.deployment.revision()?; let mut bytes = b"spacetimedb/deployment-operation\0".to_vec(); bytes.extend_from_slice( &bsatn::to_vec(&( @@ -218,7 +289,7 @@ fn normalized_request( )) .map_err(|_| DeploymentError::CorruptMetadata)?, ); - Ok((spec, revision, hash_bytes(bytes))) + Ok((revision, hash_bytes(bytes))) } /// Call before module execution, while holding the transaction later used for @@ -231,9 +302,36 @@ pub fn check_deployment_commit( ) -> Result { let now_ms = u64::try_from(now.to_micros_since_unix_epoch()).map_err(|_| DeploymentError::CorruptMetadata)? / 1000; operation_expiry_ms(request.operation_id, now_ms)?; - let (_, revision, request_hash) = normalized_request(request, limits)?; + if let Some(result) = committed_deployment_operation(tx, request)? { + return Ok(CommitAdmission::AlreadyCommitted(result)); + } + normalized_request(request, limits)?; + let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + .map(StPublishFenceRow::try_from) + .transpose()?; + if !fence.is_some_and(|f| { + f.publication_epoch == request.publication_epoch && f.operation_id == request.operation_id.as_u128() + }) { + return Err(DeploymentError::PublicationFenced); + } + if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { + return Err(DeploymentError::RevisionConflict); + } + Ok(CommitAdmission::Ready) +} + +/// Inspect the exact retained commit receipt during host recovery. Unlike +/// admitting a client retry, inspecting an existing outcome does not expire. +/// This never authorizes module execution. An absent receipt is not proof of +/// abort: close the epoch atomically before reporting an abort to control. +/// Retain active operations' receipts until their control recovery completes. +pub fn committed_deployment_operation( + state: &S, + request: &DeploymentCommit, +) -> Result, DeploymentError> { + let (revision, request_hash) = request_identity(request)?; let operation_key = AlgebraicValue::U128(request.operation_id.as_u128().into()); - if let Some(row) = tx + if let Some(row) = state .iter_by_col_eq(ST_DEPLOYMENT_OPERATION_ID, ColId(0), &operation_key)? .next() { @@ -250,20 +348,29 @@ pub fn check_deployment_commit( { return Err(DeploymentError::CorruptMetadata); } - return Ok(CommitAdmission::AlreadyCommitted(receipt.result)); + return Ok(Some(receipt.result)); } - let fence = singleton(tx, ST_PUBLISH_FENCE_ID)? + Ok(None) +} + +/// Recognize the durable closed marker when recovering a control operation +/// whose abort report was lost. The authenticated caller must resolve control's +/// immutable epoch-to-operation binding before using this host-only API. +pub fn deployment_publication_aborted( + state: &S, + request: &DeploymentCommit, +) -> Result { + request_identity(request)?; + let fence = singleton(state, ST_PUBLISH_FENCE_ID)? .map(StPublishFenceRow::try_from) .transpose()?; - if !fence.is_some_and(|f| { - f.publication_epoch == request.publication_epoch && f.operation_id == request.operation_id.as_u128() - }) { - return Err(DeploymentError::PublicationFenced); + if !fence.is_some_and(|row| row.publication_epoch == request.publication_epoch && row.operation_id == 0) { + return Ok(false); } - if current_deployment(tx)?.map(|(revision, _)| revision) != request.expected_revision { + if current_deployment(state)?.map(|(revision, _)| revision) != request.expected_revision { return Err(DeploymentError::RevisionConflict); } - Ok(CommitAdmission::Ready) + Ok(true) } /// Record after successful module initialization/migration in that same diff --git a/crates/core/src/db/deployment/tests.rs b/crates/core/src/db/deployment/tests.rs index 130a4368093..c027d017136 100644 --- a/crates/core/src/db/deployment/tests.rs +++ b/crates/core/src/db/deployment/tests.rs @@ -137,6 +137,254 @@ fn deployment_and_module_effects_roll_back_together() { .unwrap(); } +#[test] +fn publication_abort_closes_delayed_commits_and_cannot_reopen_its_epoch() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id) + }) + .unwrap(); + let aborted = AbortResult::Aborted { + previous_revision: None, + }; + assert_eq!( + transact(&db, |tx| abort_deployment_commit(tx, &first)).unwrap(), + aborted + ); + transact(&db, |tx| { + assert_eq!(abort_deployment_commit(tx, &first)?, aborted); + assert!(matches!( + record_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch + 1, Uuid::NIL), + Err(DeploymentError::PublicationFenced) + )); + assert!(current_deployment(tx)?.is_none()); + let second = request(2, None); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + let result = record_deployment_commit(tx, &second, now(), &limits)?; + assert!(matches!( + abort_deployment_commit(tx, &first), + Err(DeploymentError::PublicationFenced) + )); + assert_eq!(current_deployment(tx)?.unwrap().0, result.revision); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_commit_winning_abort_race_is_never_reported_as_aborted() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let second = request(2, Some(committed.revision)); + transact(&db, |tx| { + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed.clone()) + ); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits)?; + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed) + ); + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits)?, + CommitAdmission::AlreadyCommitted(_) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_rollback_does_not_report_a_closed_fence() { + let db = TestDB::in_memory().unwrap(); + let request = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, request.publication_epoch, request.operation_id) + }) + .unwrap(); + let failed: Result<(), DeploymentError> = transact(&db, |tx| { + abort_deployment_commit(tx, &request)?; + Err(DeploymentError::CorruptMetadata) + }); + assert!(failed.is_err()); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &request, now(), &limits)?, + CommitAdmission::Ready + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn recovery_of_expired_publication_preserves_commits_and_closes_uncommitted_epochs() { + let db = TestDB::in_memory().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &limits) + }) + .unwrap(); + let expired = Timestamp::from_micros_since_unix_epoch( + now().to_micros_since_unix_epoch() + (PUBLISH_RETRY_WINDOW_MS * 1000) as i64, + ); + let second = request(2, Some(committed.revision)); + transact(&db, |tx| { + assert!(matches!( + check_deployment_commit(tx, &first, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::AlreadyCommitted(committed.clone()) + ); + let mut wrong_publisher = first.clone(); + wrong_publisher.publisher = Identity::ZERO; + assert!(matches!( + committed_deployment_operation(tx, &wrong_publisher), + Err(DeploymentError::OperationConflict) + )); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + assert!(matches!( + check_deployment_commit(tx, &second, expired, &limits), + Err(DeploymentError::Validation(DeploymentValidationError::ExpiredOperation)) + )); + assert_eq!( + abort_deployment_commit(tx, &second)?, + AbortResult::Aborted { + previous_revision: Some(committed.revision) + } + ); + // Clock rollback cannot make a delayed old commit admissible again. + assert!(matches!( + check_deployment_commit(tx, &second, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn publication_abort_and_commit_receipts_survive_commitlog_replay() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let first = request(1, None); + let limits = ContainerSpecLimits::default(); + transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + abort_deployment_commit(tx, &first) + }) + .unwrap(); + let db = db.reopen().unwrap(); + let second = request(2, None); + let committed = transact(&db, |tx| { + assert_eq!( + abort_deployment_commit(tx, &first)?, + AbortResult::Aborted { + previous_revision: None + } + ); + assert!(matches!( + record_deployment_commit(tx, &first, now(), &limits), + Err(DeploymentError::PublicationFenced) + )); + assert!(matches!( + install_publication_fence(tx, first.publication_epoch, first.operation_id), + Err(DeploymentError::PublicationFenced) + )); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + record_deployment_commit(tx, &second, now(), &limits) + }) + .unwrap(); + let db = db.reopen().unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + assert_eq!( + committed_deployment_operation(tx, &second).unwrap(), + Some(committed.clone()) + ); + assert_eq!(current_deployment(tx).unwrap().unwrap().0, committed.revision); + }); + assert_eq!( + transact(&db, |tx| abort_deployment_commit(tx, &second)).unwrap(), + AbortResult::AlreadyCommitted(committed) + ); +} + +#[test] +fn publication_recovery_does_not_reapply_tightened_resource_admission() { + use spacetimedb_lib::container::{ + ContainerMode, ContainerResources, ContainerSpec, ImagePlatform, OciDigest, RestartPolicy, + }; + let db = TestDB::in_memory().unwrap(); + let mut first = request(1, None); + let DeploymentSpec::V1(spec) = &mut first.deployment; + spec.container = Some(ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/server".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Service, + restart: RestartPolicy::OnFailure, + env_keys: vec![], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: 10_000, + }); + let original_limits = ContainerSpecLimits::default(); + let committed = transact(&db, |tx| { + install_publication_fence(tx, first.publication_epoch, first.operation_id)?; + record_deployment_commit(tx, &first, now(), &original_limits) + }) + .unwrap(); + let mut tightened = original_limits; + tightened.resources.cpu_millicores = 500; + let mut second = request(2, Some(committed.revision)); + second.deployment = first.deployment.clone(); + transact(&db, |tx| { + assert!(first.deployment.clone().normalize(&tightened).is_err()); + assert!(matches!(check_deployment_commit(tx, &first, now(), &tightened)?, CommitAdmission::AlreadyCommitted(ref result) if result == &committed)); + assert_eq!(committed_deployment_operation(tx, &first)?, Some(committed.clone())); + assert_eq!(abort_deployment_commit(tx, &first)?, AbortResult::AlreadyCommitted(committed.clone())); + install_publication_fence(tx, second.publication_epoch, second.operation_id)?; + assert!(check_deployment_commit(tx, &second, now(), &tightened).is_err()); + assert_eq!(abort_deployment_commit(tx, &second)?, AbortResult::Aborted { previous_revision: Some(committed.revision) }); + assert!(deployment_publication_aborted(tx, &second)?); + Ok(()) + }).unwrap(); +} + #[test] fn deployment_operation_cannot_be_reused_by_another_publisher_or_changed_request() { let db = TestDB::in_memory().unwrap(); diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 596c30b2c68..2a6c328f276 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -132,7 +132,7 @@ pub enum DBError { #[error("Error reading a value from a table through BSATN: {0}")] ReadViaBsatnError(#[from] ReadViaBsatnError), #[error("Module validation errors: {0}")] - ModuleValidationErrors(#[from] ValidationErrors), + ModuleValidationErrors(#[from] Box), #[error(transparent)] Other(#[from] anyhow::Error), #[error(transparent)] @@ -151,6 +151,12 @@ pub enum DBError { View(#[from] ViewCallError), } +impl From for DBError { + fn from(errors: ValidationErrors) -> Self { + Self::ModuleValidationErrors(Box::new(errors)) + } +} + impl From for DBError { fn from(value: InvalidFieldError) -> Self { LibError::from(value).into() diff --git a/crates/core/src/host/module_common.rs b/crates/core/src/host/module_common.rs index bff864759f3..ebe857c5cb1 100644 --- a/crates/core/src/host/module_common.rs +++ b/crates/core/src/host/module_common.rs @@ -19,7 +19,7 @@ use std::sync::Arc; pub fn build_common_module_from_raw( mcc: ModuleCreationContext, raw_def: RawModuleDef, -) -> Result { +) -> Result> { // Perform a bunch of validation on the raw definition. let def: ModuleDef = raw_def.try_into()?; diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index a33c6660c8f..736965b8710 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -336,7 +336,7 @@ pub enum InitializationError { #[error(transparent)] Validation(#[from] ValidationError), #[error(transparent)] - ModuleValidation(#[from] spacetimedb_schema::error::ValidationErrors), + ModuleValidation(#[from] Box), #[error("setup function returned an error: {0}")] Setup(Box), #[error("wasm trap while calling {func:?}")] diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index 3a053a2d020..98710d5fc38 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -1776,7 +1776,13 @@ pub struct BroadcastQueue(SenderWithGauge); #[derive(thiserror::Error, Debug)] #[error(transparent)] -pub struct BroadcastError(#[from] mpsc::error::SendError); +pub struct BroadcastError(Box>); + +impl From> for BroadcastError { + fn from(error: mpsc::error::SendError) -> Self { + Self(Box::new(error)) + } +} impl BroadcastQueue { fn send(&self, message: SendWorkerMessage) -> Result<(), BroadcastError> { diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs index 43435a9e1c8..39d76e0d865 100644 --- a/crates/lib/src/deployment.rs +++ b/crates/lib/src/deployment.rs @@ -6,6 +6,8 @@ use crate::container::{ContainerAction, ContainerSpec, ContainerSpecLimits, ContainerValidationError}; use crate::{bsatn, hash_bytes, Hash, SpacetimeType, Uuid}; +pub mod manifest; + pub const PUBLISH_PROTOCOL_VERSION: u32 = 1; pub const SYSTEM_EMPTY_MODULE_VERSION: u32 = 1; /// Immutable Keccak-256 program identity of the version-1 bundled empty Wasm diff --git a/crates/lib/src/deployment/manifest.rs b/crates/lib/src/deployment/manifest.rs new file mode 100644 index 00000000000..0727643b0c1 --- /dev/null +++ b/crates/lib/src/deployment/manifest.rs @@ -0,0 +1,172 @@ +//! Immutable publication recovery inputs. The manifest's SHA-256 artifact +//! digest binds migration intent as well as the effective deployment. Its +//! digest differs from the deployment revision, which excludes migration policy. + +use super::{DeploymentSpec, DeploymentValidationError, PublishEnvelope, MAX_DEPLOYMENT_BYTES}; +use crate::container::{ContainerSpecLimits, OciDigest}; +use crate::{bsatn, Hash, SpacetimeType}; + +pub const MAX_MODULE_ARTIFACT_BYTES: u64 = 32 * 1024 * 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct ModuleArtifact { + /// SHA-256 of the complete stored bytes, distinct from the module's Keccak hash. + pub digest: OciDigest, + pub size_bytes: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde(tag = "policy", content = "token", rename_all = "snake_case", deny_unknown_fields) +)] +pub enum PreparedMigrationPolicy { + Compatible, + /// The existing migration token binds database Identity and old/new module + /// hashes. Recovery must retain the originally acknowledged policy. + BreakClients(Hash), +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct PreparedDeploymentManifestV1 { + pub envelope: PublishEnvelope, + pub deployment: DeploymentSpec, + /// Always retained, including for the bundled empty module. Genesis and + /// later recovery must select exactly the admitted program bytes. + pub module_artifact: ModuleArtifact, + pub migration_policy: PreparedMigrationPolicy, +} + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr( + feature = "serde", + serde( + tag = "version", + content = "manifest", + rename_all = "snake_case", + deny_unknown_fields + ) +)] +pub enum PreparedDeploymentManifest { + V1(PreparedDeploymentManifestV1), +} + +impl PreparedDeploymentManifest { + pub fn current(&self) -> &PreparedDeploymentManifestV1 { + match self { + Self::V1(manifest) => manifest, + } + } + + /// Validate before retaining the artifact. This checks the encoding and + /// metadata; the artifact service verifies SHA-256/length and the host + /// validates the selected program, capabilities and actual migration. + pub fn validate(&self, limits: &ContainerSpecLimits) -> Result<(), DeploymentValidationError> { + let manifest = self.current(); + if manifest.module_artifact.size_bytes == 0 || manifest.module_artifact.size_bytes > MAX_MODULE_ARTIFACT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + if manifest.deployment.clone().normalize(limits)? != manifest.deployment { + return Err(DeploymentValidationError::InvalidEncoding); + } + // Using the prepared components as the Keep baseline verifies every + // explicit Set/Remove action without inventing a prior deployment. + // The host separately checks the real prior state under its fence. + if manifest.envelope.resolve(Some(&manifest.deployment), limits)? != manifest.deployment { + return Err(DeploymentValidationError::InvalidEncoding); + } + self.encode()?; + Ok(()) + } + + pub fn encode(&self) -> Result, DeploymentValidationError> { + let bytes = bsatn::to_vec(self).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + Ok(bytes.into()) + } + + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DEPLOYMENT_BYTES { + return Err(DeploymentValidationError::TooLarge); + } + let manifest: Self = bsatn::from_slice(bytes).map_err(|_| DeploymentValidationError::InvalidEncoding)?; + // Reject trailing or noncanonical bytes even if the decoder accepts + // them, so every retained descriptor names one unambiguous manifest. + if manifest.encode()?.as_ref() != bytes { + return Err(DeploymentValidationError::InvalidEncoding); + } + Ok(manifest) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::deployment::{DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; + + fn manifest() -> PreparedDeploymentManifest { + PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope: PublishEnvelope { + version: super::super::PUBLISH_PROTOCOL_VERSION, + operation_id: crate::Uuid::from_u128(0x01991ec4000070008000000000000001), + expected_revision: None, + module_action: super::super::ModuleAction::Keep, + container_action: crate::container::ContainerAction::Keep, + }, + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: None, + }), + module_artifact: ModuleArtifact { + digest: OciDigest::sha256([17; 32]), + size_bytes: 250, + }, + migration_policy: PreparedMigrationPolicy::Compatible, + }) + } + + #[test] + fn retained_manifest_preserves_migration_intent_without_changing_revision() { + let first = manifest(); + let mut acknowledged = first.clone(); + let PreparedDeploymentManifest::V1(value) = &mut acknowledged; + value.migration_policy = PreparedMigrationPolicy::BreakClients(Hash::from_byte_array([32; 32])); + assert_eq!( + first.current().deployment.revision().unwrap(), + acknowledged.current().deployment.revision().unwrap() + ); + let encoded = acknowledged.encode().unwrap(); + assert_ne!(first.encode().unwrap(), encoded); + assert_eq!(PreparedDeploymentManifest::decode(&encoded).unwrap(), acknowledged); + acknowledged.validate(&Default::default()).unwrap(); + } + + #[test] + fn retained_manifest_rejects_unknown_encoding_and_invalid_module_bounds() { + let mut value = manifest(); + let mut trailing = value.encode().unwrap().to_vec(); + trailing.push(0); + assert!(PreparedDeploymentManifest::decode(&trailing).is_err()); + let mut unknown = value.encode().unwrap().to_vec(); + unknown[0] = 1; + assert!(PreparedDeploymentManifest::decode(&unknown).is_err()); + let PreparedDeploymentManifest::V1(inner) = &mut value; + inner.module_artifact.size_bytes = 0; + assert!(value.validate(&Default::default()).is_err()); + let PreparedDeploymentManifest::V1(inner) = &mut value; + inner.module_artifact.size_bytes = MAX_MODULE_ARTIFACT_BYTES + 1; + assert!(value.validate(&Default::default()).is_err()); + } +} diff --git a/crates/oci/src/layers.rs b/crates/oci/src/layers.rs index 4e0d0a0f537..c7d99cb39de 100644 --- a/crates/oci/src/layers.rs +++ b/crates/oci/src/layers.rs @@ -4,7 +4,7 @@ use crate::{Descriptor, OciDigest}; use anyhow::{bail, ensure, Context, Result}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ collections::BTreeMap, @@ -40,7 +40,7 @@ pub struct VerifiedLayerSize { pub regular_file_bytes: u64, } -#[derive(Clone, Copy, Debug, Serialize)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub struct VerifiedImageSize { pub compressed_bytes: u64, pub uncompressed_tar_bytes: u64, @@ -117,6 +117,29 @@ pub fn verify_layer( diff_id: OciDigest, limits: LayerLimits, ) -> Result { + verify_layer_with_check(reader, descriptor, diff_id, limits, || Ok(())) +} + +/// Check cancellation/deadline between both compressed and expanded reads. +/// The caller retains its worker admission until this synchronous operation ends. +pub fn verify_layer_with_check( + reader: impl Read, + descriptor: &Descriptor, + diff_id: OciDigest, + limits: LayerLimits, + check: impl Fn() -> io::Result<()> + Copy, +) -> Result { + struct Checked { + reader: R, + check: F, + } + impl io::Result<()>> Read for Checked { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + (self.check)()?; + self.reader.read(buf) + } + } + check()?; ensure!( descriptor.size > 0 && descriptor.size <= crate::MAX_IMAGE_BYTES, "invalid compressed layer size" @@ -126,7 +149,7 @@ pub fn verify_layer( "external layer sources are unsupported" ); let mut compressed = HashBounded { - inner: reader, + inner: Checked { reader, check }, hash: Sha256::new(), count: 0, limit: descriptor.size, @@ -146,7 +169,7 @@ pub fn verify_layer( _ => bail!("unsupported or foreign layer media type"), }; let mut expanded = HashBounded { - inner: decoder, + inner: Checked { reader: decoder, check }, hash: Sha256::new(), count: 0, limit: limits.max_uncompressed_bytes, diff --git a/crates/oci/src/lib.rs b/crates/oci/src/lib.rs index c6e9d04c29d..aa1bc63f19b 100644 --- a/crates/oci/src/lib.rs +++ b/crates/oci/src/lib.rs @@ -57,14 +57,14 @@ impl Platform { && self.architecture == requested.architecture && self.os_version.as_deref().is_none_or(str::is_empty) && self.os_features.is_empty() - && match (self.architecture.as_str(), self.variant.as_deref()) { - (_, None | Some("")) | ("arm64", Some("v8")) => true, - _ => false, - } + && matches!( + (self.architecture.as_str(), self.variant.as_deref()), + (_, None | Some("")) | ("arm64", Some("v8")) + ) } } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Manifest { pub schema_version: u32, @@ -196,7 +196,7 @@ fn validate_platform(platform: &ImagePlatform) -> Result<()> { Ok(()) } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct ImageConfig { pub architecture: String, pub os: String, @@ -207,7 +207,7 @@ pub struct ImageConfig { pub rootfs: RootFs, } -#[derive(Clone, Default, Deserialize)] +#[derive(Clone, Default, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub struct ContainerConfig { #[serde(default)] @@ -244,7 +244,7 @@ impl std::fmt::Debug for ContainerConfig { } } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct RootFs { #[serde(rename = "type")] pub kind: String, From 06a0d3498128e0c4e5db00f5a2d2dae8d9bd8d54 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 01:20:21 -0400 Subject: [PATCH 06/23] Add publication API messages and Rust UUID index generation --- crates/codegen/src/util.rs | 5 +- crates/codegen/tests/codegen.rs | 42 ++++++++ crates/lib/src/deployment.rs | 2 + crates/lib/src/deployment/api.rs | 117 +++++++++++++++++++++++ sdks/rust/src/client_cache.rs | 3 + sdks/rust/src/client_cache/uuid_tests.rs | 75 +++++++++++++++ 6 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 crates/lib/src/deployment/api.rs create mode 100644 sdks/rust/src/client_cache/uuid_tests.rs diff --git a/crates/codegen/src/util.rs b/crates/codegen/src/util.rs index f8c590af07d..938fd759146 100644 --- a/crates/codegen/src/util.rs +++ b/crates/codegen/src/util.rs @@ -85,7 +85,10 @@ pub(super) fn type_ref_name(module: &ModuleDef, typeref: AlgebraicTypeRef) -> St pub(super) fn is_type_filterable(typespace: &TypespaceForGenerate, ty: &AlgebraicTypeUse) -> bool { match ty { AlgebraicTypeUse::Primitive(prim) => !matches!(prim, PrimitiveType::F32 | PrimitiveType::F64), - AlgebraicTypeUse::String | AlgebraicTypeUse::Identity | AlgebraicTypeUse::ConnectionId => true, + AlgebraicTypeUse::String + | AlgebraicTypeUse::Identity + | AlgebraicTypeUse::ConnectionId + | AlgebraicTypeUse::Uuid => true, // Sum types with all unit variants: AlgebraicTypeUse::Never => true, AlgebraicTypeUse::Option(inner) => matches!(&**inner, AlgebraicTypeUse::Unit), diff --git a/crates/codegen/tests/codegen.rs b/crates/codegen/tests/codegen.rs index 06dc3ebe8fc..faf3966faee 100644 --- a/crates/codegen/tests/codegen.rs +++ b/crates/codegen/tests/codegen.rs @@ -39,3 +39,45 @@ declare_tests! { test_codegen_typescript => TypeScript, test_codegen_rust => Rust, } + +#[test] +fn rust_uuid_primary_and_unique_keys_generate_registered_lookup_accessors() { + use spacetimedb_lib::{ + db::raw_def::{v10::RawModuleDefV10Builder, v9::btree}, + AlgebraicType, + }; + + let mut builder = RawModuleDefV10Builder::new(); + builder + .build_table_with_new_type( + "uuid_rows", + [ + ("id", AlgebraicType::uuid()), + ("alias", AlgebraicType::uuid()), + ("sequence", AlgebraicType::U64), + ], + true, + ) + .with_primary_key(0) + .with_unique_constraint(0) + .with_index(btree(0), "uuid_rows_id_idx", "id") + .with_unique_constraint(1) + .with_index(btree(1), "uuid_rows_alias_idx", "alias") + .with_unique_constraint(2) + .with_index(btree(2), "uuid_rows_sequence_idx", "sequence") + .finish(); + let module = ModuleDef::try_from(builder.finish()).unwrap(); + let table = generate(&module, &Rust, &CodegenOptions::default()) + .into_iter() + .find(|file| file.filename == "uuid_rows_table.rs") + .unwrap() + .code; + for column in ["id", "alias"] { + assert!(table.contains(&format!("pub fn {column}(&self)"))); + assert!(table.contains(&format!( + "add_unique_constraint::<__sdk::Uuid>({column:?}, |row| &row.{column})" + ))); + } + assert!(table.contains("pub fn find(&self, col_val: &__sdk::Uuid) -> Option")); + assert!(table.contains("add_unique_constraint::(\"sequence\", |row| &row.sequence)")); +} diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs index 39d76e0d865..d05b4b0cd05 100644 --- a/crates/lib/src/deployment.rs +++ b/crates/lib/src/deployment.rs @@ -6,6 +6,8 @@ use crate::container::{ContainerAction, ContainerSpec, ContainerSpecLimits, ContainerValidationError}; use crate::{bsatn, hash_bytes, Hash, SpacetimeType, Uuid}; +#[cfg(feature = "serde")] +pub mod api; pub mod manifest; pub const PUBLISH_PROTOCOL_VERSION: u32 = 1; diff --git a/crates/lib/src/deployment/api.rs b/crates/lib/src/deployment/api.rs new file mode 100644 index 00000000000..c1adcebef6e --- /dev/null +++ b/crates/lib/src/deployment/api.rs @@ -0,0 +1,117 @@ +//! HTTP publication messages shared by the CLI, dashboard and Cloud. Artifacts +//! are uploaded separately; no message carries credentials or environment values. + +use super::{manifest::PreparedDeploymentManifest, uuid_json, DeploymentSpec, PUBLISH_PROTOCOL_VERSION}; +use crate::{container::OciDigest, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactReference { + pub digest: OciDigest, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishRequest { + pub manifest: PreparedDeploymentManifest, + /// Must match the server-generated reservation when creating a database. + pub creation: Option, + /// Original uploaded OCI index or executable manifest. Required for Set. + /// Keep uses the prior retained executable manifest; Remove has no image. + pub image_source: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CreationOptions { + pub parent: Option, + pub organization: Option, + pub num_replicas: Option, + #[serde(default = "default_anti_affinity")] + pub enforce_anti_affinity: bool, +} +fn default_anti_affinity() -> bool { + true +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReserveDatabaseRequest { + pub version: u32, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub options: CreationOptions, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DatabaseReservation { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub expires_at: String, + pub staging_open: bool, + pub artifact_endpoint: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PublicationPhase { + Prepared, + Quiescing, + Committed, + Activating, + Complete, + AbortedBeforeCommit, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicationStatus { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub operation_id: Uuid, + pub phase: PublicationPhase, + pub expected_revision: Option, + pub proposed_revision: Hash, + pub error: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeploymentStatus { + pub database_identity: Identity, + /// None means this database has not yet used managed publication. + pub revision: Option, + pub deployment: DeploymentSpec, + /// Lets Keep select exactly the currently installed program bytes. + pub module_artifact: ArtifactReference, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublishPermission { + pub identity: Identity, + pub can_publish: bool, + /// Decimal string preserves all u64 revision values in browser clients. + pub source_revision: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicationCapabilities { + pub version: u32, + pub enabled: bool, + pub artifact_endpoint: Option, +} +impl PublicationCapabilities { + pub fn disabled() -> Self { + Self { + version: PUBLISH_PROTOCOL_VERSION, + enabled: false, + artifact_endpoint: None, + } + } +} diff --git a/sdks/rust/src/client_cache.rs b/sdks/rust/src/client_cache.rs index ab0a90980bd..5222dcb06f9 100644 --- a/sdks/rust/src/client_cache.rs +++ b/sdks/rust/src/client_cache.rs @@ -2,6 +2,9 @@ //! //! This module is internal, and may incompatibly change without warning. +#[cfg(test)] +mod uuid_tests; + use crate::callbacks::CallbackId; use crate::db_connection::{debug_log, PendingMutation, SharedCell}; use crate::spacetime_module::{InModule, SpacetimeModule, TableUpdate, WithBsatn}; diff --git a/sdks/rust/src/client_cache/uuid_tests.rs b/sdks/rust/src/client_cache/uuid_tests.rs new file mode 100644 index 00000000000..a9516acc81b --- /dev/null +++ b/sdks/rust/src/client_cache/uuid_tests.rs @@ -0,0 +1,75 @@ +use super::*; +use spacetimedb_lib::Uuid; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Debug)] +struct CountedRow { + id: Uuid, + value: u64, + clones: Arc, +} +impl Clone for CountedRow { + fn clone(&self) -> Self { + self.clones.fetch_add(1, Ordering::SeqCst); + Self { + id: self.id, + value: self.value, + clones: self.clones.clone(), + } + } +} + +fn row(id: Uuid, value: u64, clones: &Arc) -> WithBsatn { + WithBsatn { + bsatn: spacetimedb_lib::bsatn::to_vec(&(id, value)).unwrap().into(), + row: CountedRow { + id, + value, + clones: clones.clone(), + }, + } +} + +#[test] +fn uuid_unique_lookup_clones_only_the_match_and_tracks_updates_and_deletes() { + let clones = Arc::new(AtomicUsize::new(0)); + let mut cache = TableCache::::new(None); + // This is the exact registration emitted by Rust codegen. + cache.add_unique_constraint::("id", |row| &row.id); + let ids = (0u8..128) + .map(|n| Uuid::from_random_bytes_v4([n; 16])) + .collect::>(); + cache.apply_diff(&TableUpdate { + inserts: ids + .iter() + .enumerate() + .map(|(n, id)| row(*id, n as u64, &clones)) + .collect(), + deletes: vec![], + }); + clones.store(0, Ordering::SeqCst); + // UniqueConstraintHandle::find delegates to this lookup and clones only + // its selected row. A whole-cache snapshot would increment this 128 times. + let found = cache.find_by_unique_index("id", &ids[63]).cloned().unwrap(); + assert_eq!((found.id, found.value), (ids[63], 63)); + assert_eq!(clones.load(Ordering::SeqCst), 1); + let absent = Uuid::from_random_bytes_v4([255; 16]); + assert!(cache.find_by_unique_index("id", &absent).cloned().is_none()); + assert_eq!(clones.load(Ordering::SeqCst), 1); + + cache.apply_diff(&TableUpdate { + inserts: vec![row(ids[63], 999, &clones)], + deletes: vec![row(ids[63], 63, &clones)], + }); + clones.store(0, Ordering::SeqCst); + assert_eq!(cache.find_by_unique_index("id", &ids[63]).cloned().unwrap().value, 999); + assert_eq!(clones.load(Ordering::SeqCst), 1); + cache.apply_diff(&TableUpdate { + inserts: vec![], + deletes: vec![row(ids[63], 999, &clones)], + }); + clones.store(0, Ordering::SeqCst); + assert!(cache.find_by_unique_index("id", &ids[63]).cloned().is_none()); + assert_eq!(clones.load(Ordering::SeqCst), 0); + assert_eq!(cache.find_by_unique_index("id", &ids[64]).unwrap().value, 64); +} From b9ef180b5e14baac7649c5ea4a0568c8dbece04c Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 02:29:11 -0400 Subject: [PATCH 07/23] Bound JavaScript execution and recover expected procedure startup failures --- crates/core/src/config.rs | 51 +++ crates/core/src/host/host_controller.rs | 3 + .../execution_deadline_tests.rs | 350 ++++++++++++++++++ crates/core/src/host/module_host.rs | 147 ++++++-- crates/core/src/host/scheduler.rs | 91 +++++ crates/core/src/host/v8/execution_deadline.rs | 272 ++++++++++++++ crates/core/src/host/v8/mod.rs | 150 ++++++-- crates/standalone/config.toml | 4 + 8 files changed, 1000 insertions(+), 68 deletions(-) create mode 100644 crates/core/src/host/host_controller/execution_deadline_tests.rs create mode 100644 crates/core/src/host/v8/execution_deadline.rs diff --git a/crates/core/src/config.rs b/crates/core/src/config.rs index 7821f61f5d9..8fc03b44165 100644 --- a/crates/core/src/config.rs +++ b/crates/core/src/config.rs @@ -171,6 +171,7 @@ impl<'de> serde::Deserialize<'de> for ConfigFile { v8: V8Config { procedure_instance_pool_size: config.v8.procedure_instance_pool_size, heap_policy: config.v8_heap_policy, + execution_timeout: config.v8.execution_timeout, }, }) } @@ -247,6 +248,9 @@ impl Default for WasmConfigToml { pub struct V8Config { pub procedure_instance_pool_size: NonZeroUsize, pub heap_policy: V8HeapPolicyConfig, + /// Wall-clock limit for one JavaScript startup, description or function call. + /// Must be positive and no greater than 120 seconds. + pub execution_timeout: Duration, } impl Default for V8Config { @@ -254,11 +258,20 @@ impl Default for V8Config { Self { procedure_instance_pool_size: default_v8_procedure_instance_pool_size(), heap_policy: V8HeapPolicyConfig::default(), + execution_timeout: default_v8_execution_timeout(), } } } impl V8Config { + pub fn validate_execution_timeout(&self) -> anyhow::Result<()> { + anyhow::ensure!( + !self.execution_timeout.is_zero() && self.execution_timeout <= default_v8_execution_timeout(), + "V8 execution timeout must be positive and no greater than 120 seconds" + ); + Ok(()) + } + pub fn normalized(mut self) -> Self { self.heap_policy = self.heap_policy.normalized(); self @@ -273,16 +286,35 @@ struct V8ConfigToml { deserialize_with = "de_nz_usize" )] pub procedure_instance_pool_size: NonZeroUsize, + #[serde( + default = "default_v8_execution_timeout", + deserialize_with = "de_v8_execution_timeout" + )] + pub execution_timeout: Duration, } impl Default for V8ConfigToml { fn default() -> Self { Self { procedure_instance_pool_size: default_v8_procedure_instance_pool_size(), + execution_timeout: default_v8_execution_timeout(), } } } +fn default_v8_execution_timeout() -> Duration { + Duration::from_secs(120) +} + +fn de_v8_execution_timeout<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + let timeout = de_nz_duration(deserializer)? + .filter(|timeout| *timeout <= default_v8_execution_timeout()) + .ok_or_else(|| { + serde::de::Error::custom("V8 execution timeout must be positive and no greater than 120 seconds") + })?; + Ok(timeout) +} + #[derive(Clone, Copy, Debug, serde::Deserialize)] #[serde(rename_all = "kebab-case")] pub struct V8HeapPolicyConfig { @@ -558,6 +590,25 @@ mod tests { .unwrap_err(); } + #[test] + fn v8_execution_timeout_is_finite_and_bounded() { + let defaults: ConfigFile = toml::from_str("").unwrap(); + assert_eq!(defaults.v8.execution_timeout, Duration::from_secs(120)); + for value in ["0", "121", "\"0s\"", "\"121s\""] { + assert!(toml::from_str::(&format!("[v8]\nexecution-timeout = {value}")).is_err()); + } + let config: ConfigFile = toml::from_str("[v8]\nexecution-timeout = \"150ms\"").unwrap(); + assert_eq!(config.v8.execution_timeout, Duration::from_millis(150)); + for timeout in [Duration::ZERO, Duration::from_secs(121)] { + assert!(V8Config { + execution_timeout: timeout, + ..V8Config::default() + } + .validate_execution_timeout() + .is_err()); + } + } + #[test] fn v8_heap_policy_defaults_when_omitted() { let config: ConfigFile = toml::from_str("").unwrap(); diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index d29614d94a6..ddda547492d 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -73,6 +73,9 @@ type Hosts = Arc>>; #[cfg(test)] mod deployment_tests; +#[cfg(test)] +mod execution_deadline_tests; + #[cfg(test)] static FAIL_NEXT_DEPLOYMENT_ACTIVATION: Mutex> = Mutex::new(std::collections::BTreeSet::new()); diff --git a/crates/core/src/host/host_controller/execution_deadline_tests.rs b/crates/core/src/host/host_controller/execution_deadline_tests.rs new file mode 100644 index 00000000000..700843c8fdc --- /dev/null +++ b/crates/core/src/host/host_controller/execution_deadline_tests.rs @@ -0,0 +1,350 @@ +//! Actual V8 module hosts, transactions and updates. No network or external service. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::FunctionArgs; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_lib::db::raw_def::{v10::RawModuleDefV10Builder, v9::Lifecycle}; +use spacetimedb_paths::FromPathUnchecked; +use spacetimedb_sats::{AlgebraicType, ProductType}; + +fn config() -> HostRuntimeConfig { + HostRuntimeConfig { + v8: V8Config { + execution_timeout: Duration::from_millis(200), + ..V8Config::default() + }, + ..HostRuntimeConfig::default() + } +} + +fn program(loop_in_first_init: bool) -> Program { + program_with_startup_cutoff(loop_in_first_init, u64::MAX) +} + +fn program_with_startup_cutoff(loop_in_first_init: bool, cutoff_millis: u64) -> Program { + let mut schema = RawModuleDefV10Builder::new(); + schema + .build_table_with_new_type("rows", [("value", AlgebraicType::U64)], true) + .finish(); + schema.add_lifecycle_reducer(Lifecycle::Init, "init", ProductType::unit()); + schema.add_reducer("loop", ProductType::unit()); + schema.add_reducer("good", ProductType::unit()); + schema.add_procedure("task", ProductType::unit(), AlgebraicType::U64); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{ register_hooks, table_id_from_name, datastore_insert_bsatn }} from "spacetime:sys@1.0"; + import {{ register_hooks as register_procedure_hooks }} from "spacetime:sys@1.2"; + if (Date.now() > {cutoff_millis}) {{ for (;;) {{}} }} + let initCalls = 0; + let nextValue = 0; + register_hooks({{ + __describe_module__: function() {{ return new Uint8Array({schema:?}); }}, + __call_reducer__: function(id) {{ + const row = new Uint8Array(8); + row[0] = ++nextValue; + datastore_insert_bsatn(table_id_from_name("rows"), row); + if (id === 1 || (id === 0 && ++initCalls === 1 && {loop_in_first_init})) {{ for (;;) {{}} }} + return {{ tag: "ok" }}; + }}, + }}); + let retained; + register_procedure_hooks({{ __call_procedure__: function() {{ + if (Date.now() > {cutoff_millis}) {{ retained = new Array(4 * 1024 * 1024).fill(1); }} + return new Uint8Array(8); + }} }}); + "# + ) + .into_bytes(), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_procedure_startup_failure_keeps_main_module_registered() { + use std::time::{SystemTime, UNIX_EPOCH}; + let cutoff = SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + Duration::from_secs(3); + let program = program_with_startup_cutoff(false, cutoff.as_millis() as u64); + let (_directory, controller, database) = controller_fixture( + 0xed05, + &program, + HostRuntimeConfig { + v8: V8Config { + // One slot makes a leaked admission permit observable on retry. + procedure_instance_pool_size: std::num::NonZeroUsize::new(1).unwrap(), + ..config().v8 + }, + ..config() + }, + ); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert!(call(&module, "good").await.is_ok()); + let until_cutoff = cutoff.saturating_sub(SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + tokio::time::sleep(until_cutoff + Duration::from_millis(50)).await; + // The module's main isolate already ran startup. Only newly created + // procedure isolates encounter the now-hostile startup branch. + for _ in 0..2 { + let result = timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary), + ) + .await + .unwrap(); + let error = result.result.unwrap_err(); + assert!(error.to_string().contains("wall-clock limit"), "{error}"); + let registered = controller.get_module_host(database.id).await.unwrap(); + assert!(call(®istered, "good").await.is_ok()); + } + assert_eq!(row_count(&module), Some(4)); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +fn database(id: u64, program: &Program) -> Database { + Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Js, + initial_program: program.hash, + } +} + +fn controller_fixture( + id: u64, + program: &Program, + config: HostRuntimeConfig, +) -> (tempfile::TempDir, HostController, Database) { + let directory = tempfile::tempdir().unwrap(); + let data_dir = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let initial = program.clone(); + let storage = move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }; + let controller = HostController::new( + data_dir.clone(), + db::Config { + storage: db::Storage::Memory, + page_pool_max_size: None, + }, + config, + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(LocalPersistenceProvider::new(data_dir)), + JobCores::without_pinned_cores(), + ); + (directory, controller, database(id, program)) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_failed_procedure_recreation_keeps_main_module_registered() { + use std::time::{SystemTime, UNIX_EPOCH}; + let cutoff = SystemTime::now().duration_since(UNIX_EPOCH).unwrap() + Duration::from_secs(3); + let program = program_with_startup_cutoff(false, cutoff.as_millis() as u64); + let (_directory, controller, database) = controller_fixture( + 0xed06, + &program, + HostRuntimeConfig { + v8: V8Config { + procedure_instance_pool_size: std::num::NonZeroUsize::new(1).unwrap(), + heap_policy: crate::config::V8HeapPolicyConfig { + heap_limit_bytes: 64 * 1024 * 1024, + heap_check_request_interval: Some(1), + heap_gc_trigger_fraction: 0.2, + heap_retire_fraction: 0.2, + ..Default::default() + }, + ..config().v8 + }, + ..config() + }, + ); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + // Populate the one-slot procedure pool before the startup branch changes. + assert!(module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + .result + .is_ok()); + let remaining = cutoff.saturating_sub(SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + tokio::time::sleep(remaining + Duration::from_millis(50)).await; + // This invocation runs in the existing isolate and leaves32MiB alive. The + // real post-call heap policy retires it and tries a new, now-looping startup. + assert!(module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + .result + .is_ok()); + let result = timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary), + ) + .await + .unwrap(); + let error = result.result.unwrap_err(); + assert!( + error.to_string().contains("procedure isolate startup failed"), + "{error}" + ); + assert!(error.to_string().contains("wall-clock limit"), "{error}"); + let registered = controller.get_module_host(database.id).await.unwrap(); + assert!(call(®istered, "good").await.is_ok()); + // A subsequent checkout also completes, proving the dead instance's slot + // was released instead of leaked after its replacement failed. + assert!(timeout( + Duration::from_secs(5), + module.call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary,) + ) + .await + .unwrap() + .result + .is_err()); + assert!(call(®istered, "good").await.is_ok()); + drop(registered); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} + +async fn launch(id: u64, program: Program) -> anyhow::Result<(Program, LaunchedModule)> { + timeout( + Duration::from_secs(5), + Host::try_init_in_memory_to_check( + &HostRuntimes::new(None, config()), + PagePool::new(None), + database(id, &program), + program, + AllocatedJobCore::default(), + BsatnRowListBuilderPool::new(), + ), + ) + .await? +} + +fn row_count(module: &ModuleHost) -> Option { + module.relational_db().with_read_only(Workload::Internal, |tx| { + tx.table_id_from_name("rows") + .unwrap() + .and_then(|table| tx.table_row_count(table)) + }) +} + +async fn call(module: &ModuleHost, name: &str) -> ReducerCallResult { + timeout( + Duration::from_secs(5), + module.call_reducer(Identity::ONE, None, None, None, None, name, FunctionArgs::Nullary), + ) + .await + .unwrap() + .unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_rolls_back_init_and_reducer_and_preserves_isolate() { + let (program, launched) = launch(0xed01, program(true)).await.unwrap(); + let module = launched.module_host; + let result = module.init_database(program.clone()).await.unwrap().unwrap(); + assert!(result.is_err(), "infinite init must fail"); + assert!(module.relational_db().program().unwrap().is_none()); + assert_eq!( + row_count(&module), + None, + "failed init must roll back the schema and inserted row" + ); + + assert!(module.init_database(program).await.unwrap().unwrap().is_ok()); + assert_eq!(row_count(&module), Some(1)); + let failed = call(&module, "loop").await; + assert!(failed.is_err()); + assert_eq!( + row_count(&module), + Some(1), + "timed-out reducer must roll back its write" + ); + + for _ in 0..64 { + assert!(call(&module, "good").await.is_ok()); + } + // Cross the original timer boundary before reusing the same isolate again. + tokio::time::sleep(Duration::from_millis(250)).await; + assert!(call(&module, "good").await.is_ok()); + assert_eq!(row_count(&module), Some(66)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn execution_deadline_bounds_startup_description_and_failed_update() { + let valid = program(false); + let looping_startup = Program::from_bytes(ModuleKind::JS, b"for (;;) {}".to_vec()); + let looping_description = Program::from_bytes( + ModuleKind::JS, + br#"import {register_hooks} from "spacetime:sys@1.0"; + register_hooks({__describe_module__: function() {for (;;) {}}, __call_reducer__: function() {}});"# + .to_vec(), + ); + for (id, bad) in [(0xed02, &looping_startup), (0xed03, &looping_description)] { + let error = match launch(id, bad.clone()).await { + Ok(_) => panic!("infinite JavaScript unexpectedly launched"), + Err(error) => error, + }; + assert!(format!("{error:#}").contains("wall-clock limit"), "{error:#}"); + } + + let (_directory, controller, database) = controller_fixture(0xed04, &valid, config()); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(row_count(&module), Some(1)); + for bad in [looping_startup, looping_description] { + assert!(timeout( + Duration::from_secs(5), + controller.update_module_host( + database.clone(), + HostType::Js, + database.id, + bad.bytes, + MigrationPolicy::Compatible, + ) + ) + .await + .unwrap() + .is_err()); + assert_eq!(module.relational_db().program().unwrap().unwrap().hash, valid.hash); + assert!(call(&module, "good").await.is_ok()); + } + let mut newer = valid.bytes.to_vec(); + newer.extend_from_slice(b"\n// Valid replacement after two deadline failures.\n"); + let newer = Program::from_bytes(ModuleKind::JS, newer); + controller + .update_module_host( + database.clone(), + HostType::Js, + database.id, + newer.bytes.clone(), + MigrationPolicy::Compatible, + ) + .await + .unwrap(); + let replacement = controller.get_module_host(database.id).await.unwrap(); + assert_eq!(replacement.relational_db().program().unwrap().unwrap().hash, newer.hash); + assert!(call(&replacement, "good").await.is_ok()); + drop(replacement); + drop(module); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); +} diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 78dab16c3ab..91c7a1dd091 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -463,7 +463,10 @@ impl WasmtimeModuleHost { A: Send + 'static, { let instance_manager = self.procedure_instances.clone(); - let ModuleInstanceLease { instance, slot } = instance_manager.get_instance().await; + let ModuleInstanceLease { instance, slot } = instance_manager + .get_instance() + .await + .unwrap_or_else(|never| match never {}); let label = label.to_owned(); self.procedure_executor.enqueue_job(async move || { scopeguard::defer_on_unwind!({ @@ -488,7 +491,8 @@ struct V8ModuleHost { /// A module; used as a bound on `InstanceManager`. trait GenericModule { type Instance: GenericModuleInstance; - async fn create_instance(&self) -> Self::Instance; + type CreationError; + async fn create_instance(&self) -> Result; fn host_type(&self) -> HostType; } @@ -518,8 +522,9 @@ impl GenericModuleInstance for Box { impl GenericModule for Arc { type Instance = Box; - async fn create_instance(&self) -> Self::Instance { - Box::new((**self).create_instance()) + type CreationError = std::convert::Infallible; + async fn create_instance(&self) -> Result { + Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { HostType::Wasm @@ -528,8 +533,9 @@ impl GenericModule for Arc { impl GenericModule for Arc { type Instance = Box; - async fn create_instance(&self) -> Self::Instance { - Box::new((**self).create_instance()) + type CreationError = std::convert::Infallible; + async fn create_instance(&self) -> Result { + Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { HostType::Wasm @@ -538,7 +544,8 @@ impl GenericModule for Arc { impl GenericModule for super::v8::JsModule { type Instance = super::v8::JsProcedureInstance; - async fn create_instance(&self) -> Self::Instance { + type CreationError = anyhow::Error; + async fn create_instance(&self) -> Result { self.create_instance().await } fn host_type(&self) -> HostType { @@ -1387,14 +1394,17 @@ impl ModuleInstanceManager { } } - async fn with_instance(&self, f: impl AsyncFnOnce(M::Instance) -> (R, M::Instance)) -> R { - let ModuleInstanceLease { instance, slot } = self.get_instance().await; + async fn with_instance( + &self, + f: impl AsyncFnOnce(M::Instance) -> (R, M::Instance), + ) -> Result { + let ModuleInstanceLease { instance, slot } = self.get_instance().await?; let (res, instance) = f(instance).await; self.return_instance(ModuleInstanceLease { instance, slot }); - res + Ok(res) } - async fn get_instance(&self) -> ModuleInstanceLease { + async fn get_instance(&self) -> Result, M::CreationError> { let slot = if let Some(instance_slots) = &self.instance_slots { Some( instance_slots @@ -1415,13 +1425,13 @@ impl ModuleInstanceManager { instance } else { let start_time = std::time::Instant::now(); - let res = self.module.create_instance().await; + let res = self.module.create_instance().await?; let elapsed_time = start_time.elapsed(); self.metrics.observe_instance_created(elapsed_time); res }; - ModuleInstanceLease { instance, slot } + Ok(ModuleInstanceLease { instance, slot }) } fn return_instance(&self, lease: ModuleInstanceLease) { @@ -1526,6 +1536,45 @@ impl UpdateDatabaseResult { #[error("no such module")] pub struct NoSuchModule; +#[derive(thiserror::Error, Debug)] +enum PooledCallError { + #[error(transparent)] + NoSuchModule(#[from] NoSuchModule), + #[error("module instance startup failed: {0}")] + Startup(anyhow::Error), +} + +impl From for ProcedureCallError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => { + Self::InternalError(format!("module instance startup failed: {error:#}")) + } + } + } +} + +impl From for HttpHandlerCallError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => { + Self::InternalError(format!("module instance startup failed: {error:#}")) + } + } + } +} + +impl From for CallScheduledFunctionError { + fn from(error: PooledCallError) -> Self { + match error { + PooledCallError::NoSuchModule(error) => Self::NoSuchModule(error), + PooledCallError::Startup(error) => Self::InstanceStartup(error), + } + } +} + #[derive(thiserror::Error, Debug)] pub enum ReducerCallError { #[error(transparent)] @@ -1903,7 +1952,7 @@ impl ModuleHost { arg: A, wasm: impl AsyncFnOnce(A, &mut ModuleInstance) -> R + Send + 'static, js: impl AsyncFnOnce(A, &JsProcedureInstance) -> R, - ) -> Result + ) -> Result where R: Send + 'static, A: Send + 'static, @@ -1931,16 +1980,17 @@ impl ModuleHost { .await }) .await + .unwrap_or_else(|never| match never {}) } - ModuleHostInner::Js(host) => { - host.procedure_instances - .with_instance(async |inst| { - drop(timer_guard); - let res = js(arg, &inst).await; - (res, inst) - }) - .await - } + ModuleHostInner::Js(host) => host + .procedure_instances + .with_instance(async |inst| { + drop(timer_guard); + let res = js(arg, &inst).await; + (res, inst) + }) + .await + .map_err(PooledCallError::Startup)?, }) } @@ -2605,9 +2655,7 @@ impl ModuleHost { ) -> CallProcedureReturn { let res = async { let call = self.prepare_procedure_call(caller.into(), caller_connection_id, timer, procedure_name, args)?; - self.call_procedure_with_params(&call.name, call.params) - .await - .map_err(Into::into) + self.call_procedure_with_params(&call.name, call.params).await } .await; @@ -2648,8 +2696,29 @@ impl ModuleHost { match &*self.inner { ModuleHostInner::Js(host) => { - let lease = host.procedure_instances.get_instance().await; - let call = lease.instance.enqueue_procedure(params).await; + let lease = match host.procedure_instances.get_instance().await { + Ok(lease) => lease, + Err(error) => { + return self.send_procedure_error( + &procedure_name, + timer, + target, + PooledCallError::Startup(error).into(), + ); + } + }; + let call = match lease.instance.enqueue_procedure(params).await { + Ok(call) => call, + Err(error) => { + self.return_js_procedure_instance(lease); + return self.send_procedure_error( + &procedure_name, + timer, + target, + ProcedureCallError::InternalError(error.to_string()), + ); + } + }; let module = self.clone(); tokio::spawn(async move { match call.receive().await { @@ -2659,6 +2728,16 @@ impl ModuleHost { log::warn!("failed to send procedure result: {err:#}"); } } + JsProcedureCallCompletion::StartupFailed(error) => { + if let Err(error) = module.send_procedure_error( + &procedure_name, + timer, + target, + ProcedureCallError::InternalError(error.to_string()), + ) { + log::warn!("failed to send procedure startup error: {error:#}"); + } + } JsProcedureCallCompletion::Panicked | JsProcedureCallCompletion::WorkerExited => { log::warn!("detached JS procedure worker failed before returning a result"); (module.on_panic)(); @@ -2871,7 +2950,7 @@ impl ModuleHost { &self, name: &str, params: CallProcedureParams, - ) -> Result { + ) -> Result { call_pooled_instance!( self, name, @@ -2879,6 +2958,7 @@ impl ModuleHost { |params, inst| inst.call_procedure(params).await, |params, inst| inst.call_procedure(params).await, ) + .map_err(Into::into) } pub async fn call_http_handler( @@ -2929,10 +3009,13 @@ impl ModuleHost { self, "scheduled procedure", params, - |params, inst| inst.call_scheduled_procedure(params).await, - |params, inst| inst.call_scheduled_procedure(params).await, + |params, inst| Ok(inst.call_scheduled_procedure(params).await), + |params, inst| inst + .call_scheduled_procedure(params) + .await + .map_err(|error| CallScheduledFunctionError::InstanceStartup(error.into())), ) - .map_err(Into::into) + .map_err(CallScheduledFunctionError::from)? } /// Materializes the views return by the `view_collector`, if not already materialized, diff --git a/crates/core/src/host/scheduler.rs b/crates/core/src/host/scheduler.rs index f001a19ab53..9e60d770bb2 100644 --- a/crates/core/src/host/scheduler.rs +++ b/crates/core/src/host/scheduler.rs @@ -318,6 +318,8 @@ impl ScheduledFunctionParams { pub(crate) enum CallScheduledFunctionError { #[error(transparent)] NoSuchModule(#[from] NoSuchModule), + #[error("module instance startup failed: {0}")] + InstanceStartup(anyhow::Error), } #[cfg(target_pointer_width = "64")] @@ -395,6 +397,17 @@ impl SchedulerActor { // If the module already exited, leave the `ScheduledFunction` in // the database for when the module restarts. Err(CallScheduledFunctionError::NoSuchModule(_)) => {} + Err(CallScheduledFunctionError::InstanceStartup(error)) => { + // No transaction or procedure ran. Keep the schedule available + // and retry with a delay, including under shared deadline pressure. + log::warn!("scheduled procedure instance startup failed: {error:#}"); + if startup_retry_is_needed(module_host.info().relational_db(), &item) { + let key = self.queue.insert(item, Duration::from_secs(1)); + if let Some(id) = id { + self.key_map.insert(id, key); + } + } + } Ok(CallScheduledFunctionResult { reschedule: None }) => { // nothing to do } @@ -418,6 +431,26 @@ impl SchedulerActor { } } +/// A cancelled table-backed schedule must not retry forever when every new +/// procedure instance fails before reaching the normal schedule-row lookup. +fn startup_retry_is_needed(db: &RelationalDB, item: &QueueItem) -> bool { + let QueueItem::Id { id, .. } = item else { + return true; + }; + let exists = db.with_read_only(Workload::Internal, |tx| { + db.iter_by_col_eq(tx, id.table_id, id.id_column, &id.schedule_id.into()) + .map(|mut rows| rows.next().is_some()) + }); + match exists { + Ok(exists) => exists, + Err(error) => { + // A failed read is not proof that the user cancelled the schedule. + log::warn!("could not check scheduled procedure startup retry: {error:#}"); + true + } + } +} + #[derive(Debug)] pub(crate) struct CallScheduledFunctionResult { reschedule: Option, @@ -827,3 +860,61 @@ fn read_schedule_at(row: &RowRef<'_>, at_column: ColId) -> anyhow::Result QueueItem { + QueueItem::Id { + id: ScheduledFunctionId { + table_id, + schedule_id, + id_column: 0.into(), + at_column: 0.into(), + }, + function_name: "task".into(), + at: Timestamp::now(), + } + } + + #[test] + fn execution_deadline_startup_retry_stops_after_schedule_deletion() -> anyhow::Result<()> { + let db = TestDB::in_memory()?; + let table = db.create_table_for_test("pending", &[("id", AlgebraicType::U64)], &[0.into()])?; + with_auto_commit(&db, |tx| { + insert(&db, tx, table, &(7u64,))?; + insert(&db, tx, table, &(8u64,))?; + Ok::<_, anyhow::Error>(()) + })?; + + let cancelled = item(table, 7); + assert!(startup_retry_is_needed(&db, &cancelled)); + with_auto_commit(&db, |tx| { + assert_eq!(db.delete_by_rel(tx, table, [product!(7u64)]), 1); + Ok::<_, anyhow::Error>(()) + })?; + + // An unrelated remaining schedule cannot keep the cancelled ID alive. + assert!(!startup_retry_is_needed(&db, &cancelled)); + assert!(startup_retry_is_needed(&db, &item(table, 8))); + Ok(()) + } + + #[test] + fn execution_deadline_startup_retry_preserves_read_errors_and_volatile_calls() -> anyhow::Result<()> { + let db = TestDB::in_memory()?; + // The point read fails rather than positively observing a missing row. + assert!(startup_retry_is_needed(&db, &item(u32::MAX.into(), 7))); + assert!(startup_retry_is_needed( + &db, + &QueueItem::VolatileNonatomicImmediate { + function_name: "task".into(), + args: FunctionArgs::Nullary, + }, + )); + Ok(()) + } +} diff --git a/crates/core/src/host/v8/execution_deadline.rs b/crates/core/src/host/v8/execution_deadline.rs new file mode 100644 index 00000000000..ac6b1f3d1c9 --- /dev/null +++ b/crates/core/src/host/v8/execution_deadline.rs @@ -0,0 +1,272 @@ +//! Direct cross-thread termination, without executing a V8 interrupt callback. +//! +//! Pinned v8 145's IsolateHandle is Send+Sync and protects termination with its +//! annex mutex. No scope, raw isolate pointer, Rc or Cell leaves the isolate +//! thread. One persistent timer serves bounded active registrations. Finishing +//! or dropping a guard waits out any in-flight termination before the isolate +//! may reset termination or begin another invocation. + +use std::{ + collections::BTreeMap, + io, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Condvar, Mutex, OnceLock, + }, + time::{Duration, Instant}, +}; + +#[derive(Debug, thiserror::Error)] +#[error("JavaScript execution exceeded its wall-clock limit")] +pub(super) struct ExecutionTimedOut; + +const MAX_ACTIVE_DEADLINES: usize = 16_384; +static ACTIVE_DEADLINES: AtomicUsize = AtomicUsize::new(0); +type Key = (Instant, u64); +static SERVICE: OnceLock, String>> = OnceLock::new(); + +struct Capacity; +impl Capacity { + fn acquire() -> io::Result { + ACTIVE_DEADLINES + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { + (count < MAX_ACTIVE_DEADLINES).then_some(count + 1) + }) + .map_err(|_| io::Error::other("JavaScript execution deadline capacity exhausted"))?; + Ok(Self) + } +} +impl Drop for Capacity { + fn drop(&mut self) { + ACTIVE_DEADLINES.fetch_sub(1, Ordering::Relaxed); + } +} + +struct Registration { + expires: Instant, + state: Mutex, +} +struct State { + finished: bool, + expired: bool, + handle: v8::IsolateHandle, +} +#[derive(Default)] +struct Queue { + sequence: u64, + entries: BTreeMap>, +} +struct DeadlineService { + queue: Mutex, + wake: Condvar, +} + +impl DeadlineService { + fn start() -> io::Result> { + let service = Arc::new(Self { + queue: Mutex::new(Queue::default()), + wake: Condvar::new(), + }); + std::thread::Builder::new().name("v8-deadlines".into()).spawn({ + let service = service.clone(); + move || service.run() + })?; + Ok(service) + } + + fn run(&self) { + loop { + let registration = { + let mut queue = self.queue.lock().unwrap_or_else(|error| error.into_inner()); + loop { + let Some((&(expires, _), _)) = queue.entries.first_key_value() else { + queue = self.wake.wait(queue).unwrap_or_else(|error| error.into_inner()); + continue; + }; + let remaining = expires.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + break queue.entries.pop_first().unwrap().1; + } + queue = self + .wake + .wait_timeout(queue, remaining) + .unwrap_or_else(|error| error.into_inner()) + .0; + } + }; + // Never hold the queue lock while acquiring a registration lock. + // Cancellation may hold this lock while removing its queued entry. + let mut state = registration.state.lock().unwrap_or_else(|error| error.into_inner()); + if !state.finished { + state.expired = true; + // finish() must acquire this same lock before resetting V8. + state.handle.terminate_execution(); + } + } + } +} + +pub(super) struct ExecutionDeadline { + service: Arc, + registration: Arc, + key: Key, + finished: bool, + // Includes expired calls still returning from a native syscall. The timer + // popping its queue entry never releases this active-call reservation. + _capacity: Capacity, +} + +impl ExecutionDeadline { + pub fn start(handle: v8::IsolateHandle, timeout: Duration) -> io::Result { + let capacity = Capacity::acquire()?; + let service = SERVICE + .get_or_init(|| DeadlineService::start().map_err(|error| error.to_string())) + .as_ref() + .map_err(|error| io::Error::other(error.clone()))? + .clone(); + let expires = Instant::now() + timeout; + let registration = Arc::new(Registration { + expires, + state: Mutex::new(State { + finished: false, + expired: false, + handle, + }), + }); + let mut queue = service.queue.lock().unwrap_or_else(|error| error.into_inner()); + queue.sequence = queue + .sequence + .checked_add(1) + .ok_or_else(|| io::Error::other("JavaScript execution deadline sequence exhausted"))?; + let key = (expires, queue.sequence); + queue.entries.insert(key, registration.clone()); + drop(queue); + service.wake.notify_one(); + Ok(Self { + service, + registration, + key, + finished: false, + _capacity: capacity, + }) + } + + pub fn finish(mut self) -> bool { + let expired = self.cancel(); + self.finished = true; + expired + } + + fn cancel(&self) -> bool { + let mut state = self + .registration + .state + .lock() + .unwrap_or_else(|error| error.into_inner()); + if !state.finished { + // A timer delayed by OS scheduling must still make a late return + // roll back, even if it has not issued its termination request yet. + state.expired |= Instant::now() >= self.registration.expires; + state.finished = true; + } + let expired = state.expired; + // The timer releases the queue lock before taking this lock, so this + // order cannot invert its locks. Remove completed calls immediately, + // rather than accumulating their registrations for 120 seconds. + self.service + .queue + .lock() + .unwrap_or_else(|error| error.into_inner()) + .entries + .remove(&self.key); + drop(state); + self.service.wake.notify_one(); + expired + } +} + +impl Drop for ExecutionDeadline { + fn drop(&mut self) { + if !self.finished { + self.cancel(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::host::v8::to_value::test::with_scope; + + fn run(scope: &mut v8::PinScope<'_, '_>, source: &str) -> bool { + let source = v8::String::new(scope, source).unwrap(); + v8::Script::compile(scope, source, None).unwrap().run(scope).is_some() + } + + #[test] + fn deadline_terminates_and_isolate_can_run_again() { + with_scope(|scope| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_millis(40)).unwrap(); + assert!(!run(scope, "for (;;) {}")); + assert!(deadline.finish()); + scope.cancel_terminate_execution(); + assert!(run(scope, "1 + 1")); + }); + } + + #[test] + fn finished_and_dropped_registrations_cannot_terminate_later_execution() { + with_scope(|scope| { + for drop_guard in [false, true] { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_millis(40)).unwrap(); + let service = deadline.service.clone(); + let key = deadline.key; + if drop_guard { + drop(deadline); + } else { + assert!(!deadline.finish()); + } + assert!(!service.queue.lock().unwrap().entries.contains_key(&key)); + assert!(run( + scope, + "{ const end = Date.now() + 80; while (Date.now() < end) {} }" + )); + } + for _ in 0..1_000 { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), Duration::from_secs(1)).unwrap(); + let service = deadline.service.clone(); + let key = deadline.key; + assert!(run(scope, "1 + 1")); + assert!(!deadline.finish()); + assert!(!service.queue.lock().unwrap().entries.contains_key(&key)); + } + }); + } + + #[test] + fn late_return_is_expired_even_before_timer_observes_it() { + with_scope(|scope| { + // A service without a timer deterministically models OS delay. + let expires = Instant::now(); + let deadline = ExecutionDeadline { + service: Arc::new(DeadlineService { + queue: Mutex::new(Queue::default()), + wake: Condvar::new(), + }), + registration: Arc::new(Registration { + expires, + state: Mutex::new(State { + finished: false, + expired: false, + handle: scope.thread_safe_handle(), + }), + }), + key: (expires, 1), + finished: false, + _capacity: Capacity::acquire().unwrap(), + }; + assert!(deadline.finish()); + assert!(run(scope, "1 + 1")); + }); + } +} diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index d6c49aef093..c21139238cd 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -58,6 +58,7 @@ use self::error::{ catch_exception, exception_already_thrown, log_traceback, ErrorOrException, ExcResult, ExceptionThrown, PinTryCatch, Throwable, }; +use self::execution_deadline::{ExecutionDeadline, ExecutionTimedOut}; use self::ser::serialize_to_js; use self::string::{str_from_ident, IntoJsString}; use self::syscall::{ @@ -86,6 +87,7 @@ use crate::host::wasm_common::module_host_actor::{ ReducerExecuteResult, ReducerOp, ViewExecuteResult, ViewOp, WasmInstance, }; use crate::host::wasm_common::{RowIters, TimingSpanSet}; +use crate::host::ProcedureCallError; use crate::host::{ModuleHost, ReducerCallError, ReducerCallResult, Scheduler}; use crate::messages::control_db::HostType; use crate::module_host_context::ModuleCreationContext; @@ -110,8 +112,8 @@ use std::cell::Cell; use std::num::NonZeroUsize; use std::os::raw::c_void; use std::panic::{self, AssertUnwindSafe}; -use std::sync::{Arc, LazyLock}; -use std::time::Instant; +use std::sync::{Arc, LazyLock, OnceLock}; +use std::time::{Duration, Instant}; use tokio::sync::{mpsc, oneshot}; use v8::script_compiler::{compile_module, Source}; use v8::{ @@ -123,6 +125,7 @@ mod budget; mod builtins; mod de; mod error; +mod execution_deadline; mod from_value; mod ser; mod string; @@ -154,6 +157,7 @@ impl V8Runtime { program_bytes: &[u8], core: AllocatedJobCore, ) -> anyhow::Result { + self.config.validate_execution_timeout()?; V8_RUNTIME_GLOBAL .make_actor(mcc, program_bytes, core, self.config) .await @@ -256,6 +260,7 @@ impl V8RuntimeInner { load_balance_guard.clone(), core_pinner.clone(), heap_policy, + config.execution_timeout, metrics.clone(), ) .await?; @@ -266,6 +271,7 @@ impl V8RuntimeInner { core_pinner, procedure_instance_pool_size: config.procedure_instance_pool_size, heap_policy: config.heap_policy, + execution_timeout: config.execution_timeout, metrics, }; @@ -281,6 +287,7 @@ pub struct JsModule { core_pinner: CorePinner, procedure_instance_pool_size: NonZeroUsize, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, } @@ -305,7 +312,7 @@ impl JsModule { self.procedure_instance_pool_size } - async fn create_procedure_instance(&self) -> JsProcedureInstance { + async fn create_procedure_instance(&self) -> anyhow::Result { let program = self.program.clone(); let common = self.common.clone(); let load_balance_guard = self.load_balance_guard.clone(); @@ -320,14 +327,14 @@ impl JsModule { load_balance_guard, core_pinner, heap_policy, + self.execution_timeout, metrics, ) - .await - .expect("`spawn_procedure_instance_worker` should succeed when passed `ModuleCommon`"); - instance + .await?; + Ok(instance) } - pub async fn create_instance(&self) -> JsProcedureInstance { + pub async fn create_instance(&self) -> anyhow::Result { self.create_procedure_instance().await } } @@ -470,8 +477,18 @@ pub struct JsMainInstance { /// only execute procedure-style requests. pub struct JsProcedureInstance { tx: mpsc::Sender, + startup_failure: ProcedureStartupStatus, } +// Set only when a replacement generation fails before receiving another +// request. Publishing before closing the receiver proves queued calls did not +// execute. Unexplained worker exits and actual Rust panics remain fatal. +type ProcedureStartupStatus = Arc>; + +#[derive(Clone, Debug, thiserror::Error)] +#[error("procedure isolate startup failed: {0}")] +pub(in crate::host) struct JsProcedureStartupError(Arc); + impl JsMainInstance { async fn request(&self, request: R) -> R::Response { send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| request.into_worker_request(reply_tx)).await @@ -700,15 +717,15 @@ js_main_request! { impl JsProcedureInstance { pub(in crate::host) fn is_closed(&self) -> bool { - self.tx.is_closed() + self.startup_failure.get().is_some() || self.tx.is_closed() } async fn send_request( &self, ctx: &'static str, request: impl FnOnce(JsReplyTx) -> JsProcedureWorkerRequest, - ) -> T { - send_js_request(ctx, &self.tx, request).await + ) -> Result { + send_js_request(ctx, &self.tx, &self.startup_failure, request).await } pub async fn call_procedure(&self, params: CallProcedureParams) -> CallProcedureReturn { @@ -717,6 +734,10 @@ impl JsProcedureInstance { params, }) .await + .unwrap_or_else(|error| CallProcedureReturn { + result: Err(ProcedureCallError::InternalError(error.to_string())), + tx_offset: None, + }) } pub async fn call_http_handler( @@ -727,9 +748,13 @@ impl JsProcedureInstance { JsProcedureWorkerRequest::CallHttpHandler { reply_tx, params } }) .await + .map_err(|error| HttpHandlerCallError::InternalError(error.to_string()))? } - pub(in crate::host) async fn enqueue_procedure(&self, params: CallProcedureParams) -> JsProcedureCall { + pub(in crate::host) async fn enqueue_procedure( + &self, + params: CallProcedureParams, + ) -> Result { let (reply_tx, reply_rx) = oneshot::channel(); if self .tx @@ -737,15 +762,21 @@ impl JsProcedureInstance { .await .is_err() { + if let Some(error) = self.startup_failure.get() { + return Err(error.clone()); + } panic!("JS worker exited before accepting `call_procedure`"); } - JsProcedureCall { reply_rx } + Ok(JsProcedureCall { + reply_rx, + startup_failure: self.startup_failure.clone(), + }) } pub(in crate::host) async fn call_scheduled_procedure( &self, params: ScheduledFunctionParams, - ) -> CallScheduledFunctionResult { + ) -> Result { self.send_request("scheduled_procedure", |reply_tx| { JsProcedureWorkerRequest::ScheduledProcedure { reply_tx, params } }) @@ -756,19 +787,26 @@ impl JsProcedureInstance { async fn send_js_request( ctx: &'static str, tx: &mpsc::Sender, + startup_failure: &ProcedureStartupStatus, request: impl FnOnce(JsReplyTx) -> Req, -) -> T +) -> Result where Req: Send + 'static, { let (reply_tx, reply_rx) = oneshot::channel(); if tx.send(request(reply_tx)).await.is_err() { + if let Some(error) = startup_failure.get() { + return Err(error.clone()); + } panic!("JS worker exited before accepting `{ctx}`"); } match reply_rx.await { - Ok(Ok(value)) => value, + Ok(Ok(value)) => Ok(value), Ok(Err(panic)) => panic::resume_unwind(panic), - Err(_) => panic!("JS worker exited before replying to `{ctx}`"), + Err(_) => match startup_failure.get() { + Some(error) => Err(error.clone()), + None => panic!("JS worker exited before replying to `{ctx}`"), + }, } } @@ -795,11 +833,13 @@ pub(in crate::host) type JsFatalHook = Arc; pub(in crate::host) struct JsProcedureCall { reply_rx: oneshot::Receiver>, + startup_failure: ProcedureStartupStatus, } pub(in crate::host) enum JsProcedureCallCompletion { Completed(CallProcedureReturn), Panicked, + StartupFailed(JsProcedureStartupError), WorkerExited, } @@ -808,7 +848,10 @@ impl JsProcedureCall { match self.reply_rx.await { Ok(Ok(ret)) => JsProcedureCallCompletion::Completed(ret), Ok(Err(_panic)) => JsProcedureCallCompletion::Panicked, - Err(_) => JsProcedureCallCompletion::WorkerExited, + Err(_) => match self.startup_failure.get() { + Some(error) => JsProcedureCallCompletion::StartupFailed(error.clone()), + None => JsProcedureCallCompletion::WorkerExited, + }, } } } @@ -1196,7 +1239,9 @@ fn startup_instance_worker<'scope>( scope: &mut PinScope<'scope, '_>, program: Arc, module_or_mcc: Either, + execution_timeout: Duration, ) -> anyhow::Result<(HookFunctions<'scope>, ModuleCommon)> { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), execution_timeout)?; let hook_functions = catch_exception(scope, |scope| { // Start-up the user's module. let exports_obj = eval_user_module(scope, &program)?; @@ -1205,13 +1250,19 @@ fn startup_instance_worker<'scope>( let hooks = get_hooks(scope, exports_obj)?.ok_or_else(|| anyhow::anyhow!("must export schema as default export"))?; Ok(hooks) - })?; + }); + let expired = deadline.finish(); + scope.cancel_terminate_execution(); + if expired { + return Err(ExecutionTimedOut.into()); + } + let hook_functions = hook_functions?; // If we don't have a module, make one. let module_common = match module_or_mcc { Either::Left(module_common) => module_common, Either::Right(mcc) => { - let def = extract_description(scope, &hook_functions, &mcc.replica_ctx)?; + let def = extract_description(scope, &hook_functions, &mcc.replica_ctx, execution_timeout)?; // Validate and create a common module from the raw definition. build_common_module_from_raw(mcc, def)? @@ -1274,6 +1325,7 @@ async fn spawn_main_instance_worker( load_balance_guard: Arc, core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, ) -> anyhow::Result<(ModuleCommon, JsMainInstance)> { spawn_instance_worker::( @@ -1282,6 +1334,7 @@ async fn spawn_main_instance_worker( load_balance_guard, core_pinner, heap_policy, + execution_timeout, metrics, ) .await @@ -1293,6 +1346,7 @@ async fn spawn_procedure_instance_worker( load_balance_guard: Arc, core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, metrics: InstanceManagerMetrics, ) -> anyhow::Result<(ModuleCommon, JsProcedureInstance)> { spawn_instance_worker::( @@ -1301,6 +1355,7 @@ async fn spawn_procedure_instance_worker( load_balance_guard, core_pinner, heap_policy, + execution_timeout, metrics, ) .await @@ -1320,7 +1375,7 @@ trait JsWorkerSpec { fn channel(database_identity: &Identity) -> (Self::Sender, Self::Receiver); - fn make_instance(tx: Self::Sender) -> Self::Instance; + fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance; fn blocking_recv(rx: &mut Self::Receiver) -> Option; @@ -1352,7 +1407,7 @@ impl JsWorkerSpec for MainJsWorker { ) } - fn make_instance(tx: Self::Sender) -> Self::Instance { + fn make_instance(tx: Self::Sender, _startup_failure: ProcedureStartupStatus) -> Self::Instance { JsMainInstance { tx } } @@ -1383,8 +1438,8 @@ impl JsWorkerSpec for ProcedureJsWorker { mpsc::channel(JS_PROCEDURE_INSTANCE_QUEUE_CAPACITY) } - fn make_instance(tx: Self::Sender) -> Self::Instance { - JsProcedureInstance { tx } + fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance { + JsProcedureInstance { tx, startup_failure } } fn blocking_recv(rx: &mut Self::Receiver) -> Option { @@ -1588,8 +1643,8 @@ fn spawn_v8_worker_thread(worker_kind: JsWorkerKind, database_identity: Identity /// Spawns an instance worker for `program` and returns on success the /// corresponding instance handle that talks to the worker. /// -/// When [`ModuleCommon`] is passed, it's assumed that this program has already -/// been validated. In that case, `Ok(_)` should be returned. +/// When [`ModuleCommon`] is passed, the program has already been validated. +/// Starting another isolate can still fail, including its execution deadline. /// /// Otherwise, when [`ModuleCreationContext`] is passed, this is the first time /// both the module and instance are created. @@ -1602,6 +1657,7 @@ async fn spawn_instance_worker( load_balance_guard: Arc, mut core_pinner: CorePinner, heap_policy: V8HeapPolicyConfig, + execution_timeout: Duration, instance_metrics: InstanceManagerMetrics, ) -> anyhow::Result<(ModuleCommon, W::Instance)> where @@ -1615,6 +1671,8 @@ where Either::Right(mcc) => mcc.replica_ctx.database_identity, }; let (request_tx, mut request_rx) = W::channel(&database_identity); + let startup_failure = ProcedureStartupStatus::default(); + let worker_startup_failure = startup_failure.clone(); let rt = tokio::runtime::Handle::current(); @@ -1659,7 +1717,7 @@ where .expect("our builtin code shouldn't error"); // Setup the JS module, find call_reducer, and maybe build the module. - startup_instance_worker(scope, program.clone(), generation_module_or_mcc) + startup_instance_worker(scope, program.clone(), generation_module_or_mcc, execution_timeout) })); let (hooks, module_common) = match startup_result { @@ -1682,6 +1740,7 @@ where log::error!("startup result receiver disconnected"); } } else { + let _ = worker_startup_failure.set(JsProcedureStartupError(format!("{err:#}").into())); log::error!("failed to restart JS worker: {err:#}"); } return; @@ -1726,6 +1785,7 @@ where .v8_heap_limit_hit .with_label_values(&info.database_identity), initial_heap_limit: heap_policy.heap_limit_bytes, + execution_timeout, }; let _initial_heap_stats = sample_heap_stats(inst.scope, &mut heap_metrics); @@ -1782,7 +1842,7 @@ where // Get the module, if any, and get any setup errors from the worker. let res: Result = result_rx.await.expect("should have a sender"); res.map(|opt_mc| { - let inst = W::make_instance(request_tx); + let inst = W::make_instance(request_tx, startup_failure); (opt_mc, inst) }) } @@ -1872,11 +1932,12 @@ struct V8Instance<'a, 'scope, 'isolate> { /// Metric for the number of times the v8 heap limit has been hit. heap_limit_hit_metric: &'a IntCounter, initial_heap_limit: usize, + execution_timeout: Duration, } impl WasmInstance for V8Instance<'_, '_, '_> { fn extract_descriptions(&mut self) -> Result { - extract_description(self.scope, self.hooks, self.replica_ctx) + extract_description(self.scope, self.hooks, self.replica_ctx, self.execution_timeout) } fn replica_ctx(&self) -> &Arc { @@ -1997,7 +2058,6 @@ where // are released when the reducer/view/procedure returns. v8::scope!(let scope, scope); - // TODO(v8): Start the budget timeout and long-running logger. let env = env_on_isolate_unwrap(scope); // Start the timer. @@ -2014,7 +2074,12 @@ where // opened by the caller before entering `common_call`. v8::tc_scope!(let scope, scope); - let call_result = call(scope, inst.hooks, op).map_err(|mut e| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), inst.execution_timeout); + let mut call_result = match &deadline { + Ok(_) => call(scope, inst.hooks, op), + Err(error) => Err(anyhow::anyhow!("cannot start JavaScript execution deadline: {error}").into()), + } + .map_err(|mut e| { if let ErrorOrException::Exception(_) = e { // If we're terminating execution, don't try to check `instanceof`. if scope.can_continue() @@ -2034,10 +2099,7 @@ where // We can continue. ExecutionError::Recoverable(e.unwrap_or_else(Into::into)) } else if scope.has_terminated() { - // We can continue if we do `Isolate::cancel_terminate_execution`. - // Must be called *after* we check `has_terminated()`, or else it will - // cause it to return `false`. - scope.cancel_terminate_execution(); + // Reset only after synchronizing with this invocation's timer. let e = e.unwrap_or_else(|unknown| termination_error.unwrap_or_else(|| unknown.into())); ExecutionError::Recoverable(e) } else { @@ -2046,6 +2108,14 @@ where } }); + // The timer also covers exception inspection, which can execute user + // getters or Symbol.hasInstance. Synchronizing with it before resetting prevents a + // delayed timer from terminating the next invocation of this isolate. + if deadline.is_ok_and(ExecutionDeadline::finish) { + // Even a call that returned at the exact boundary must roll back. + call_result = Err(ExecutionError::Recoverable(ExecutionTimedOut.into())); + } + // Ensure there's no lingering termination request. termination_flag.clear(); scope.cancel_terminate_execution(); @@ -2086,14 +2156,22 @@ fn extract_description<'scope>( scope: &mut PinScope<'scope, '_>, hooks: &HookFunctions<'_>, replica_ctx: &ReplicaContext, + execution_timeout: Duration, ) -> Result { run_describer( |a, b, c| log_traceback(replica_ctx, a, b, c), || { - Ok(catch_exception(scope, |scope| { + let deadline = ExecutionDeadline::start(scope.thread_safe_handle(), execution_timeout)?; + let result = catch_exception(scope, |scope| { let def = call_describe_module(scope, hooks)?; Ok(def) - })?) + }); + let expired = deadline.finish(); + scope.cancel_terminate_execution(); + if expired { + return Err(ExecutionTimedOut.into()); + } + Ok(result?) }, ) } diff --git a/crates/standalone/config.toml b/crates/standalone/config.toml index 9eeef5d3535..2b58326552a 100644 --- a/crates/standalone/config.toml +++ b/crates/standalone/config.toml @@ -27,6 +27,10 @@ directives = [ # Maximum number of JS procedure isolates per database. Omit to use the number # of cores reported by the OS. # procedure-instance-pool-size = 8 +# Wall-clock bound for each JS startup, schema description and function call. +# Defaults to 120 seconds; must be positive and no greater than 120 seconds. +# Timed-out reducers fail and roll back, including lifecycle calls during publish. +# execution-timeout = "120s" [v8-heap-policy] # Check the V8 heap after this many requests. Set to 0 to disable. From d4a5dbffecaf3136764c92a98be783209595f978 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 03:01:31 -0400 Subject: [PATCH 08/23] Make Rust SDK connection cancellation release pending callbacks --- sdks/rust/src/client_cache.rs | 4 +- sdks/rust/src/db_connection.rs | 180 ++++++++++--- sdks/rust/src/db_connection/builder_tests.rs | 89 +++++++ sdks/rust/src/db_connection/terminal_tests.rs | 250 ++++++++++++++++++ sdks/rust/src/lib.rs | 3 + sdks/rust/src/subscription.rs | 40 +-- 6 files changed, 508 insertions(+), 58 deletions(-) create mode 100644 sdks/rust/src/db_connection/builder_tests.rs create mode 100644 sdks/rust/src/db_connection/terminal_tests.rs diff --git a/sdks/rust/src/client_cache.rs b/sdks/rust/src/client_cache.rs index 5222dcb06f9..afe8237ffa6 100644 --- a/sdks/rust/src/client_cache.rs +++ b/sdks/rust/src/client_cache.rs @@ -452,7 +452,9 @@ impl TableHandle { /// See [`DbContextImpl::queue_mutation`]. fn queue_mutation(&self, mutation: PendingMutation) { - self.pending_mutations.unbounded_send(mutation).unwrap(); + // A retained table handle may outlive terminal connection cleanup. + // The closed queue rejects and releases its callback captures. + let _ = self.pending_mutations.unbounded_send(mutation); } /// Called by the autogenerated implementation of the [`crate::Table`] method of the same name. diff --git a/sdks/rust/src/db_connection.rs b/sdks/rust/src/db_connection.rs index 332aac1b322..8d114f4cbce 100644 --- a/sdks/rust/src/db_connection.rs +++ b/sdks/rust/src/db_connection.rs @@ -55,6 +55,9 @@ use tokio::{ pub(crate) type SharedCell = Arc>; +#[cfg(all(test, not(feature = "browser")))] +mod builder_tests; + #[cfg(not(feature = "browser"))] type SharedAsyncCell = Arc>; #[cfg(feature = "browser")] @@ -136,6 +139,12 @@ impl DbContextImpl { /// applying its mutations to the client cache and invoking callbacks. fn process_message(&self, msg: ParsedMessage) -> crate::Result<()> { self.debug_log(|out| writeln!(out, "`process_message`: {msg:?}")); + if !self.is_active() && !matches!(&msg, ParsedMessage::Error(_)) { + // Local disconnect invalidated callbacks. Drain already received + // results without interpreting them as outstanding healthy calls; + // the socket's terminal event must still reach end_connection. + return Ok(()); + } match msg { // Error: route as a connection error if we never finished connecting, // otherwise treat it as an erroneous disconnect. @@ -317,17 +326,27 @@ impl DbContextImpl { /// /// Returns the terminal error that should be returned from `advance_*` methods. fn end_connection(&self, callback_error: Option) -> crate::Error { - let mut inner = self.inner.lock().unwrap(); let return_error = callback_error.clone().unwrap_or(crate::Error::Disconnected); + let (lifecycle, db_callbacks, mut subscriptions, connect_callback, disconnect_callback, connect_error_callback) = { + let mut inner = self.inner.lock().unwrap(); + let lifecycle = inner.connection_lifecycle; + inner.connection_lifecycle = ConnectionLifecycle::Ended; + ( + lifecycle, + std::mem::take(&mut inner.db_callbacks), + std::mem::take(&mut inner.subscriptions), + inner.on_connect.take(), + inner.on_disconnect.take(), + inner.on_connect_error.take(), + ) + }; - let lifecycle = inner.connection_lifecycle; - if lifecycle == ConnectionLifecycle::Ended { - return return_error; - } - inner.connection_lifecycle = ConnectionLifecycle::Ended; - - // Set `send_chan` to `None`, since `Self::is_active` checks that. - *self.send_chan.lock().unwrap() = None; + // Serialize request enqueueing with terminal closure. A retained + // DbConnection must not retain in-flight callback captures indefinitely. + let outgoing = self.send_chan.lock().unwrap().take(); + self.discard_pending_requests(); + drop((outgoing, db_callbacks, connect_callback)); + subscriptions.on_disconnect(&self.make_event_ctx(callback_error.clone())); match lifecycle { ConnectionLifecycle::Connecting => { @@ -335,26 +354,59 @@ impl DbContextImpl { source: InternalError::new("Connection closed before receiving the initial connection message"), }); let ctx: M::ErrorContext = self.make_event_ctx(Some(callback_error.clone())); - if let Some(connect_error_callback) = inner.on_connect_error.take() { + if let Some(connect_error_callback) = connect_error_callback { connect_error_callback(&ctx, callback_error.clone()); } callback_error } ConnectionLifecycle::Connected => { let ctx: M::ErrorContext = self.make_event_ctx(callback_error.clone()); - if let Some(disconnect_callback) = inner.on_disconnect.take() { + if let Some(disconnect_callback) = disconnect_callback { disconnect_callback(&ctx, callback_error.clone()); } - - // Call the `on_disconnect` method for all subscriptions. - inner.subscriptions.on_disconnect(&ctx); - return_error } ConnectionLifecycle::Ended => return_error, } } + fn discard_pending_requests(&self) { + let (reducer_callbacks, procedure_callbacks) = { + let mut inner = self.inner.lock().unwrap(); + ( + std::mem::take(&mut inner.reducer_callbacks), + std::mem::take(&mut inner.procedure_callbacks), + ) + }; + let mut queued = Vec::new(); + { + // There is exactly one supported advance_* caller, and get_message + // releases this guard before applying a terminal message. Avoid a + // blocking_lock here because advance_one_message_async runs in Tokio. + #[cfg(not(feature = "browser"))] + let mut pending = self + .pending_mutations_recv + .try_lock() + .expect("concurrent SDK message advancement"); + #[cfg(feature = "browser")] + let mut pending = self.pending_mutations_recv.lock().unwrap(); + pending.close(); + while let Ok(Some(mutation)) = pending.try_next() { + queued.push(mutation); + } + } + // Destructors may reenter SDK methods. Drop user captures only after + // releasing the connection and pending-queue locks. These calls have an + // unknown outcome; do not manufacture a successful reducer completion. + for mutation in queued { + if let PendingMutation::Subscribe { handle, .. } = &mutation { + handle.cancel_pending_callbacks(); + } + drop(mutation); + } + drop((reducer_callbacks, procedure_callbacks)); + } + fn make_event_ctx>(&self, event: E) -> Ctx { let imp = self.clone(); Ctx::new(imp, event) @@ -362,7 +414,9 @@ impl DbContextImpl { /// Apply all queued [`PendingMutation`]s. fn apply_pending_mutations(&self) -> crate::Result<()> { - while let Ok(Some(pending_mutation)) = get_lock_sync(&self.pending_mutations_recv).try_next() { + loop { + let pending = get_lock_sync(&self.pending_mutations_recv).try_next(); + let Ok(Some(pending_mutation)) = pending else { break }; self.apply_mutation(pending_mutation)?; } @@ -372,6 +426,16 @@ impl DbContextImpl { /// Apply an individual [`PendingMutation`]. fn apply_mutation(&self, mutation: PendingMutation) -> crate::Result<()> { self.debug_log(|out| writeln!(out, "`apply_mutation`: {mutation:?}")); + if !self.is_active() + && matches!( + &mutation, + PendingMutation::InvokeReducerWithCallback { .. } | PendingMutation::InvokeProcedureWithCallback { .. } + ) + { + // A call may have been queued behind the disconnect mutation. Drop + // its captures and continue driving the actual terminal event. + return Ok(()); + } match mutation { // Subscribe: register the subscription in the [`SubscriptionManager`] // and send the `Subscribe` WS message. @@ -491,6 +555,7 @@ impl DbContextImpl { // eventually resulting in disconnect callbacks being called // if the initial connection had completed. *self.send_chan.lock().unwrap() = None; + self.discard_pending_requests(); } // Callback stuff: these all do what you expect. @@ -605,13 +670,13 @@ impl DbContextImpl { // This may be unnecessary, but `tokio::select` does not document any ordering guarantees, // and if both `pending_mutations.next()` and `recv.next()` have values ready, // we want to process the pending mutation first. - if let Ok(pending_mutation) = pending_mutations.try_next() { - return Message::Local(pending_mutation.unwrap()); + if let Ok(Some(pending_mutation)) = pending_mutations.try_next() { + return Message::Local(pending_mutation); } #[cfg(not(feature = "browser"))] tokio::select! { - pending_mutation = pending_mutations.next() => Message::Local(pending_mutation.unwrap()), + Some(pending_mutation) = pending_mutations.next() => Message::Local(pending_mutation), incoming_message = recv.next() => Message::Ws(incoming_message), } @@ -621,7 +686,10 @@ impl DbContextImpl { pin_mut!(pending_fut, recv_fut); futures::select! { - pending_mutation = pending_fut => Message::Local(pending_mutation.unwrap()), + pending_mutation = pending_fut => match pending_mutation { + Some(pending_mutation) => Message::Local(pending_mutation), + None => Message::Ws(recv_fut.await), + }, incoming_message = recv_fut => Message::Ws(incoming_message), } } @@ -716,8 +784,7 @@ impl DbContextImpl { } self.pending_mutations_send .unbounded_send(PendingMutation::Disconnect) - .unwrap(); - Ok(()) + .map_err(|_| crate::Error::Disconnected) } /// Add a [`PendingMutation`] to the `pending_mutations` queue, @@ -725,8 +792,14 @@ impl DbContextImpl { /// /// This is used to defer operations which would otherwise need to hold a lock on `self.inner`, /// as otherwise running those operations within a callback would deadlock. - fn queue_mutation(&self, mutation: PendingMutation) { + fn queue_mutation(&self, mutation: PendingMutation) -> crate::Result<()> { + let outgoing = self.send_chan.lock().unwrap(); + if outgoing.is_none() { + drop(outgoing); + return Err(crate::Error::Disconnected); + } self.pending_mutations_send.unbounded_send(mutation).unwrap(); + Ok(()) } /// Called by autogenerated table access methods. @@ -757,8 +830,7 @@ impl DbContextImpl { self.queue_mutation(PendingMutation::InvokeReducerWithCallback { reducer: reducer.into(), callback: Box::new(callback), - }); - Ok(()) + }) } /// Called by the autogenerated `DbConnection` method of the same name. @@ -788,7 +860,7 @@ impl DbContextImpl { + Send + 'static, ) { - self.queue_mutation(PendingMutation::InvokeProcedureWithCallback { + let _ = self.queue_mutation(PendingMutation::InvokeProcedureWithCallback { procedure: procedure_name, args: bsatn::to_vec(&args).expect("Failed to BSATN serialize procedure args"), callback: Box::new(move |ctx, ret| { @@ -953,6 +1025,25 @@ but you must call one of them, or else the connection will never progress. Ok(::new(imp)) } + /// Open a connection asynchronously on the current Tokio runtime. + /// + /// Unlike [`Self::build`], the WebSocket handshake does not block this task. + /// Dropping this future while the handshake is pending closes its socket + /// and releases the builder's callbacks. Callers can impose their own + /// connection deadline with [`tokio::time::timeout`]. Background connection + /// tasks are started only after the handshake succeeds. + /// + /// Requires an active Tokio runtime. As with [`Self::build`], the returned + /// connection must be advanced explicitly to receive events. + #[cfg(not(feature = "browser"))] + pub async fn build_async(self) -> crate::Result { + let handle = runtime::Handle::try_current().map_err(|error| { + InternalError::new("DbConnectionBuilder::build_async requires a Tokio runtime").with_cause(error) + })?; + let imp = self.build_native_impl(handle).await?; + Ok(::new(imp)) + } + #[cfg(feature = "browser")] pub async fn build(self) -> crate::Result { let imp = self.build_impl().await?; @@ -963,6 +1054,19 @@ but you must call one of them, or else the connection will never progress. /// to construct a [`DbContextImpl`]. #[cfg(not(feature = "browser"))] fn build_impl(self) -> crate::Result> { + let (runtime, handle) = enter_or_create_runtime()?; + // Keep an SDK-owned runtime outside its own block_on future. If the + // handshake fails, dropping that runtime inside the future would panic. + let imp = tokio::task::block_in_place(|| handle.block_on(self.build_native_impl(handle.clone())))?; + imp.inner.lock().unwrap().runtime = runtime; + Ok(imp) + } + + /// Share native construction between the synchronous and asynchronous API. + /// The handshake is the only suspension point. Until it succeeds, this + /// future owns the socket and callbacks without spawning background tasks. + #[cfg(not(feature = "browser"))] + async fn build_native_impl(self, handle: runtime::Handle) -> crate::Result> { let extra_logging = self .additional_logging_path .map(|path| { @@ -973,18 +1077,15 @@ but you must call one of them, or else the connection will never progress. .transpose()? .map(|file| Arc::new(StdMutex::new(file))); - let (runtime, handle) = enter_or_create_runtime()?; - let connection_id_override = get_connection_id_override(); - let ws_connection = tokio::task::block_in_place(|| { - handle.block_on(WsConnection::connect( - self.uri.unwrap(), - self.database_name.as_ref().unwrap(), - self.token.as_deref(), - connection_id_override, - self.params, - )) - }) + let ws_connection = WsConnection::connect( + self.uri.unwrap(), + self.database_name.as_ref().unwrap(), + self.token.as_deref(), + connection_id_override, + self.params, + ) + .await .map_err(|source| crate::Error::FailedToConnect { source: InternalError::new("Failed to initiate WebSocket connection").with_cause(source), })?; @@ -998,7 +1099,7 @@ but you must call one of them, or else the connection will never progress. let (pending_mutations_send, pending_mutations_recv) = mpsc::unbounded(); let pending_mutations_recv = Arc::new(TokioMutex::new(pending_mutations_recv)); - let inner_ctx = build_db_ctx_inner(runtime, self.on_connect, self.on_connect_error, self.on_disconnect); + let inner_ctx = build_db_ctx_inner(None, self.on_connect, self.on_connect_error, self.on_disconnect); Ok(build_db_ctx( handle, inner_ctx, @@ -1622,3 +1723,6 @@ pub(crate) fn next_query_set_id() -> QuerySetId { id: NEXT_QUERY_SET_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed), } } + +#[cfg(all(test, not(feature = "browser")))] +mod terminal_tests; diff --git a/sdks/rust/src/db_connection/builder_tests.rs b/sdks/rust/src/db_connection/builder_tests.rs new file mode 100644 index 00000000000..7158c02b9e5 --- /dev/null +++ b/sdks/rust/src/db_connection/builder_tests.rs @@ -0,0 +1,89 @@ +use super::terminal_tests::bindings::RemoteModule; +use super::*; +use std::{net::SocketAddr, time::Duration}; +use tokio::{io::AsyncReadExt, net::TcpListener, sync::oneshot, task::JoinHandle}; + +async fn stalled_handshake_peer() -> (SocketAddr, oneshot::Receiver<()>, JoinHandle<()>) { + // This owned numeric-loopback listener never completes the HTTP upgrade. + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (ready_tx, ready_rx) = oneshot::channel(); + let peer = tokio::spawn(async move { + let (mut socket, remote) = listener.accept().await.unwrap(); + assert!(remote.ip().is_loopback()); + let mut request = Vec::new(); + let mut buffer = [0u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = socket.read(&mut buffer).await.unwrap(); + assert_ne!(count, 0, "client closed before sending the upgrade request"); + request.extend_from_slice(&buffer[..count]); + assert!(request.len() <= 16 * 1024); + } + ready_tx.send(()).unwrap(); + assert_eq!( + socket.read(&mut buffer).await.unwrap(), + 0, + "pending handshake socket was retained" + ); + }); + (address, ready_rx, peer) +} + +async fn check_pending_handshake_cleanup(use_timeout: bool) { + let (address, ready, peer) = stalled_handshake_peer().await; + let capture = Arc::new(()); + let weak_capture = Arc::downgrade(&capture); + let mut connect = Box::pin( + DbConnectionBuilder::::new() + .with_uri(format!("http://{address}")) + .with_database_name("disposable-builder-handshake-test") + .on_connect(move |_, _, _| drop(capture)) + .build_async(), + ); + tokio::time::timeout(Duration::from_secs(2), async { + tokio::select! { + result = &mut connect => panic!("stalled handshake unexpectedly completed: {:?}", result.err()), + result = ready => result.unwrap(), + } + }) + .await + .expect("async builder blocked the executor or never initiated its handshake"); + assert!(weak_capture.upgrade().is_some()); + + if use_timeout { + assert!(tokio::time::timeout(Duration::from_millis(30), connect).await.is_err()); + } else { + drop(connect); + } + assert!( + weak_capture.upgrade().is_none(), + "cancelled builder retained callback captures" + ); + tokio::time::timeout(Duration::from_secs(2), peer) + .await + .expect("cancelled handshake did not close its socket") + .unwrap(); +} + +// A current-thread runtime also proves the asynchronous builder does not use +// block_in_place or block_on while waiting for the peer's handshake response. +#[tokio::test] +async fn async_builder_timeout_closes_stalled_handshake_and_releases_callbacks() { + check_pending_handshake_cleanup(true).await; +} + +#[tokio::test] +async fn async_builder_cancellation_closes_stalled_handshake_and_releases_callbacks() { + check_pending_handshake_cleanup(false).await; +} + +#[test] +fn sync_builder_returns_connect_errors_without_dropping_its_runtime_inside_block_on() { + // The SDK rejects a query in the host URI before attempting any network I/O. + // Running outside Tokio exercises the synchronous API's owned runtime. + let result = DbConnectionBuilder::::new() + .with_uri("http://127.0.0.1:1/?unexpected=query") + .with_database_name("disposable-builder-error-test") + .build(); + assert!(matches!(result, Err(crate::Error::FailedToConnect { .. }))); +} diff --git a/sdks/rust/src/db_connection/terminal_tests.rs b/sdks/rust/src/db_connection/terminal_tests.rs new file mode 100644 index 00000000000..397c36a34ae --- /dev/null +++ b/sdks/rust/src/db_connection/terminal_tests.rs @@ -0,0 +1,250 @@ +use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[path = "../../tests/connect_disconnect_client/src/module_bindings/mod.rs"] +pub(super) mod bindings; +use bindings::RemoteModule; + +#[derive(spacetimedb_lib::ser::Serialize)] +#[sats(crate = spacetimedb_lib)] +struct Args {} +impl InModule for Args { + type Module = RemoteModule; +} +impl From for bindings::Reducer { + fn from(_: Args) -> Self { + Self::IdentityConnected + } +} + +struct DropProbe { + context: DbContextImpl, + drops: Arc, +} +impl Drop for DropProbe { + fn drop(&mut self) { + assert!( + self.context.inner.try_lock().is_ok(), + "capture dropped under inner lock" + ); + assert!( + self.context.send_chan.try_lock().is_ok(), + "capture dropped under send lock" + ); + assert!( + self.context.pending_mutations_recv.try_lock().is_ok(), + "capture dropped under queue lock" + ); + self.drops.fetch_add(1, Ordering::SeqCst); + } +} + +fn fixture( + runtime: &Runtime, + disconnects: Arc, +) -> ( + DbContextImpl, + mpsc::UnboundedSender>, +) { + let inner = build_db_ctx_inner::( + None, + None, + None, + Some(Box::new(move |_, error| { + assert!(error.is_none()); + disconnects.fetch_add(1, Ordering::SeqCst); + })), + ); + inner.lock().unwrap().connection_lifecycle = ConnectionLifecycle::Connected; + let (outgoing, outgoing_recv) = mpsc::unbounded(); + // Keep the transport receiver alive without any network or server fixture. + let (incoming, incoming_recv) = mpsc::unbounded(); + let (pending, pending_recv) = mpsc::unbounded(); + let context = build_db_ctx( + runtime.handle().clone(), + inner, + outgoing, + Arc::new(TokioMutex::new(incoming_recv)), + pending, + Arc::new(TokioMutex::new(pending_recv)), + Some(ConnectionId::from_u128(1)), + None, + ); + runtime.spawn(async move { + let mut outgoing_recv = outgoing_recv; + while outgoing_recv.next().await.is_some() {} + }); + (context, incoming) +} + +fn queue_calls(context: &DbContextImpl, drops: &Arc) { + let reducer = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context + .invoke_reducer_with_callback(Args {}, move |_, _| { + let _capture = reducer; + panic!("unknown-outcome reducer must not report completion"); + }) + .unwrap(); + let procedure = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context.invoke_procedure_with_callback::<_, ()>("procedure", Args {}, move |_, _| { + let _capture = procedure; + panic!("unknown-outcome procedure must not report completion"); + }); +} + +#[test] +fn terminal_disconnect_releases_inflight_and_queued_requests_with_retained_context() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, _incoming) = fixture(&runtime, disconnects.clone()); + queue_calls(&context, &drops); + context.frame_tick().unwrap(); + queue_calls(&context, &drops); + assert_eq!(drops.load(Ordering::SeqCst), 0); + assert!(matches!(context.end_connection(None), crate::Error::Disconnected)); + assert_eq!(drops.load(Ordering::SeqCst), 4); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); + assert!(!context.is_active()); + + let late = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + assert!(matches!( + context.invoke_reducer_with_callback(Args {}, move |_, _| drop(late)), + Err(crate::Error::Disconnected) + )); + let late = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + context.invoke_procedure_with_callback::<_, ()>("procedure", Args {}, move |_, _| drop(late)); + assert_eq!(drops.load(Ordering::SeqCst), 6); + context.end_connection(None); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); +} + +#[test] +fn queued_disconnect_releases_calls_and_drains_late_results_until_terminal_event() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, incoming) = fixture(&runtime, disconnects.clone()); + queue_calls(&context, &drops); + context.frame_tick().unwrap(); + context.disconnect().unwrap(); + queue_calls(&context, &drops); + incoming + .unbounded_send(ParsedMessage::ReducerResult { + request_id: u32::MAX, + timestamp: Timestamp::UNIX_EPOCH, + result: Ok(Ok(bindings::DbUpdate::default())), + }) + .unwrap(); + incoming + .unbounded_send(ParsedMessage::ProcedureResult { + request_id: u32::MAX, + result: Ok(Bytes::new()), + }) + .unwrap(); + context.frame_tick().unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 4); + assert_eq!( + disconnects.load(Ordering::SeqCst), + 0, + "local request is not the terminal callback" + ); + assert!(!context.is_active()); + drop(incoming); + assert!(matches!( + runtime.block_on(context.advance_one_message_async()), + Err(crate::Error::Disconnected) + )); + assert_eq!(disconnects.load(Ordering::SeqCst), 1); +} + +#[test] +fn retained_table_and_subscription_handles_cannot_retain_callbacks_after_terminal() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, _incoming) = fixture(&runtime, disconnects); + let table = context.get_table::("connected"); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + table.on_insert(move |_, _| { + let _ = &capture; + }); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let registered = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.frame_tick().unwrap(); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let queued = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.end_connection(None); + assert_eq!(drops.load(Ordering::SeqCst), 3); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(®istered)); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&queued)); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + table.on_insert(move |_, _| { + let _ = &capture; + }); + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let subscription = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + assert_eq!(drops.load(Ordering::SeqCst), 5); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&subscription)); + assert_eq!(table.iter().count(), 0); +} + +#[test] +fn cancelling_before_initial_message_releases_registered_subscription_without_callbacks() { + let runtime = Runtime::new().unwrap(); + let disconnects = Arc::new(AtomicUsize::new(0)); + let drops = Arc::new(AtomicUsize::new(0)); + let (context, incoming) = fixture(&runtime, disconnects.clone()); + context.inner.lock().unwrap().connection_lifecycle = ConnectionLifecycle::Connecting; + let capture = DropProbe { + context: context.clone(), + drops: drops.clone(), + }; + let subscription = crate::subscription::SubscriptionBuilder::::new(&context) + .on_applied(move |_| drop(capture)) + .subscribe("SELECT * FROM connected"); + context.frame_tick().unwrap(); + context.disconnect().unwrap(); + context.frame_tick().unwrap(); + drop(incoming); + assert!(matches!( + runtime.block_on(context.advance_one_message_async()), + Err(crate::Error::Disconnected) + )); + assert_eq!(disconnects.load(Ordering::SeqCst), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert!(crate::spacetime_module::SubscriptionHandle::is_ended(&subscription)); +} diff --git a/sdks/rust/src/lib.rs b/sdks/rust/src/lib.rs index d99447d440f..2a8ec759600 100644 --- a/sdks/rust/src/lib.rs +++ b/sdks/rust/src/lib.rs @@ -10,6 +10,9 @@ // code generated by the CLI's codegen references them, // but users should not. +#[cfg(test)] +extern crate self as spacetimedb_sdk; + mod callbacks; mod client_cache; mod compression; diff --git a/sdks/rust/src/subscription.rs b/sdks/rust/src/subscription.rs index 2204985ea15..3fb4be3b5b2 100644 --- a/sdks/rust/src/subscription.rs +++ b/sdks/rust/src/subscription.rs @@ -47,23 +47,11 @@ pub(crate) enum PendingUnsubscribeResult { impl SubscriptionManager { pub(crate) fn on_disconnect(&mut self, _ctx: &M::ErrorContext) { - // We need to clear all the subscriptions. - // TODO: is this correct? We don't remove them from the client cache, - // we may want to resume them in the future if we impl reconnecting, - // and users can already register on-disconnect callbacks which will run in this case. - - // NOTE(cloutiertyler) - // This function previously invoke `on_error` for all subscriptions. - // However, this is inconsistent behavior given that `on_disconnect` for - // connections no longer always has an error argument and that the user - // can add an `on_ended` callback when unsubscribing. - // - // We propose instead that `on_ended` be added to the subscription - // builder so that it can be invoked when the subscription is ended - // because of a normal disconnect, but without the user calling - // `unsubscribe_then`. This can be done in a non-breaking way. - // - // For now, we will just do nothing when a subscription ends normally. + // Disconnect does not synthesize on_error/on_ended callbacks. Retained + // handles must nevertheless stop retaining callback captures. + for (_, handle) in std::mem::take(&mut self.subscriptions) { + handle.cancel_pending_callbacks(); + } } /// Register a new subscription. This does not send the subscription to the server. @@ -187,13 +175,17 @@ impl SubscriptionBuilder { self.on_applied, self.on_error, )); - self.conn + if self + .conn .pending_mutations_send .unbounded_send(PendingMutation::Subscribe { query_set_id, handle: handle.clone(), }) - .unwrap(); + .is_err() + { + handle.cancel_pending_callbacks(); + } M::SubscriptionHandle::new(handle) } @@ -471,6 +463,16 @@ impl SubscriptionHandleImpl { } } + pub(crate) fn cancel_pending_callbacks(&self) { + let callbacks = { + let mut inner = self.inner.lock().unwrap(); + inner.status = SubscriptionServerState::Ended; + (inner.on_applied.take(), inner.on_error.take(), inner.on_ended.take()) + }; + // Callback destructors may inspect or unsubscribe this same handle. + drop(callbacks); + } + pub(crate) fn start(&self) -> Option { let mut inner = self.inner.lock().unwrap(); inner.start() From f6a2cef7438b32bdbc20cebc1f336b89fb504b60 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 03:51:26 -0400 Subject: [PATCH 09/23] Retain immutable container environment snapshots behind host-only access --- crates/core/src/db/container_environment.rs | 354 +++++++++++++++ .../src/db/container_environment/tests.rs | 427 ++++++++++++++++++ crates/core/src/db/mod.rs | 1 + crates/core/src/host/container_environment.rs | 161 +++++++ crates/core/src/host/instance_env.rs | 5 +- crates/core/src/host/mod.rs | 1 + crates/core/src/sql/ast.rs | 3 + .../src/locking_tx_datastore/datastore.rs | 10 +- crates/datastore/src/system_tables.rs | 7 +- .../datastore/src/system_tables/deployment.rs | 46 +- crates/lib/src/container_environment.rs | 40 ++ crates/lib/src/lib.rs | 1 + 12 files changed, 1045 insertions(+), 11 deletions(-) create mode 100644 crates/core/src/db/container_environment.rs create mode 100644 crates/core/src/db/container_environment/tests.rs create mode 100644 crates/core/src/host/container_environment.rs create mode 100644 crates/lib/src/container_environment.rs diff --git a/crates/core/src/db/container_environment.rs b/crates/core/src/db/container_environment.rs new file mode 100644 index 00000000000..def8e8569ed --- /dev/null +++ b/crates/core/src/db/container_environment.rs @@ -0,0 +1,354 @@ +//! Transactional immutable container environment snapshots. +//! +//! These are host-only operations. Before invoking them, the adapter must +//! authenticate the assigned service and confirm the exact current control +//! intent and authoritative leader. Identity equality and a cached control row +//! are insufficient. Helpers retain the caller's transaction and do no IO. +//! +//! Historical restore must close admission and reconcile newer operational +//! fences before any snapshot request. A missing Ready snapshot is an error, +//! never permission to recapture values from a restored or current `st_env`. + +use super::{ + deployment, environment, + relational_db::{MutTx, RelationalDB}, +}; +use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; +use spacetimedb_datastore::system_tables::{ + StContainerEnvironmentRow, StContainerFenceRow, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, +}; +use spacetimedb_lib::container::{validate_env_key, validate_exec_size, ContainerSpec, MAX_ENV_KEYS}; +use spacetimedb_lib::container_environment::{EnvironmentSnapshotReceipt, EnvironmentSnapshotScope}; +use spacetimedb_lib::{bsatn, SpacetimeType, Uuid}; +use spacetimedb_primitives::ColId; +use spacetimedb_sats::AlgebraicValue; +use std::{collections::BTreeMap, fmt}; + +pub const MAX_SNAPSHOT_BYTES: usize = 256 * 1024; +pub const MAX_RETAINED_SNAPSHOTS: u64 = 64; + +/// Error text and Debug never include selected values or serialized records. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum EnvironmentSnapshotError { + #[error("container environment request is invalid")] + InvalidScope, + #[error("container environment generation is fenced")] + Fenced, + #[error("container environment deployment revision does not match")] + RevisionConflict, + #[error("container environment snapshot belongs to another immutable scope")] + ScopeConflict, + #[error("container environment snapshot has not been captured")] + NotCaptured, + #[error("required container environment keys are missing: {0:?}")] + MissingKeys(Vec), + #[error("container environment does not satisfy startup limits")] + InvalidEnvironment, + #[error("container environment snapshot capacity exhausted")] + Capacity, + #[error("container environment snapshot metadata is invalid")] + CorruptMetadata, + #[error("container environment storage operation failed")] + Storage, + #[error("container environment requires durable storage")] + DurabilityUnavailable, + #[error("container environment durability could not be confirmed")] + DurabilityFailed, +} + +impl From for EnvironmentSnapshotError { + fn from(_: crate::error::DBError) -> Self { + Self::Storage + } +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +enum Record { + V1(RecordV1), +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct RecordV1 { + receipt: EnvironmentSnapshotReceipt, + selected_values: Vec, +} + +#[derive(Clone, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +struct CapturedValue { + key: String, + value: String, +} + +/// Selected database values only. The trusted adapter merges verified image +/// defaults and current platform variables, then validates the complete exec. +pub struct SecretEnvironment { + pub receipt: EnvironmentSnapshotReceipt, + pub selected_values: BTreeMap, +} + +impl fmt::Debug for SecretEnvironment { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SecretEnvironment") + .field("receipt", &self.receipt) + .field("selected_values", &"[redacted]") + .finish() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnvironmentClosedReceipt { + pub scope: EnvironmentSnapshotScope, + pub closed_through_generation: u64, +} + +fn valid_uuid(id: Uuid) -> bool { + matches!( + id.get_version(), + Some(spacetimedb_sats::uuid::Version::V4 | spacetimedb_sats::uuid::Version::V7) + ) +} + +fn validate_scope(db: &RelationalDB, scope: &EnvironmentSnapshotScope) -> Result<(), EnvironmentSnapshotError> { + if db.database_identity() != scope.database_identity + || scope.database_id == 0 + || scope.node_id == 0 + || scope.generation == 0 + || scope.cluster.is_empty() + || scope.cluster.len() > 256 + || scope.cluster.contains('\0') + || !valid_uuid(scope.node_incarnation) + || !valid_uuid(scope.start_request) + || !valid_uuid(scope.env_generation) + || scope.env_keys.len() > MAX_ENV_KEYS + || scope.env_keys.windows(2).any(|keys| keys[0] >= keys[1]) + || scope.env_keys.iter().any(|key| validate_env_key(key).is_err()) + { + return Err(EnvironmentSnapshotError::InvalidScope); + } + Ok(()) +} + +fn fence( + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result { + state + .iter_by_col_eq( + ST_CONTAINER_FENCE_ID, + ColId(0), + &AlgebraicValue::U256(scope.database_identity.to_u256().into()), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .map(StContainerFenceRow::try_from) + .transpose() + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::Fenced) +} + +fn admitted_spec( + db: &RelationalDB, + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result { + validate_scope(db, scope)?; + let current = fence(state, scope)?; + if current.generation != scope.generation || !current.allowed { + return Err(EnvironmentSnapshotError::Fenced); + } + let (revision, deployment) = deployment::current_deployment(state) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if revision != scope.deployment_revision { + return Err(EnvironmentSnapshotError::RevisionConflict); + } + let spec = deployment + .current() + .container + .as_ref() + .ok_or(EnvironmentSnapshotError::RevisionConflict)?; + if spec.env_keys != scope.env_keys { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + Ok(spec.clone()) +} + +fn lookup( + state: &impl StateView, + scope: &EnvironmentSnapshotScope, +) -> Result, EnvironmentSnapshotError> { + let Some(row) = state + .iter_by_col_eq( + ST_CONTAINER_ENVIRONMENT_ID, + ColId(0), + &AlgebraicValue::U64(scope.generation), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .map(StContainerEnvironmentRow::try_from) + .transpose() + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)? + else { + return Ok(None); + }; + if row.payload.len() > MAX_SNAPSHOT_BYTES { + return Err(EnvironmentSnapshotError::CorruptMetadata); + } + let Record::V1(record) = bsatn::from_slice(&row.payload).map_err(|_| EnvironmentSnapshotError::CorruptMetadata)?; + if record.receipt.scope != *scope { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + if !valid_uuid(record.receipt.capture_receipt) + || !record + .selected_values + .iter() + .map(|entry| &entry.key) + .eq(scope.env_keys.iter()) + { + return Err(EnvironmentSnapshotError::CorruptMetadata); + } + Ok(Some(record)) +} + +fn validate_values(spec: &ContainerSpec, values: &BTreeMap) -> Result<(), EnvironmentSnapshotError> { + // The at-most-256 individually bounded values also bound this temporary allocation. + if values + .values() + .any(|value| value.contains('\0') || spacetimedb_lib::environment::validate_value(value).is_err()) + { + return Err(EnvironmentSnapshotError::InvalidEnvironment); + } + let env = values + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>(); + validate_exec_size(&spec.argv, &env).map_err(|_| EnvironmentSnapshotError::InvalidEnvironment) +} + +/// Capture exactly once under an open confirmed control intent. Returning this +/// receipt is not a durability acknowledgment; the host wrapper supplies that. +pub fn capture( + db: &RelationalDB, + tx: &mut MutTx, + scope: &EnvironmentSnapshotScope, +) -> Result { + let spec = admitted_spec(db, tx, scope)?; + if let Some(record) = lookup(tx, scope)? { + validate_values( + &spec, + &record + .selected_values + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect(), + )?; + return Ok(record.receipt); + } + if tx + .table_row_count(ST_CONTAINER_ENVIRONMENT_ID) + .ok_or(EnvironmentSnapshotError::Storage)? + >= MAX_RETAINED_SNAPSHOTS + { + return Err(EnvironmentSnapshotError::Capacity); + } + let mut values = BTreeMap::new(); + let mut missing = Vec::new(); + for key in &scope.env_keys { + match environment::get(tx, key).map_err(|_| EnvironmentSnapshotError::Storage)? { + Some(value) => { + values.insert(key.clone(), value); + } + None => missing.push(key.clone()), + } + } + if !missing.is_empty() { + return Err(EnvironmentSnapshotError::MissingKeys(missing)); + } + validate_values(&spec, &values)?; + let receipt = EnvironmentSnapshotReceipt { + scope: scope.clone(), + capture_receipt: Uuid::from_u128(uuid::Uuid::new_v4().as_u128()), + }; + let payload = bsatn::to_vec(&Record::V1(RecordV1 { + receipt: receipt.clone(), + selected_values: values + .into_iter() + .map(|(key, value)| CapturedValue { key, value }) + .collect(), + })) + .map_err(|_| EnvironmentSnapshotError::CorruptMetadata)?; + if payload.len() > MAX_SNAPSHOT_BYTES { + return Err(EnvironmentSnapshotError::Capacity); + } + tx.insert_via_serialize_bsatn( + ST_CONTAINER_ENVIRONMENT_ID, + &StContainerEnvironmentRow { + generation: scope.generation, + payload: payload.into(), + }, + ) + .map_err(|_| EnvironmentSnapshotError::Storage)?; + Ok(receipt) +} + +/// Read only an existing receipt after fresh exact control/lease confirmation. +pub fn read( + db: &RelationalDB, + state: &impl StateView, + receipt: &EnvironmentSnapshotReceipt, +) -> Result { + let spec = admitted_spec(db, state, &receipt.scope)?; + let record = lookup(state, &receipt.scope)?.ok_or(EnvironmentSnapshotError::NotCaptured)?; + if record.receipt != *receipt { + return Err(EnvironmentSnapshotError::ScopeConflict); + } + let selected_values = record + .selected_values + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect(); + validate_values(&spec, &selected_values)?; + Ok(SecretEnvironment { + receipt: record.receipt, + selected_values, + }) +} + +/// Delete only after a newer own-source fence irreversibly rejects every old +/// capture. The adapter also confirms positive historical control closure. +/// Missing rows remain idempotently closed; no per-UUID TTL is an authority. +pub fn close( + db: &RelationalDB, + tx: &mut MutTx, + scope: &EnvironmentSnapshotScope, + closed_through_generation: u64, +) -> Result { + validate_scope(db, scope)?; + let current = fence(tx, scope)?; + if closed_through_generation <= scope.generation || current.generation < closed_through_generation { + return Err(EnvironmentSnapshotError::Fenced); + } + if lookup(tx, scope)?.is_some() { + let pointer = tx + .iter_by_col_eq( + ST_CONTAINER_ENVIRONMENT_ID, + ColId(0), + &AlgebraicValue::U64(scope.generation), + ) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .next() + .ok_or(EnvironmentSnapshotError::Storage)? + .pointer(); + db.delete(tx, ST_CONTAINER_ENVIRONMENT_ID, [pointer]); + } + Ok(EnvironmentClosedReceipt { + scope: scope.clone(), + closed_through_generation, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/db/container_environment/tests.rs b/crates/core/src/db/container_environment/tests.rs new file mode 100644 index 00000000000..45a91d21cfe --- /dev/null +++ b/crates/core/src/db/container_environment/tests.rs @@ -0,0 +1,427 @@ +use super::*; +use crate::db::deployment::{ + install_container_fence, install_publication_fence, record_deployment_commit, DeploymentCommit, +}; +use crate::db::relational_db::tests_utils::TestDB; +use crate::host::container_environment as host; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_lib::container::*; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; +use spacetimedb_lib::{hash_bytes, Identity, Timestamp}; +use std::sync::{Arc, Barrier}; + +fn uuid() -> Uuid { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) +} + +fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: keys, + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } + .normalize(&Default::default()) + .unwrap(); + let request = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: db.owner_identity(), + expected_revision: None, + prepared_manifest_hash: hash_bytes(b"prepared"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: Some(spec.clone()), + }), + }; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + record_deployment_commit(tx, &request, Timestamp::now(), &Default::default())?; + install_container_fence(db, tx, &self_fence(db, 1, true))?; + for key in &spec.env_keys { + environment::set(db, tx, key, "before")?; + } + Ok(()) + }) + .unwrap(); + EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: 1, + database_identity: db.database_identity(), + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: request.deployment.revision().unwrap(), + start_request: request.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + } +} + +fn self_fence(db: &RelationalDB, generation: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed, + } +} + +fn tx( + db: &RelationalDB, + action: impl FnOnce(&mut MutTx) -> Result, +) -> Result { + db.with_auto_commit(Workload::ForTests, action) +} + +#[test] +fn container_environment_capture_retry_read_and_durable_reopen() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + let captured = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), scope.clone())) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "A", "changed")?; + environment::delete(&db, tx, "B")?; + Ok(()) + }) + .unwrap(); + let db = db.reopen().unwrap(); + let retried = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), scope)) + .unwrap(); + assert_eq!(retried.receipt, captured.receipt); + assert!(retried.durable_through >= captured.durable_through); + let values = db + .runtime() + .unwrap() + .block_on(host::read(db.db.clone(), retried.receipt)) + .unwrap() + .receipt; + assert_eq!( + values.selected_values, + BTreeMap::from([("A".into(), "before".into()), ("B".into(), "before".into())]) + ); + assert!(!format!("{values:?}").contains("before")); +} + +#[test] +fn container_environment_capture_is_atomic_against_concurrent_environment_mutation() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + let barrier = Arc::new(Barrier::new(2)); + let writer = { + let db = db.db.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "A", "after")?; + environment::set(&db, tx, "B", "after")?; + Ok(()) + }) + .unwrap(); + }) + }; + barrier.wait(); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + writer.join().unwrap(); + let values = db + .with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap(); + assert_eq!(values.selected_values["A"], values.selected_values["B"]); +} + +#[test] +fn container_environment_closure_fences_delayed_capture_and_new_instance_observes_new_values() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let old = setup(&db, vec!["A".into()]); + let captured = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), old.clone())) + .unwrap() + .receipt; + assert_eq!( + tx(&db, |tx| close(&db, tx, &old, 1)), + Err(EnvironmentSnapshotError::Fenced) + ); + assert_eq!( + tx(&db, |tx| close(&db, tx, &old, 2)), + Err(EnvironmentSnapshotError::Fenced) + ); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 2, true))?; + environment::set(&db, tx, "A", "new-boot")?; + Ok(()) + }) + .unwrap(); + db.runtime() + .unwrap() + .block_on(host::close(db.db.clone(), old.clone(), 2)) + .unwrap(); + let db = db.reopen().unwrap(); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &old)), + Err(EnvironmentSnapshotError::Fenced) + ); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &captured)), + Err(EnvironmentSnapshotError::Fenced) + )); + db.runtime() + .unwrap() + .block_on(host::close(db.db.clone(), old.clone(), 2)) + .unwrap(); + let new = EnvironmentSnapshotScope { + generation: 2, + env_generation: uuid(), + ..old + }; + let receipt = db + .runtime() + .unwrap() + .block_on(host::capture(db.db.clone(), new)) + .unwrap() + .receipt; + assert_ne!(receipt.capture_receipt, captured.capture_receipt); + assert_eq!( + db.runtime() + .unwrap() + .block_on(host::read(db.db.clone(), receipt)) + .unwrap() + .receipt + .selected_values["A"], + "new-boot" + ); +} + +#[test] +fn container_environment_full_scope_conflicts_and_missing_ready_record_fail_closed() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + let receipt = tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let mut variants = Vec::new(); + let mut changed = scope.clone(); + changed.cluster = "other".into(); + variants.push(changed); + let mut changed = scope.clone(); + changed.database_id += 1; + variants.push(changed); + let mut changed = scope.clone(); + changed.node_id += 1; + variants.push(changed); + let mut changed = scope.clone(); + changed.node_incarnation = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.start_request = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.env_generation = uuid(); + variants.push(changed); + let mut changed = scope.clone(); + changed.env_keys.clear(); + variants.push(changed); + for changed in variants { + assert_eq!( + tx(&db, |tx| capture(&db, tx, &changed)), + Err(EnvironmentSnapshotError::ScopeConflict) + ); + } + let mut changed = scope.clone(); + changed.deployment_revision = hash_bytes(b"other"); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &changed)), + Err(EnvironmentSnapshotError::RevisionConflict) + ); + let mut fork = scope; + fork.database_identity = Identity::from_u256(99u64.into()); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &fork)), + Err(EnvironmentSnapshotError::InvalidScope) + ); + // Simulate a restored/lost Ready record. Resolve may not reinterpret its UUID. + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + tx.clear_table(ST_CONTAINER_ENVIRONMENT_ID)?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)), + Err(EnvironmentSnapshotError::NotCaptured) + )); +} + +#[test] +fn container_environment_missing_empty_invalid_and_capacity_are_atomic_and_redacted() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into(), "B".into()]); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::delete(&db, tx, "B")?; + Ok(()) + }) + .unwrap(); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::MissingKeys(vec!["B".into()])) + ); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "B", "secret\0value")?; + Ok(()) + }) + .unwrap(); + let failure = tx(&db, |tx| capture(&db, tx, &scope)).unwrap_err(); + assert_eq!(failure, EnvironmentSnapshotError::InvalidEnvironment); + assert!(!format!("{failure:?}: {failure}").contains("secret")); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + environment::set(&db, tx, "B", "")?; + Ok(()) + }) + .unwrap(); + for generation in 1..=MAX_RETAINED_SNAPSHOTS { + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, generation, true))?; + Ok(()) + }) + .unwrap(); + let request = EnvironmentSnapshotScope { + generation, + env_generation: uuid(), + ..scope.clone() + }; + let receipt = tx(&db, |tx| capture(&db, tx, &request)).unwrap(); + assert_eq!( + db.with_read_only(Workload::ForTests, |state| read(&db, state, &receipt)) + .unwrap() + .selected_values["B"], + "" + ); + } + let generation = MAX_RETAINED_SNAPSHOTS + 1; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, generation, true))?; + Ok(()) + }) + .unwrap(); + assert_eq!( + tx(&db, |tx| capture( + &db, + tx, + &EnvironmentSnapshotScope { generation, ..scope } + )), + Err(EnvironmentSnapshotError::Capacity) + ); +} + +#[test] +fn container_environment_history_is_hidden_from_privileged_sql_and_subscriptions() { + use crate::sql::ast::SchemaViewer; + use spacetimedb_expr::check::SchemaView; + use spacetimedb_lib::identity::AuthCtx; + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + tx(&db, |tx| capture(&db, tx, &scope)).unwrap(); + let auth = AuthCtx::for_current(db.owner_identity()); + db.with_read_only(Workload::ForTests, |state| { + let schema = SchemaViewer::new(state, &auth); + assert!(schema.schema_for_table(ST_CONTAINER_ENVIRONMENT_ID).is_none()); + assert!(schema.table_id("st_container_environment").is_none()); + for sql in [ + "SELECT * FROM st_container_environment", + "SELECT h.* FROM st_container_environment h JOIN st_env e ON h.generation = 1", + "DELETE FROM st_container_environment", + "UPDATE st_container_environment SET generation = 2", + ] { + assert!( + spacetimedb_query::compile_sql_stmt(sql, &schema, &auth).is_err(), + "{sql}" + ); + } + assert!(spacetimedb_query::compile_sql_stmt("SELECT * FROM st_env", &schema, &auth).is_ok()); + assert!(crate::subscription::query::compile_read_only_query( + &auth, + state, + "SELECT * FROM st_container_environment" + ) + .is_err()); + assert!(crate::subscription::subscription::get_all( + |db, tx| db.get_all_tables(tx).map(Vec::into_iter), + &db, + state, + &auth + ) + .unwrap() + .is_empty()); + }); +} + +#[test] +fn container_environment_host_api_requires_durability() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec![]); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + assert!(matches!( + rt.block_on(host::capture(db.db.clone(), scope)), + Err(EnvironmentSnapshotError::DurabilityUnavailable) + )); +} + +#[test] +fn container_environment_concurrent_closure_cannot_reopen_collected_generation() { + let db = TestDB::in_memory().unwrap(); + let scope = setup(&db, vec!["A".into()]); + let barrier = Arc::new(Barrier::new(2)); + let capture_thread = { + let db = db.db.clone(); + let scope = scope.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + tx(&db, |tx| capture(&db, tx, &scope)) + }) + }; + barrier.wait(); + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_container_fence(&db, tx, &self_fence(&db, 2, false))?; + close(&db, tx, &scope, 2)?; + Ok(()) + }) + .unwrap(); + assert!(matches!( + capture_thread.join().unwrap(), + Ok(_) | Err(EnvironmentSnapshotError::Fenced) + )); + assert_eq!( + tx(&db, |tx| capture(&db, tx, &scope)), + Err(EnvironmentSnapshotError::Fenced) + ); + db.with_read_only(Workload::ForTests, |state| { + assert_eq!(state.table_row_count(ST_CONTAINER_ENVIRONMENT_ID), Some(0)) + }); +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index e206ffba2fa..13b6b10f7f0 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -9,6 +9,7 @@ use crate::subscription::ExecutionCounters; use spacetimedb_datastore::execution_context::WorkloadType; use spacetimedb_datastore::{locking_tx_datastore::datastore::TxMetrics, traits::TxData}; +pub mod container_environment; pub mod deployment; mod durability; pub mod environment; diff --git a/crates/core/src/host/container_environment.rs b/crates/core/src/host/container_environment.rs new file mode 100644 index 00000000000..e3afd2da0e4 --- /dev/null +++ b/crates/core/src/host/container_environment.rs @@ -0,0 +1,161 @@ +//! Bounded, durable host operations for immutable container environments. +//! +//! The caller must confirm current control authority and authoritative leader +//! before calling. These local host APIs are not an external authentication +//! interface. Historical restore must keep admission closed until current +//! operational fences have been reconciled. No in-memory production fallback. + +use crate::db::container_environment::{ + self as storage, EnvironmentClosedReceipt, EnvironmentSnapshotError, SecretEnvironment, +}; +use crate::db::relational_db::{MutTx, RelationalDB}; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::container_environment::{EnvironmentSnapshotReceipt, EnvironmentSnapshotScope}; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Semaphore; + +/// Bounds queued/blocked transactions even when the async caller is cancelled. +static OPERATIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(8))); + +/// The barrier is specific to this proof. It can advance on an exact retry. +#[derive(Debug)] +pub struct Durable { + pub receipt: T, + pub durable_through: u64, +} + +pub type DurableSnapshotReceipt = Durable; +pub type DurableEnvironmentValues = Durable; +pub type DurableClosedReceipt = Durable; + +pub async fn capture( + db: Arc, + scope: EnvironmentSnapshotScope, +) -> Result { + let action_db = db.clone(); + mutate(db, move |tx| storage::capture(&action_db, tx, &scope)).await +} + +pub async fn read( + db: Arc, + receipt: EnvironmentSnapshotReceipt, +) -> Result { + let mut durability = db + .durable_tx_offset() + .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; + let permit = OPERATIONS + .clone() + .try_acquire_owned() + .map_err(|_| EnvironmentSnapshotError::Capacity)?; + let (_permit, durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = db.begin_tx(Workload::Internal); + let result = storage::read(&db, &tx, &receipt); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + result.map(|result| (permit, offset, result)) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + Ok(Durable { + receipt, + durable_through, + }) +} + +pub async fn close( + db: Arc, + scope: EnvironmentSnapshotScope, + closed_through_generation: u64, +) -> Result { + let action_db = db.clone(); + mutate(db, move |tx| { + storage::close(&action_db, tx, &scope, closed_through_generation) + }) + .await +} + +async fn mutate( + db: Arc, + action: impl FnOnce(&mut MutTx) -> Result + Send + 'static, +) -> Result, EnvironmentSnapshotError> { + mutate_with_capacity(db, OPERATIONS.clone(), action).await +} + +async fn mutate_with_capacity( + db: Arc, + capacity: Arc, + action: impl FnOnce(&mut MutTx) -> Result + Send + 'static, +) -> Result, EnvironmentSnapshotError> { + let mut durability = db + .durable_tx_offset() + .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| EnvironmentSnapshotError::Capacity)?; + let (_permit, durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, result) = db.with_auto_rollback(tx, action)?; + let (offset, data, metrics, reducer) = db + .commit_tx(tx) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .ok_or(EnvironmentSnapshotError::Storage)?; + db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, EnvironmentSnapshotError>((permit, offset, result)) + }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + Ok(Durable { + receipt, + durable_through, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::relational_db::tests_utils::TestDB; + use std::time::Duration; + + #[test] + fn container_environment_cancelled_transaction_retains_capacity_until_worker_finishes() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let (entered, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release, release_rx) = std::sync::mpsc::sync_channel(1); + let blocked = db + .runtime() + .unwrap() + .spawn(mutate_with_capacity(db.db.clone(), capacity.clone(), move |_| { + entered.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + Err::<(), _>(EnvironmentSnapshotError::InvalidEnvironment) + })); + entered_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + blocked.abort(); + db.runtime().unwrap().block_on(async { + assert!(blocked.await.unwrap_err().is_cancelled()); + let retry = mutate_with_capacity(db.db.clone(), capacity.clone(), |_| Ok(())).await; + assert!(matches!(retry, Err(EnvironmentSnapshotError::Capacity))); + }); + release.send(()).unwrap(); + db.runtime().unwrap().block_on(async { + // This waits for the actual cancelled caller's blocking worker to + // return, not merely for cancellation of its async JoinHandle. + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + mutate_with_capacity(db.db.clone(), capacity, |_| Ok(())).await.unwrap(); + }); + } +} diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index b811a9efd9a..888fe5558c0 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -1592,8 +1592,8 @@ mod test { #[test] fn module_cannot_access_hosted_system_records_by_guessed_ids() -> Result<()> { use spacetimedb_datastore::system_tables::{ - ST_CONNECTION_AUTH_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, - ST_PUBLISH_FENCE_ID, + ST_CONNECTION_AUTH_ID, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, + ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, ST_PUBLISH_FENCE_ID, }; let db = relational_db()?; let (env, _runtime) = instance_env(db.clone())?; @@ -1604,6 +1604,7 @@ mod test { (ST_PUBLISH_FENCE_ID, "st_publish_fence", to_vec(&0u8)?), (ST_DEPLOYMENT_OPERATION_ID, "st_deployment_operation", to_vec(&0u128)?), (ST_CONNECTION_AUTH_ID, "st_connection_auth", to_vec(&0u128)?), + (ST_CONTAINER_ENVIRONMENT_ID, "st_container_environment", to_vec(&0u64)?), ( ST_CONTAINER_FENCE_ID, "st_container_fence", diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 0be568862c9..e3401bbb0ab 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -10,6 +10,7 @@ use spacetimedb_lib::ProductValue; use spacetimedb_schema::def::deserialize::{ArgsSeed, FunctionDef}; use spacetimedb_schema::def::ModuleDef; +pub mod container_environment; mod disk_storage; pub mod empty_module; mod host_controller; diff --git a/crates/core/src/sql/ast.rs b/crates/core/src/sql/ast.rs index 892430ba1ea..4759c89e91d 100644 --- a/crates/core/src/sql/ast.rs +++ b/crates/core/src/sql/ast.rs @@ -34,6 +34,9 @@ impl SchemaView for SchemaViewer<'_, T> { } fn schema_for_table(&self, table_id: TableId) -> Option> { + if spacetimedb_datastore::system_tables::is_host_only_read_table(table_id) { + return None; + } self.tx .get_schema(table_id) .filter(|schema| self.auth.has_read_access(schema.table_access)) diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index bbd4b6a63d9..f1f9a89fe67 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1007,8 +1007,8 @@ pub(crate) mod tests { ST_VIEW_PARAM_NAME, ST_VIEW_SUB_ID, ST_VIEW_SUB_NAME, }; use crate::system_tables::{ - ST_CONNECTION_AUTH_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, - ST_PUBLISH_FENCE_ID, + ST_CONNECTION_AUTH_ID, ST_CONTAINER_ENVIRONMENT_ID, ST_CONTAINER_FENCE_ID, ST_DEPLOYMENT_ID, + ST_DEPLOYMENT_OPERATION_ID, ST_ENV_ID, ST_PUBLISH_FENCE_ID, }; use crate::traits::{IsolationLevel, MutTx}; use crate::Result; @@ -1482,6 +1482,7 @@ pub(crate) mod tests { TableRow { id: ST_DEPLOYMENT_OPERATION_ID.into(), name: "st_deployment_operation", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, TableRow { id: ST_CONTAINER_FENCE_ID.into(), name: "st_container_fence", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, TableRow { id: ST_CONNECTION_AUTH_ID.into(), name: "st_connection_auth", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, + TableRow { id: ST_CONTAINER_ENVIRONMENT_ID.into(), name: "st_container_environment", ty: StTableType::System, access: StAccess::Private, primary_key: Some(ColId(0)) }, ])); #[rustfmt::skip] @@ -1601,6 +1602,8 @@ pub(crate) mod tests { ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 0, name: "connection_id", ty: AlgebraicType::U128 }, ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 1, name: "sender_identity", ty: AlgebraicType::U256 }, ColRow { table: ST_CONNECTION_AUTH_ID.into(), pos: 2, name: "call_auth_flags", ty: AlgebraicType::U32 }, + ColRow { table: ST_CONTAINER_ENVIRONMENT_ID.into(), pos: 0, name: "generation", ty: AlgebraicType::U64 }, + ColRow { table: ST_CONTAINER_ENVIRONMENT_ID.into(), pos: 1, name: "payload", ty: AlgebraicType::bytes() }, ])); #[rustfmt::skip] assert_eq!(query.scan_st_indexes()?, map_array([ @@ -1639,6 +1642,7 @@ pub(crate) mod tests { IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col(0), name: "st_deployment_operation_operation_id_idx_btree", }, IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col(0), name: "st_container_fence_source_identity_idx_btree", }, IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col(0), name: "st_connection_auth_connection_id_idx_btree", }, + IndexRow { id: 36, table: ST_CONTAINER_ENVIRONMENT_ID.into(), col: col(0), name: "st_container_environment_generation_idx_btree", }, ])); let start = ST_RESERVED_SEQUENCE_RANGE as i128 + 1; #[rustfmt::skip] @@ -1690,6 +1694,7 @@ pub(crate) mod tests { ConstraintRow { constraint_id: 29, table_id: ST_DEPLOYMENT_OPERATION_ID.into(), unique_columns: col(0), constraint_name: "st_deployment_operation_operation_id_key", }, ConstraintRow { constraint_id: 30, table_id: ST_CONTAINER_FENCE_ID.into(), unique_columns: col(0), constraint_name: "st_container_fence_source_identity_key", }, ConstraintRow { constraint_id: 31, table_id: ST_CONNECTION_AUTH_ID.into(), unique_columns: col(0), constraint_name: "st_connection_auth_connection_id_key", }, + ConstraintRow { constraint_id: 32, table_id: ST_CONTAINER_ENVIRONMENT_ID.into(), unique_columns: col(0), constraint_name: "st_container_environment_generation_key", }, ])); // Verify we get back the tables correctly with the proper ids... @@ -2129,6 +2134,7 @@ pub(crate) mod tests { IndexRow { id: 33, table: ST_DEPLOYMENT_OPERATION_ID.into(), col: col(0), name: "st_deployment_operation_operation_id_idx_btree", }, IndexRow { id: 34, table: ST_CONTAINER_FENCE_ID.into(), col: col(0), name: "st_container_fence_source_identity_idx_btree", }, IndexRow { id: 35, table: ST_CONNECTION_AUTH_ID.into(), col: col(0), name: "st_connection_auth_connection_id_idx_btree", }, + IndexRow { id: 36, table: ST_CONTAINER_ENVIRONMENT_ID.into(), col: col(0), name: "st_container_environment_generation_idx_btree", }, IndexRow { id: seq_start, table: FIRST_NON_SYSTEM_ID, col: col(0), name: "Foo_id_idx_btree", }, IndexRow { id: seq_start + 1, table: FIRST_NON_SYSTEM_ID, col: col(1), name: "Foo_name_idx_btree", }, IndexRow { id: seq_start + 2, table: FIRST_NON_SYSTEM_ID, col: col(2), name: "Foo_age_idx_btree", }, diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index 0f30b180294..546e4464de9 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -175,7 +175,7 @@ pub fn is_built_in_meta_row(table_id: TableId, row: &ProductValue) -> Result false, + ST_CONNECTION_CREDENTIALS_ID | ST_CONNECTION_AUTH_ID | ST_CONTAINER_ENVIRONMENT_ID => false, // We don't define any system views, so none of the view-related tables can be system meta-descriptors. ST_VIEW_ID | ST_VIEW_PARAM_ID | ST_VIEW_COLUMN_ID | ST_VIEW_SUB_ID | ST_VIEW_ARG_ID => false, ST_EVENT_TABLE_ID => { @@ -205,8 +205,8 @@ pub enum SystemTable { st_table_accessor, } -pub fn system_tables() -> [TableSchema; 26] { - let [env, deployment, publish_fence, deployment_operation, container_fence, connection_auth] = +pub fn system_tables() -> [TableSchema; 27] { + let [env, deployment, publish_fence, deployment_operation, container_fence, connection_auth, container_environment] = deployment_system_schemas(); [ // The order should match the `id` of the system table, that start with [ST_TABLE_IDX]. @@ -236,6 +236,7 @@ pub fn system_tables() -> [TableSchema; 26] { deployment_operation, container_fence, connection_auth, + container_environment, ] } diff --git a/crates/datastore/src/system_tables/deployment.rs b/crates/datastore/src/system_tables/deployment.rs index a10c64ab0c3..03a43a30ece 100644 --- a/crates/datastore/src/system_tables/deployment.rs +++ b/crates/datastore/src/system_tables/deployment.rs @@ -13,6 +13,7 @@ pub const ST_PUBLISH_FENCE_ID: TableId = TableId(23); pub const ST_DEPLOYMENT_OPERATION_ID: TableId = TableId(24); pub const ST_CONTAINER_FENCE_ID: TableId = TableId(25); pub const ST_CONNECTION_AUTH_ID: TableId = TableId(26); +pub const ST_CONTAINER_ENVIRONMENT_ID: TableId = TableId(27); pub const ST_ENV_NAME: &str = "st_env"; pub const ST_DEPLOYMENT_NAME: &str = "st_deployment"; @@ -20,6 +21,7 @@ pub const ST_PUBLISH_FENCE_NAME: &str = "st_publish_fence"; pub const ST_DEPLOYMENT_OPERATION_NAME: &str = "st_deployment_operation"; pub const ST_CONTAINER_FENCE_NAME: &str = "st_container_fence"; pub const ST_CONNECTION_AUTH_NAME: &str = "st_connection_auth"; +pub const ST_CONTAINER_ENVIRONMENT_NAME: &str = "st_container_environment"; st_fields_enum!(enum StEnvFields { "key", Key = 0, @@ -56,6 +58,28 @@ st_fields_enum!(enum StConnectionAuthFields { "call_auth_flags", CallAuthFlags = 2, }); +st_fields_enum!(enum StContainerEnvironmentFields { + "generation", Generation = 0, + "payload", Payload = 1, +}); + +/// Secret-bearing host state. Never expose its retained history through SQL or module syscalls. +#[derive(Clone, PartialEq, Eq, SpacetimeType)] +#[sats(crate = spacetimedb_lib)] +pub struct StContainerEnvironmentRow { + pub generation: u64, + pub payload: Box<[u8]>, +} + +impl std::fmt::Debug for StContainerEnvironmentRow { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StContainerEnvironmentRow") + .field("generation", &self.generation) + .field("payload", &"[redacted]") + .finish() + } +} + #[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] #[sats(crate = spacetimedb_lib)] pub struct StEnvRow { @@ -131,7 +155,8 @@ row_conversions!( StPublishFenceRow, StDeploymentOperationRow, StContainerFenceRow, - StConnectionAuthRow + StConnectionAuthRow, + StContainerEnvironmentRow ); pub(super) fn register_tables(builder: &mut RawModuleDefV9Builder) { @@ -151,6 +176,7 @@ pub(super) fn register_tables(builder: &mut RawModuleDefV9Builder) { register::(builder, ST_DEPLOYMENT_OPERATION_NAME); register::(builder, ST_CONTAINER_FENCE_NAME); register::(builder, ST_CONNECTION_AUTH_NAME); + register::(builder, ST_CONTAINER_ENVIRONMENT_NAME); } pub(super) fn validate_tables(def: &ModuleDef) { @@ -160,9 +186,10 @@ pub(super) fn validate_tables(def: &ModuleDef) { validate_system_table::(def, ST_DEPLOYMENT_OPERATION_NAME); validate_system_table::(def, ST_CONTAINER_FENCE_NAME); validate_system_table::(def, ST_CONNECTION_AUTH_NAME); + validate_system_table::(def, ST_CONTAINER_ENVIRONMENT_NAME); } -pub(crate) fn deployment_system_schemas() -> [TableSchema; 6] { +pub(crate) fn deployment_system_schemas() -> [TableSchema; 7] { [ st_schema(ST_ENV_NAME, ST_ENV_ID), st_schema(ST_DEPLOYMENT_NAME, ST_DEPLOYMENT_ID), @@ -170,6 +197,7 @@ pub(crate) fn deployment_system_schemas() -> [TableSchema; 6] { st_schema(ST_DEPLOYMENT_OPERATION_NAME, ST_DEPLOYMENT_OPERATION_ID), st_schema(ST_CONTAINER_FENCE_NAME, ST_CONTAINER_FENCE_ID), st_schema(ST_CONNECTION_AUTH_NAME, ST_CONNECTION_AUTH_ID), + st_schema(ST_CONTAINER_ENVIRONMENT_NAME, ST_CONTAINER_ENVIRONMENT_ID), ] } @@ -181,6 +209,7 @@ pub(super) fn system_schema(table: TableId) -> Option { ST_DEPLOYMENT_OPERATION_ID => ST_DEPLOYMENT_OPERATION_NAME, ST_CONTAINER_FENCE_ID => ST_CONTAINER_FENCE_NAME, ST_CONNECTION_AUTH_ID => ST_CONNECTION_AUTH_NAME, + ST_CONTAINER_ENVIRONMENT_ID => ST_CONTAINER_ENVIRONMENT_NAME, _ => return None, }; Some(st_schema(name, table)) @@ -197,9 +226,15 @@ pub fn is_module_restricted_table(table: TableId) -> bool { | ST_DEPLOYMENT_OPERATION_ID | ST_CONTAINER_FENCE_ID | ST_CONNECTION_AUTH_ID + | ST_CONTAINER_ENVIRONMENT_ID ) } +/// Snapshot history is exclusively available through authenticated host operations. +pub fn is_host_only_read_table(table: TableId) -> bool { + table == ST_CONTAINER_ENVIRONMENT_ID +} + pub fn is_module_restricted_index(index: IndexId) -> bool { INDEXES.iter().any(|(_, restricted)| *restricted == index) } @@ -214,23 +249,26 @@ pub fn is_host_managed_deployment_table(table: TableId) -> bool { | ST_DEPLOYMENT_OPERATION_ID | ST_CONTAINER_FENCE_ID | ST_CONNECTION_AUTH_ID + | ST_CONTAINER_ENVIRONMENT_ID ) } -pub(super) const CONSTRAINTS: [(&str, ConstraintId); 6] = [ +pub(super) const CONSTRAINTS: [(&str, ConstraintId); 7] = [ ("st_env_key_key", ConstraintId(26)), ("st_deployment_key_key", ConstraintId(27)), ("st_publish_fence_key_key", ConstraintId(28)), ("st_deployment_operation_operation_id_key", ConstraintId(29)), ("st_container_fence_source_identity_key", ConstraintId(30)), ("st_connection_auth_connection_id_key", ConstraintId(31)), + ("st_container_environment_generation_key", ConstraintId(32)), ]; -pub(super) const INDEXES: [(&str, IndexId); 6] = [ +pub(super) const INDEXES: [(&str, IndexId); 7] = [ ("st_env_key_idx_btree", IndexId(30)), ("st_deployment_key_idx_btree", IndexId(31)), ("st_publish_fence_key_idx_btree", IndexId(32)), ("st_deployment_operation_operation_id_idx_btree", IndexId(33)), ("st_container_fence_source_identity_idx_btree", IndexId(34)), ("st_connection_auth_connection_id_idx_btree", IndexId(35)), + ("st_container_environment_generation_idx_btree", IndexId(36)), ]; diff --git a/crates/lib/src/container_environment.rs b/crates/lib/src/container_environment.rs new file mode 100644 index 00000000000..f2ff648a45e --- /dev/null +++ b/crates/lib/src/container_environment.rs @@ -0,0 +1,40 @@ +//! Immutable application environment identity, independent of expiring launch credentials. +//! +//! These values describe a request. They do not authenticate a client or grant +//! access to environment values. Only the trusted host/control protocol may +//! capture, read, or close a snapshot. + +use crate::{Hash, Identity, SpacetimeType, Uuid}; + +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct EnvironmentSnapshotScope { + pub cluster: String, + pub database_id: u64, + pub database_identity: Identity, + pub node_id: u64, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub node_incarnation: Uuid, + pub generation: u64, + pub deployment_revision: Hash, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub start_request: Uuid, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub env_generation: Uuid, + /// The complete sorted, unique key list of the committed container declaration. + pub env_keys: Vec, +} + +/// Stable identity of a committed capture. No secret values or value hashes. +/// A durability barrier belongs to each proof, not to this immutable receipt. +#[derive(Clone, Debug, PartialEq, Eq, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(deny_unknown_fields))] +pub struct EnvironmentSnapshotReceipt { + pub scope: EnvironmentSnapshotScope, + #[cfg_attr(feature = "serde", serde(with = "crate::deployment::uuid_json"))] + pub capture_receipt: Uuid, +} diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index df7485ee282..ae5d93e5e26 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -12,6 +12,7 @@ use std::collections::{btree_map, BTreeMap}; pub mod connection_id; pub mod container; +pub mod container_environment; pub mod db; pub mod deployment; mod direct_index_key; From ff55f223cdc3e57d6f1c58d155d7f920e83d8f29 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 05:45:30 -0400 Subject: [PATCH 10/23] Add local container configuration and verified OCI preparation --- Cargo.lock | 4 + crates/cli/Cargo.toml | 6 + crates/cli/docs/container-build.md | 149 +++++ crates/cli/src/container/config.rs | 122 ++++ crates/cli/src/container/mod.rs | 394 +++++++++++++ crates/cli/src/container/oci.rs | 262 +++++++++ crates/cli/src/container/process.rs | 172 ++++++ crates/cli/src/container/tests.rs | 713 ++++++++++++++++++++++++ crates/cli/src/lib.rs | 14 + crates/cli/src/main.rs | 4 + crates/cli/src/spacetime_config.rs | 5 + crates/cli/src/subcommands/container.rs | 174 ++++++ crates/cli/src/subcommands/mod.rs | 1 + crates/cli/src/subcommands/publish.rs | 2 + 14 files changed, 2022 insertions(+) create mode 100644 crates/cli/docs/container-build.md create mode 100644 crates/cli/src/container/config.rs create mode 100644 crates/cli/src/container/mod.rs create mode 100644 crates/cli/src/container/oci.rs create mode 100644 crates/cli/src/container/process.rs create mode 100644 crates/cli/src/container/tests.rs create mode 100644 crates/cli/src/subcommands/container.rs diff --git a/Cargo.lock b/Cargo.lock index f080620b262..bc61994dfc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7857,10 +7857,12 @@ dependencies = [ "rolldown_common", "rolldown_error", "rolldown_utils", + "rustix 1.1.2", "rustyline", "serde", "serde_json", "serde_with", + "sha2", "slab", "spacetimedb-auth", "spacetimedb-client-api-messages", @@ -7869,6 +7871,7 @@ dependencies = [ "spacetimedb-fs-utils", "spacetimedb-jsonwebtoken", "spacetimedb-lib 2.3.0", + "spacetimedb-oci", "spacetimedb-paths", "spacetimedb-primitives 2.3.0", "spacetimedb-schema", @@ -7883,6 +7886,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-tungstenite 0.27.0", + "tokio-util", "toml 0.8.23", "toml_edit 0.22.27", "tracing", diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index bb71fb8b6e5..acd1414422d 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -27,6 +27,7 @@ spacetimedb-codegen.workspace = true spacetimedb-data-structures.workspace = true spacetimedb-fs-utils.workspace = true spacetimedb-lib.workspace = true +spacetimedb-oci = { path = "../oci" } spacetimedb-paths.workspace = true spacetimedb-primitives.workspace = true spacetimedb-schema.workspace = true @@ -65,6 +66,8 @@ syntect.workspace = true tabled.workspace = true tar.workspace = true tempfile.workspace = true +sha2 = "0.10" +tokio-util.workspace = true termcolor.workspace = true termtree.workspace = true thiserror.workspace = true @@ -98,6 +101,9 @@ fs_extra.workspace = true tikv-jemallocator = { workspace = true } tikv-jemalloc-ctl = { workspace = true } +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +rustix = { version = "1", features = ["fs", "process"] } + [target.'cfg(windows)'.dependencies] windows-sys = { workspace = true, features = ["Win32_System_Console"] } diff --git a/crates/cli/docs/container-build.md b/crates/cli/docs/container-build.md new file mode 100644 index 00000000000..643a488dc08 --- /dev/null +++ b/crates/cli/docs/container-build.md @@ -0,0 +1,149 @@ +# Local container image preparation + +`spacetime container build` reads one database's `container` declaration from +`spacetime.json` and prepares a verified local OCI image. It does not resolve a +database through a server or publish anything. The optional `DATABASE` argument +selects an exact local configuration target. Omit it only when the configuration +contains one container target. Container declarations do not pass to children. +The command is dispatched before saved CLI server settings or credentials are +opened; only project configuration and explicitly selected build credentials +are read. + +This slice implements local preparation. Managed publication and container +lifecycle commands are separate integration work. The existing `publish` command +rejects a selected container declaration so it cannot silently publish only the +module. + +## Configuration + +```json +{ + "database": "example", + "container": { + "image": { + "build": { + "builder": "dockerfile", + "context": ".", + "dockerfile": "Dockerfile" + } + }, + "env_keys": ["API_KEY"], + "resources": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824, + "scratch_bytes": 1073741824, + "pids_max": 256 + } + } +} +``` + +`image` accepts exactly one source: + +- `{"build":{"context":"."}}` selects Dockerfile by default. +- `{"build":{"builder":"railpack","context":"."}}` explicitly selects Railpack. +- `{"oci_ref":"registry.example/team/image:tag"}` imports a registry image. +- `{"oci_ref":"oci:./existing-layout"}` imports a local OCI image-layout directory. + +Build contexts and local layout paths are relative to the configuration +directory. The Dockerfile path is relative to its build context. A tag is +resolved once during import; the result records the selected immutable manifest +digest. Registry import requires Skopeo. A local directory import requires no +external image tools. + +The image's `Entrypoint` followed by `Cmd` supplies the command by default. +`command` replaces the complete argv. Image `User` and `WorkingDir` are preserved +unless `user` or `working_directory` overrides them. An omitted or empty image +`WorkingDir` normalizes to `/`. Image `Env` remains in the +immutable configuration blob. `env_keys` contains runtime store references, +never runtime values. Reserved `SPACETIMEDB_` keys are rejected. Mounts must be +empty in Stage 1. Shared resource, startup-string, port, and environment-key +limits apply to the normalized specification. + +## Tools and credentials + +Source builds require an explicitly selected local BuildKit Unix socket: + +```sh +spacetime container build example \ + --project-path ./project \ + --platform linux/amd64 \ + --out-dir ./prepared-image \ + --buildkit-host unix:///absolute/path/to/disposable-buildkit.sock +``` + +Use an endpoint you own and have verified. The command never selects a saved +Docker daemon or BuildKit endpoint. This implementation invokes `buildctl` +directly and currently runs external tools on Linux and macOS. An existing local +OCI directory can also be imported without those subprocesses on Windows. +`--platform` is required and accepts `linux/amd64` or `linux/arm64`; it never +silently uses the build computer's platform. `--out-dir` must not already exist, +and its parent must exist. + +`--buildctl`, `--skopeo`, and `--railpack` select absolute executable paths or +command names on PATH. Missing tools +produce installation/path errors. No executable is automatically downloaded. +Railpack is pinned to `0.35.0`, paired with +`ghcr.io/railwayapp/railpack-frontend:v0.35.0`. Detection or version failures end +the build without changing builders. Its `prepare` output is given to BuildKit's +matching gateway frontend, following the [Railpack production integration](https://railpack.com/platforms/running-railpack-in-production). + +Registry access is anonymous unless `--registry-auth-file FILE` explicitly names +an auth JSON file. The CLI creates an isolated auth directory and does not load +saved Docker credentials. Registry credentials are distinct from Spacetime +credentials and are never copied into prepared metadata. + +Pass build secrets as `--build-secret NAME=FILE`. Files enter BuildKit's secret +interface; Railpack receives only their names during plan generation. Builds +with secrets disable cached build results. Build secrets are separate from +`env_keys`, and prebuilt imports reject them. Credential and secret files are +bounded to 1 MiB each. Subprocess environments are cleared except for basic +executable/temporary-directory paths and the explicitly isolated tool paths. +Do not put secret values in a Dockerfile, image command, or image defaults. + +## Output and failure behavior + +The new output directory contains an OCI image layout plus `prepared.json`: + +- `container` is the normalized container specification. +- `manifest` is the selected immutable manifest descriptor. +- `objects` contains the executable manifest/config/layer closure, with digest, + media type, size, purpose, and a path relative to the layout. + +Prepared metadata omits image environment values and build credentials. The +image configuration blob necessarily retains its image defaults. Consumers must +verify descriptors when reopening output; the local metadata is not an +admission receipt from a server. + +`prepare_container` returns `PreparedContainer`, which owns its temporary +layout. Dropping it removes the temporary output. `persist` transfers the verified +layout with an atomic operation that does not replace an existing output. +Credentials, plans, and intermediate builder files remain in the temporary +workspace and are removed. Failed verification never returns a prepared object. +The shared `Runner` interface permits structured fake builders in tests and +reuse by a later managed publication path. + +Verification checks manifest/config digests, platform, exact compressed layer +digests, uncompressed diff IDs, and bounded tar structure. Layers are inspected +as streams and are never unpacked into the project. Imports reject archive +links, special entries, unknown paths, and duplicate object paths. Bounds include +256 layers, 64 GiB compressed image data, 128 GiB expanded tar data, and one +million expanded entries, with the shared per-object/decompression limits. +Two image tools and two verification workers may run concurrently. Tool calls +have a 30-minute deadline; verification has a 5-minute deadline. Captured stdout +and stderr each have a 4 MiB limit and are not echoed, avoiding accidental +secret disclosure in tool diagnostics. + +Cancellation retains the tool's workspace until its Unix process group has been +signalled and its leader reaped. The group leader remains unreaped until the +signal, preventing reuse of its numeric process-group identity. This cleanup is +for locally trusted tools; it is not a sandbox for a tool that deliberately +escapes its group. BuildKit daemon work is subject to the selected daemon's own +client-disconnect cancellation and retention policies. Blocking verification +retains its worker permit and workspace while checking cancellation between +bounded reads. + +Tests use generated local OCI fixtures and fake tool invocations, including an +owned shell fixture for process cleanup. They do not execute Docker, BuildKit, +Railpack, Skopeo, or any server operation. Actual supported-builder acceptance +remains a separate integration check. diff --git a/crates/cli/src/container/config.rs b/crates/cli/src/container/config.rs new file mode 100644 index 00000000000..b9f12a6cf66 --- /dev/null +++ b/crates/cli/src/container/config.rs @@ -0,0 +1,122 @@ +//! Per-database declarations. Values and build credentials are never part of this configuration. +use anyhow::{ensure, Result}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::container::{ + ContainerMode, ContainerMount, ContainerPort, ContainerResources, ContainerSpec, ContainerSpecLimits, + ImagePlatform, OciDigest, RestartPolicy, +}; +use spacetimedb_oci::ContainerConfig as ImageConfig; +use std::path::PathBuf; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerConfig { + pub image: ImageSource, + pub command: Option>, + pub user: Option, + pub working_directory: Option, + #[serde(default = "default_mode")] + pub mode: ContainerMode, + #[serde(default = "default_restart")] + pub restart: RestartPolicy, + #[serde(default)] + pub env_keys: Vec, + pub resources: ContainerResources, + #[serde(default)] + pub ports: Vec, + #[serde(default)] + pub mounts: Vec, + #[serde(default = "default_grace")] + pub stop_grace_ms: u32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ImageSource { + Build(BuildImage), + Prebuilt(PrebuiltImage), +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BuildImage { + #[serde(deserialize_with = "source_build")] + pub build: SourceBuild, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PrebuiltImage { + pub oci_ref: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "builder", rename_all = "lowercase", deny_unknown_fields)] +pub enum SourceBuild { + Dockerfile { + #[serde(default = "default_context")] + context: PathBuf, + #[serde(default = "default_dockerfile")] + dockerfile: PathBuf, + }, + Railpack { + #[serde(default = "default_context")] + context: PathBuf, + }, +} + +// A missing builder is Dockerfile. Keep deserialization strict after applying +// that one default, including rejection of Dockerfile-only fields on Railpack. +fn source_build<'de, D: serde::Deserializer<'de>>(deserializer: D) -> std::result::Result { + let mut value = serde_json::Value::deserialize(deserializer)?; + if let Some(object) = value.as_object_mut() { + object.entry("builder").or_insert_with(|| "dockerfile".into()); + } + serde_json::from_value(value).map_err(serde::de::Error::custom) +} +fn default_context() -> PathBuf { + PathBuf::from(".") +} +fn default_dockerfile() -> PathBuf { + PathBuf::from("Dockerfile") +} +fn default_mode() -> ContainerMode { + ContainerMode::Service +} +fn default_restart() -> RestartPolicy { + RestartPolicy::OnFailure +} +fn default_grace() -> u32 { + 30_000 +} + +impl ContainerConfig { + pub fn normalize( + &self, + manifest: OciDigest, + platform: ImagePlatform, + image: &ImageConfig, + ) -> Result { + ensure!(self.mounts.is_empty(), "container mounts are not supported in Stage 1"); + let argv = image.argv(self.command.as_deref())?; + spacetimedb_lib::container::validate_exec_size(&argv, image.env.as_deref().unwrap_or_default())?; + Ok(ContainerSpec { + image_manifest: manifest, + image_platform: platform, + argv, + user: self.user.clone().unwrap_or_else(|| image.user.clone()), + working_directory: self.working_directory.clone().unwrap_or_else(|| { + if image.working_directory.is_empty() { + "/".into() + } else { + image.working_directory.clone() + } + }), + mode: self.mode, + restart: self.restart, + env_keys: self.env_keys.clone(), + resources: self.resources, + ports: self.ports.clone(), + mounts: vec![], + stop_grace_ms: self.stop_grace_ms, + } + .normalize(&ContainerSpecLimits::default())?) + } +} diff --git a/crates/cli/src/container/mod.rs b/crates/cli/src/container/mod.rs new file mode 100644 index 00000000000..056f7d1f545 --- /dev/null +++ b/crates/cli/src/container/mod.rs @@ -0,0 +1,394 @@ +//! Local image preparation shared by build-only commands and managed publication. +pub mod config; +pub mod oci; +pub mod process; + +#[cfg(test)] +mod tests; + +use anyhow::{ensure, Context, Result}; +use config::{ContainerConfig, ImageSource, SourceBuild}; +use oci::{LocalArtifact, VerifiedImage}; +use process::{Invocation, Runner}; +use serde::{Deserialize, Serialize}; +use spacetimedb_lib::container::{ContainerSpec, ImagePlatform}; +use spacetimedb_oci::Descriptor; +use std::{ + ffi::OsString, + fs, + io::Read, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +pub const RAILPACK_VERSION: &str = "0.35.0"; +pub const RAILPACK_FRONTEND: &str = "ghcr.io/railwayapp/railpack-frontend:v0.35.0"; +const BUILD_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const VERIFY_TIMEOUT: Duration = Duration::from_secs(5 * 60); +static VERIFIERS: std::sync::LazyLock> = + std::sync::LazyLock::new(|| Arc::new(tokio::sync::Semaphore::new(2))); + +pub struct BuildSecret { + pub name: String, + pub file: PathBuf, +} +pub struct BuildTools { + pub buildctl: PathBuf, + pub buildkit_host: Option, + pub railpack: PathBuf, + pub skopeo: PathBuf, + pub registry_auth_file: Option, + pub secrets: Vec, +} +impl Default for BuildTools { + fn default() -> Self { + Self { + buildctl: "buildctl".into(), + buildkit_host: None, + railpack: "railpack".into(), + skopeo: "skopeo".into(), + registry_auth_file: None, + secrets: vec![], + } + } +} + +/// Public metadata contains immutable descriptors and relative paths, not image +/// environment values or credentials. Reopening output must verify the bytes. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PreparedMetadata { + pub version: u32, + pub container: ContainerSpec, + pub manifest: Descriptor, + pub objects: Vec, +} +/// Owns all temporary output. Drop removes it; persist transfers only verified +/// artifacts. A failed/cancelled prepare never yields this type. +pub struct PreparedContainer { + workspace: Arc, + pub metadata: PreparedMetadata, +} +impl PreparedContainer { + pub fn layout(&self) -> PathBuf { + self.workspace.path().join("verified") + } + pub fn persist(self, output: &Path) -> Result<()> { + ensure!( + !output.exists(), + "output directory already exists: {}", + output.display() + ); + // The final transition must not replace a directory created after the + // initial check. The workspace is placed alongside the requested output. + #[cfg(any(target_os = "linux", target_os = "macos"))] + rustix::fs::renameat_with( + rustix::fs::CWD, + self.layout(), + rustix::fs::CWD, + output, + rustix::fs::RenameFlags::NOREPLACE, + ) + .context("could not publish local OCI output without replacing an existing path")?; + #[cfg(windows)] + fs::rename(self.layout(), output) + .context("could not publish local OCI output without replacing an existing path")?; + #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] + anyhow::bail!("atomic container output is supported on Linux, macOS, and Windows"); + Ok(()) + } +} +struct CancelOnDrop(CancellationToken); +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.cancel(); + } +} +fn path_argument(key: &str, path: &Path) -> Result { + let path = path.to_str().context("image tools require UTF-8 paths")?; + Ok(format!("{key}={path}").into()) +} +fn csv_argument(key: &str, path: &Path) -> Result { + let path = path.to_str().context("image tools require UTF-8 paths")?; + Ok(format!("\"{key}={}\"", path.replace('"', "\"\""))) +} +fn read_credential(path: &Path) -> Result> { + let mut bytes = vec![]; + fs::File::open(path)?.take(1024 * 1024 + 1).read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= 1024 * 1024, + "explicit build credential file exceeds 1 MiB" + ); + Ok(bytes) +} +fn local_buildkit_host(value: Option<&str>) -> Result<&str> { + let value = value.context("source builds require --buildkit-host unix:///absolute/path/to/buildkitd.sock")?; + let path = value + .strip_prefix("unix://") + .context("local source builds require an explicit BuildKit Unix socket")?; + ensure!( + Path::new(path).is_absolute() && !path.contains(['\0', '\n', '\r']), + "invalid local BuildKit socket path" + ); + Ok(value) +} + +pub async fn prepare_container( + declaration: &ContainerConfig, + config_dir: &Path, + platform: ImagePlatform, + tools: &BuildTools, + workspace_parent: &Path, + runner: &impl Runner, + cancel: CancellationToken, +) -> Result { + ensure!( + platform.os == "linux" && matches!(platform.architecture.as_str(), "amd64" | "arm64"), + "choose linux/amd64 or linux/arm64 explicitly" + ); + ensure!( + declaration.mounts.is_empty(), + "container mounts are not supported in Stage 1" + ); + let cancel = cancel.child_token(); + let _cancel = CancelOnDrop(cancel.clone()); + let workspace = Arc::new( + tempfile::Builder::new() + .prefix(".spacetime-image-") + .tempdir_in(workspace_parent)?, + ); + let base = workspace.path(); + fs::create_dir(base.join("auth"))?; + let auth_file = base.join("auth/config.json"); + let auth = tools + .registry_auth_file + .as_deref() + .map(read_credential) + .transpose()? + .unwrap_or_else(|| br#"{"auths":{}}"#.to_vec()); + fs::write(&auth_file, auth)?; + let environment = vec![ + ("DOCKER_CONFIG".into(), base.join("auth").into_os_string()), + ("REGISTRY_AUTH_FILE".into(), auth_file.clone().into_os_string()), + ("XDG_CONFIG_HOME".into(), base.join("tool-config").into_os_string()), + ("XDG_CACHE_HOME".into(), base.join("tool-cache").into_os_string()), + ]; + let invoke = |tool: PathBuf, label, args, cwd: PathBuf| Invocation { + tool, + label, + args, + env: environment.clone(), + cwd, + workspace: workspace.clone(), + timeout: BUILD_TIMEOUT, + cancel: cancel.clone(), + }; + let input = base.join("input"); + let mut archive = None; + match &declaration.image { + ImageSource::Prebuilt(image) => { + ensure!( + tools.secrets.is_empty(), + "build secrets cannot be supplied with a prebuilt image" + ); + if let Some(path) = image.oci_ref.strip_prefix("oci:") { + let path = config_dir + .join(path) + .canonicalize() + .context("prebuilt OCI layout does not exist")?; + ensure!(path.is_dir(), "oci: must name an OCI layout directory"); + return verify(declaration.clone(), platform, workspace.clone(), path, None, cancel).await; + } + ensure!( + !image.oci_ref.is_empty() + && !image.oci_ref.starts_with('-') + && !image.oci_ref.contains(['\0', '\n', '\r']), + "invalid OCI registry reference" + ); + let reference = image.oci_ref.strip_prefix("docker://").unwrap_or(&image.oci_ref); + ensure!( + !reference.contains("://") + && (!reference.contains('@') + || reference.rsplit_once('@').is_some_and(|(_, digest)| digest + .parse::() + .is_ok())), + "invalid OCI registry reference" + ); + let args = vec![ + "--override-os".into(), + platform.os.clone().into(), + "--override-arch".into(), + platform.architecture.clone().into(), + "copy".into(), + "--preserve-digests".into(), + "--authfile".into(), + auth_file.into_os_string(), + format!("docker://{reference}").into(), + format!("oci:{}:prepared", input.display()).into(), + ]; + runner + .run(invoke( + tools.skopeo.clone(), + "Skopeo image import", + args, + config_dir.to_path_buf(), + )) + .await?; + } + ImageSource::Build(image) => { + let endpoint = local_buildkit_host(tools.buildkit_host.as_deref())?; + let (context, dockerfile, railpack) = match &image.build { + SourceBuild::Dockerfile { context, dockerfile } => (context, Some(dockerfile), false), + SourceBuild::Railpack { context } => (context, None, true), + }; + let context = config_dir + .join(context) + .canonicalize() + .context("build context does not exist")?; + ensure!(context.is_dir(), "build context must be a directory"); + let mut secret_arguments = vec![]; + let mut secret_names = std::collections::BTreeSet::new(); + for (index, secret) in tools.secrets.iter().enumerate() { + spacetimedb_lib::container::validate_env_key(&secret.name)?; + ensure!(secret_names.insert(&secret.name), "duplicate build secret name"); + let path = base.join(format!("secret-{index}")); + fs::write(&path, read_credential(&secret.file)?)?; + secret_arguments.extend([ + OsString::from("--secret"), + format!("id={},{}", secret.name, csv_argument("src", &path)?).into(), + ]); + } + let dockerfile = if railpack { + let version = runner + .run(invoke( + tools.railpack.clone(), + "Railpack version check", + vec!["--version".into()], + context.clone(), + )) + .await?; + let version = std::str::from_utf8(&version.stdout).context("invalid Railpack version response")?; + ensure!( + version + .split_whitespace() + .any(|word| word.trim_start_matches('v') == RAILPACK_VERSION), + "install Railpack {RAILPACK_VERSION} to match the pinned frontend" + ); + let plan = base.join("railpack-plan.json"); + let mut args = vec![ + "prepare".into(), + context.clone().into_os_string(), + "--plan-out".into(), + plan.clone().into_os_string(), + "--info-out".into(), + base.join("railpack-info.json").into_os_string(), + ]; + // Only names enter the plan; BuildKit receives the actual files. + for name in secret_names { + args.extend(["--env".into(), format!("{name}=").into()]); + } + runner + .run(invoke( + tools.railpack.clone(), + "Railpack detection", + args, + context.clone(), + )) + .await?; + ensure!( + plan.is_file() && plan.metadata()?.len() <= 4 * 1024 * 1024, + "Railpack did not produce a bounded build plan" + ); + plan + } else { + context + .join(dockerfile.unwrap()) + .canonicalize() + .context("Dockerfile does not exist")? + }; + ensure!( + dockerfile.is_file(), + "Dockerfile or Railpack plan must be a regular file" + ); + let output = base.join("image.tar"); + let mut args = vec![ + "--addr".into(), + endpoint.into(), + "build".into(), + "--frontend".into(), + if railpack { + "gateway.v0".into() + } else { + "dockerfile.v0".into() + }, + "--local".into(), + path_argument("context", &context)?, + "--local".into(), + path_argument("dockerfile", dockerfile.parent().unwrap())?, + "--opt".into(), + path_argument("filename", Path::new(dockerfile.file_name().unwrap()))?, + "--opt".into(), + format!("platform={}/{}", platform.os, platform.architecture).into(), + "--output".into(), + format!("type=oci,{}", csv_argument("dest", &output)?).into(), + ]; + if railpack { + args.extend(["--opt".into(), format!("source={RAILPACK_FRONTEND}").into()]); + } + if !secret_arguments.is_empty() { + args.push("--no-cache".into()); + args.extend(secret_arguments); + } + runner + .run(invoke(tools.buildctl.clone(), "BuildKit OCI build", args, context)) + .await?; + archive = Some(output); + } + } + verify(declaration.clone(), platform, workspace.clone(), input, archive, cancel).await +} + +async fn verify( + declaration: ContainerConfig, + platform: ImagePlatform, + workspace: Arc, + input: PathBuf, + archive: Option, + cancel: CancellationToken, +) -> Result { + let permit = VERIFIERS + .clone() + .try_acquire_owned() + .context("two OCI images are already being verified")?; + let owner = workspace.clone(); + let metadata = tokio::task::spawn_blocking(move || { + let _permit = permit; + let deadline = Instant::now() + VERIFY_TIMEOUT; + if let Some(archive) = archive { + oci::extract_archive(&archive, &input, &cancel, deadline)?; + } + let VerifiedImage { + manifest, + config, + objects, + } = oci::verify_layout(&input, &owner.path().join("verified"), &platform, &cancel, deadline)?; + let container = declaration.normalize(manifest.digest, platform, &config.config)?; + let metadata = PreparedMetadata { + version: 1, + container, + manifest, + objects, + }; + fs::write( + owner.path().join("verified/prepared.json"), + serde_json::to_vec_pretty(&metadata)?, + )?; + Ok::<_, anyhow::Error>(metadata) + }) + .await + .context("OCI verification worker stopped")??; + Ok(PreparedContainer { workspace, metadata }) +} diff --git a/crates/cli/src/container/oci.rs b/crates/cli/src/container/oci.rs new file mode 100644 index 00000000000..3e5ec888a57 --- /dev/null +++ b/crates/cli/src/container/oci.rs @@ -0,0 +1,262 @@ +//! Bounded OCI import. Layers are inspected as streams, never unpacked into the project. +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::container::{ImagePlatform, OciDigest}; +use spacetimedb_oci::{ + self as oci, + layers::{verify_layer_with_check, LayerLimits}, + Descriptor, ImageConfig, +}; +use std::{ + collections::BTreeSet, + fs::{self, File}, + io::{Read, Seek, Write}, + path::{Path, PathBuf}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; + +pub const MAX_EXPANDED_IMAGE_BYTES: u64 = 128 * 1024 * 1024 * 1024; +const MAX_ARCHIVE_ENTRIES: usize = 1024; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ArtifactKind { + Manifest, + Config, + Layer, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LocalArtifact { + pub kind: ArtifactKind, + pub descriptor: Descriptor, + /// Relative to the owned OCI layout directory. + pub path: PathBuf, +} +pub(crate) struct VerifiedImage { + pub manifest: Descriptor, + pub config: ImageConfig, + pub objects: Vec, +} + +pub(crate) fn check(cancel: &CancellationToken, deadline: Instant) -> std::io::Result<()> { + if cancel.is_cancelled() || Instant::now() >= deadline { + Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "container preparation cancelled or timed out", + )) + } else { + Ok(()) + } +} +fn blob_path(digest: OciDigest) -> PathBuf { + PathBuf::from("blobs/sha256").join(digest.to_string().strip_prefix("sha256:").expect("SHA-256 digest")) +} +fn bounded_file(path: &Path, limit: u64) -> Result { + ensure!( + fs::symlink_metadata(path)?.is_file(), + "OCI object must be a regular file" + ); + let file = File::open(path)?; + ensure!(file.metadata()?.len() <= limit, "OCI object exceeds its size bound"); + Ok(file) +} +fn read_small(path: &Path, limit: usize) -> Result> { + let mut bytes = vec![]; + bounded_file(path, limit as u64)? + .take(limit as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!(bytes.len() <= limit, "OCI metadata exceeds its size bound"); + Ok(bytes) +} + +pub(crate) fn extract_archive( + archive: &Path, + output: &Path, + cancel: &CancellationToken, + deadline: Instant, +) -> Result<()> { + fs::create_dir_all(output.join("blobs/sha256"))?; + let mut seen = BTreeSet::new(); + let mut total = 0u64; + let archive = bounded_file(archive, oci::MAX_IMAGE_BYTES + 16 * 1024 * 1024)?; + for (index, entry) in tar::Archive::new(archive).entries()?.enumerate() { + check(cancel, deadline)?; + ensure!(index < MAX_ARCHIVE_ENTRIES, "too many OCI archive entries"); + let mut entry = entry?; + let path = entry.path()?.into_owned(); + let name = path.to_str().context("OCI archive path is not UTF-8")?; + if entry.header().entry_type().is_dir() { + ensure!( + matches!(name.trim_end_matches('/'), "." | "blobs" | "blobs/sha256"), + "unexpected OCI archive directory" + ); + continue; + } + ensure!( + entry.header().entry_type().is_file(), + "OCI archive links and special files are unsupported" + ); + let blob = name.strip_prefix("blobs/sha256/"); + ensure!( + matches!(name, "index.json" | "oci-layout") + || blob.is_some_and( + |v| v.len() == 64 && v.bytes().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()) + ), + "unexpected OCI archive path" + ); + ensure!(seen.insert(path.clone()), "duplicate OCI archive path"); + let size = entry.size(); + total = total.checked_add(size).context("OCI archive size overflow")?; + ensure!( + total <= oci::MAX_IMAGE_BYTES + 16 * 1024 * 1024, + "OCI archive exceeds image size bound" + ); + if blob.is_none() { + ensure!(size <= oci::MAX_MANIFEST_BYTES as u64, "OCI archive metadata too large"); + } + let mut file = File::options().write(true).create_new(true).open(output.join(&path))?; + let mut buffer = [0u8; 64 * 1024]; + loop { + check(cancel, deadline)?; + let n = entry.read(&mut buffer)?; + if n == 0 { + break; + } + file.write_all(&buffer[..n])?; + } + } + Ok(()) +} + +pub(crate) fn verify_layout( + input: &Path, + output: &Path, + platform: &ImagePlatform, + cancel: &CancellationToken, + deadline: Instant, +) -> Result { + check(cancel, deadline)?; + let layout: serde_json::Value = serde_json::from_slice(&read_small(&input.join("oci-layout"), 1024)?)?; + ensure!( + layout.get("imageLayoutVersion").and_then(|v| v.as_str()) == Some("1.0.0"), + "unsupported OCI layout version" + ); + let index = read_small(&input.join("index.json"), oci::MAX_MANIFEST_BYTES)?; + // Layout metadata is not a hashed image object and may omit mediaType. + // Defaults apply only here, never to immutable registry index bytes. + let mut index: serde_json::Value = serde_json::from_slice(&index)?; + index + .as_object_mut() + .context("OCI layout index must be an object")? + .entry("mediaType") + .or_insert_with(|| oci::OCI_INDEX.into()); + let index = serde_json::to_vec(&index)?; + let parsed: oci::ImageIndex = serde_json::from_slice(&index)?; + ensure!( + parsed.media_type == oci::OCI_INDEX, + "unsupported OCI layout index media type" + ); + ensure!( + parsed.schema_version == 2 && parsed.manifests.len() <= oci::MAX_INDEX_ENTRIES, + "invalid OCI layout index" + ); + // A local layout index commonly names one manifest without platform metadata. + // The immutable image config below must still match the explicit platform. + let descriptor = if parsed.manifests.len() == 1 && parsed.manifests[0].platform.is_none() { + parsed.manifests.into_iter().next().unwrap() + } else { + oci::select_platform(&index, platform)? + }; + // A layout can name a multi-platform index; select exactly one executable manifest. + let source = read_small(&input.join(blob_path(descriptor.digest)), oci::MAX_MANIFEST_BYTES)?; + oci::verify_object(&descriptor, &source)?; + let (descriptor, bytes) = if matches!(descriptor.media_type.as_str(), oci::OCI_INDEX | oci::DOCKER_INDEX) { + let selected = oci::select_platform(&source, platform)?; + let bytes = read_small(&input.join(blob_path(selected.digest)), oci::MAX_MANIFEST_BYTES)?; + oci::verify_object(&selected, &bytes)?; + (selected, bytes) + } else { + (descriptor, source) + }; + ensure!( + matches!(descriptor.media_type.as_str(), oci::OCI_MANIFEST | oci::DOCKER_MANIFEST), + "OCI layout does not select an executable image" + ); + let manifest = oci::parse_manifest(&bytes)?; + let config_bytes = read_small(&input.join(blob_path(manifest.config.digest)), oci::MAX_CONFIG_BYTES)?; + let config = oci::parse_config(&config_bytes, &manifest, platform)?; + let mut compressed = 0u64; + let mut expanded = 0u64; + let mut entries = 0u64; + for (layer, diff_id) in manifest.layers.iter().zip(&config.rootfs.diff_ids) { + check(cancel, deadline)?; + compressed = compressed.checked_add(layer.size).context("image size overflow")?; + ensure!(compressed <= oci::MAX_IMAGE_BYTES, "compressed image exceeds bound"); + let file = bounded_file(&input.join(blob_path(layer.digest)), layer.size)?; + let size = verify_layer_with_check(file, layer, *diff_id, LayerLimits::default(), || { + check(cancel, deadline) + })?; + expanded = expanded + .checked_add(size.uncompressed_tar_bytes) + .context("expanded image size overflow")?; + entries = entries + .checked_add(size.entries) + .context("image entry count overflow")?; + ensure!( + expanded <= MAX_EXPANDED_IMAGE_BYTES && entries <= 1_000_000, + "expanded image exceeds bound" + ); + } + fs::create_dir_all(output.join("blobs/sha256"))?; + let mut objects = vec![]; + for object in oci::object_closure(descriptor.clone(), &manifest)? { + check(cancel, deadline)?; + let path = blob_path(object.digest); + let mut source = bounded_file(&input.join(&path), object.size)?; + source.rewind()?; + let mut target = File::options().write(true).create_new(true).open(output.join(&path))?; + let mut hash = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + check(cancel, deadline)?; + let n = source.read(&mut buffer)?; + if n == 0 { + break; + } + total = total.checked_add(n as u64).context("object size overflow")?; + ensure!(total <= object.size, "OCI object changed while copying"); + hash.update(&buffer[..n]); + target.write_all(&buffer[..n])?; + } + ensure!( + total == object.size && OciDigest::sha256(hash.finalize().into()) == object.digest, + "OCI object changed while copying" + ); + objects.push(LocalArtifact { + kind: if object.digest == descriptor.digest { + ArtifactKind::Manifest + } else if object.digest == manifest.config.digest { + ArtifactKind::Config + } else { + ArtifactKind::Layer + }, + descriptor: object, + path, + }); + } + fs::write(output.join("oci-layout"), br#"{"imageLayoutVersion":"1.0.0"}"#)?; + fs::write( + output.join("index.json"), + serde_json::to_vec( + &serde_json::json!({"schemaVersion":2,"mediaType":oci::OCI_INDEX,"manifests":[descriptor]}), + )?, + )?; + Ok(VerifiedImage { + manifest: descriptor, + config, + objects, + }) +} diff --git a/crates/cli/src/container/process.rs b/crates/cli/src/container/process.rs new file mode 100644 index 00000000000..ea0a03ea5f6 --- /dev/null +++ b/crates/cli/src/container/process.rs @@ -0,0 +1,172 @@ +//! Local trusted tools run in a dedicated Unix process group. Cancellation +//! retains the workspace until that group is signalled and its leader reaped. +//! This is process cleanup, not containment of tools that deliberately escape +//! their process group. +use anyhow::Result; +use std::{ffi::OsString, future::Future, path::PathBuf, sync::Arc, time::Duration}; +use tempfile::TempDir; +use tokio_util::sync::CancellationToken; + +pub struct Invocation { + pub tool: PathBuf, + pub label: &'static str, + pub args: Vec, + pub env: Vec<(OsString, OsString)>, + pub cwd: PathBuf, + pub workspace: Arc, + pub timeout: Duration, + pub cancel: CancellationToken, +} +pub struct Output { + pub stdout: Vec, +} +pub trait Runner: Sync { + fn run(&self, invocation: Invocation) -> impl Future> + Send; +} +pub struct LocalRunner; + +impl Runner for LocalRunner { + async fn run(&self, invocation: Invocation) -> Result { + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + local::run(invocation).await + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + let _ = invocation; + anyhow::bail!( + "local image tools currently require Linux or macOS; import an existing oci: directory instead" + ) + } + } +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +mod local { + use super::*; + use anyhow::{bail, ensure, Context}; + use rustix::process::{Pid, WaitIdOptions}; + use std::process::Stdio; + use tokio::{ + io::AsyncReadExt, + process::Command, + sync::{oneshot, Semaphore}, + }; + + static PROCESSES: std::sync::LazyLock> = std::sync::LazyLock::new(|| Arc::new(Semaphore::new(2))); + const MAX_OUTPUT: usize = 4 * 1024 * 1024; + + async fn read_bounded(mut reader: impl tokio::io::AsyncRead + Unpin) -> Result> { + let mut bytes = vec![]; + (&mut reader) + .take(MAX_OUTPUT as u64 + 1) + .read_to_end(&mut bytes) + .await?; + ensure!(bytes.len() <= MAX_OUTPUT, "builder diagnostic output exceeded 4 MiB"); + Ok(bytes) + } + + // Keep the group leader unreaped until after killpg. Its waitable PID pins + // the numeric process-group identity even when it exits before descendants. + struct ProcessGroup(Option); + impl ProcessGroup { + fn kill(&mut self) -> std::io::Result<()> { + if let Some(pid) = self.0.take() { + match rustix::process::kill_process_group(pid, rustix::process::Signal::KILL) { + Ok(()) | Err(rustix::io::Errno::SRCH) => (), + Err(error) => return Err(error.into()), + } + } + Ok(()) + } + async fn observe_exit(&mut self) -> Result<()> { + let pid = self.0.context("local image tool ownership lost")?; + loop { + match rustix::process::waitid( + rustix::process::WaitId::Pid(pid), + WaitIdOptions::EXITED | WaitIdOptions::NOWAIT | WaitIdOptions::NOHANG, + ) { + Ok(Some(_)) => return Ok(()), + Ok(None) | Err(rustix::io::Errno::INTR) => (), + Err(error) => { + // An external reaper invalidates numeric PID ownership. + // Never signal that group after this boundary. + if error == rustix::io::Errno::CHILD { + self.0 = None; + } + return Err(error).context("could not observe local image tool exit"); + } + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + } + impl Drop for ProcessGroup { + fn drop(&mut self) { + let _ = self.kill(); + } + } + + pub(super) async fn run(invocation: Invocation) -> Result { + let permit = PROCESSES + .clone() + .try_acquire_owned() + .context("two local image tools are already running")?; + let (mut send, receive) = oneshot::channel(); + tokio::spawn(async move { + let _permit = permit; + let result = async { + ensure!(!send.is_closed() && !invocation.cancel.is_cancelled(), "container build cancelled"); + let _workspace = invocation.workspace; + let mut command = Command::new(&invocation.tool); + command.args(&invocation.args).current_dir(&invocation.cwd).env_clear(); + // No implicit registry, Spacetime, proxy or builder credentials. + for key in ["PATH", "SystemRoot", "TMPDIR", "TEMP", "TMP"] { + if let Some(value) = std::env::var_os(key) { command.env(key, value); } + } + command.envs(invocation.env).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped()).process_group(0); + let mut child = command.spawn().with_context(|| format!("could not run {}; install it or provide its executable path", invocation.tool.display()))?; + let mut group = ProcessGroup(Some(Pid::from_raw(child.id().context("builder PID unavailable")? as i32).context("invalid builder PID")?)); + let stdout = child.stdout.take().context("builder stdout unavailable")?; + let stderr = child.stderr.take().context("builder stderr unavailable")?; + let outcome = { + let completion = async { + let (status, stdout, _) = tokio::try_join!( + async { + group.observe_exit().await?; + group.kill().context("failed to stop builder descendants")?; + Ok::<_, anyhow::Error>(child.wait().await?) + }, + read_bounded(stdout), + read_bounded(stderr), + )?; + ensure!(status.success(), "{} failed ({status}); no prepared output was accepted", invocation.label); + Ok(Output { stdout }) + }; + tokio::select! { + biased; + _ = send.closed() => Err(anyhow::anyhow!("container build caller closed")), + _ = invocation.cancel.cancelled() => Err(anyhow::anyhow!("container build cancelled")), + _ = tokio::time::sleep(invocation.timeout) => Err(anyhow::anyhow!("{} exceeded its build deadline", invocation.label)), + result = completion => result, + } + }; + // Disarmed before every reap, including completion above. Child + // wait is cached if completion already reaped it. + group.kill().context("failed to stop local image tool group")?; + match tokio::time::timeout(Duration::from_secs(5), child.wait()).await { + Ok(result) => { result.context("local image tool could not be reaped")?; }, + Err(_) => { + // The owner retains the workspace and permit throughout + // delayed physical cleanup, even if the caller is gone. + child.wait().await.context("local image tool could not be reaped")?; + bail!("local image tool required delayed physical cleanup"); + } + } + outcome + }.await; + let _ = send.send(result); + }); + receive.await.context("local image tool owner stopped")? + } +} diff --git a/crates/cli/src/container/tests.rs b/crates/cli/src/container/tests.rs new file mode 100644 index 00000000000..9d56133f360 --- /dev/null +++ b/crates/cli/src/container/tests.rs @@ -0,0 +1,713 @@ +use super::*; +use crate::spacetime_config::SpacetimeConfig; +use serde_json::json; +use std::{collections::BTreeMap, sync::Mutex}; + +static TEST_PREPARATIONS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn platform() -> ImagePlatform { + ImagePlatform { + os: "linux".into(), + architecture: "amd64".into(), + } +} +fn declaration(image: serde_json::Value) -> ContainerConfig { + serde_json::from_value(json!({"image":image,"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}})).unwrap() +} +fn blob(layout: &Path, bytes: &[u8], media: &str) -> Descriptor { + let digest = spacetimedb_oci::sha256(bytes); + fs::create_dir_all(layout.join("blobs/sha256")).unwrap(); + fs::write( + layout + .join("blobs/sha256") + .join(digest.to_string().strip_prefix("sha256:").unwrap()), + bytes, + ) + .unwrap(); + Descriptor { + media_type: media.into(), + digest, + size: bytes.len() as u64, + platform: None, + urls: vec![], + data: None, + artifact_type: None, + } +} +fn fixture(layout: &Path) -> Descriptor { + let mut archive = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(4); + header.set_mode(0o644); + header.set_cksum(); + archive.append_data(&mut header, "app.js", &b"code"[..]).unwrap(); + let layer = archive.into_inner().unwrap(); + let layer = blob(layout, &layer, "application/vnd.oci.image.layer.v1.tar"); + let config = blob(layout, &serde_json::to_vec(&json!({"architecture":"amd64","os":"linux","config":{"Entrypoint":["/usr/bin/env"],"Cmd":["node","app.js"],"User":"1000:1000","WorkingDir":"/app","Env":["BAKED=private-image-value"]},"rootfs":{"type":"layers","diff_ids":[layer.digest]}})).unwrap(), spacetimedb_oci::OCI_CONFIG); + let manifest = blob( + layout, + &serde_json::to_vec( + &json!({"schemaVersion":2,"mediaType":spacetimedb_oci::OCI_MANIFEST,"config":config,"layers":[layer]}), + ) + .unwrap(), + spacetimedb_oci::OCI_MANIFEST, + ); + fs::write(layout.join("oci-layout"), br#"{"imageLayoutVersion":"1.0.0"}"#).unwrap(); + fs::write( + layout.join("index.json"), + serde_json::to_vec(&json!({"schemaVersion":2,"mediaType":spacetimedb_oci::OCI_INDEX,"manifests":[manifest]})) + .unwrap(), + ) + .unwrap(); + manifest +} +fn copy_layout(source: &Path, output: &Path) { + fs::create_dir_all(output.join("blobs/sha256")).unwrap(); + for name in ["oci-layout", "index.json"] { + fs::copy(source.join(name), output.join(name)).unwrap(); + } + for file in fs::read_dir(source.join("blobs/sha256")).unwrap() { + let file = file.unwrap(); + fs::copy(file.path(), output.join("blobs/sha256").join(file.file_name())).unwrap(); + } +} +type RecordedCall = (String, Vec, Vec); + +struct FakeRunner { + layout: PathBuf, + calls: Mutex>, + fail_detection: bool, + version: &'static str, +} +impl FakeRunner { + fn new(layout: &Path) -> Self { + Self { + layout: layout.into(), + calls: Mutex::new(vec![]), + fail_detection: false, + version: "railpack 0.35.0\n", + } + } +} +impl Runner for FakeRunner { + async fn run(&self, invocation: Invocation) -> Result { + self.calls.lock().unwrap().push(( + invocation.label.into(), + invocation.args.clone(), + invocation.env.iter().map(|(name, _)| name.clone()).collect(), + )); + match invocation.label { + "Railpack version check" => { + return Ok(process::Output { + stdout: self.version.as_bytes().to_vec(), + }) + } + "Railpack detection" => { + ensure!(!self.fail_detection, "unsupported source detection"); + fs::write(invocation.workspace.path().join("railpack-plan.json"), b"{}")?; + } + "Skopeo image import" => copy_layout(&self.layout, &invocation.workspace.path().join("input")), + "BuildKit OCI build" => { + let file = fs::File::create(invocation.workspace.path().join("image.tar"))?; + let mut tar = tar::Builder::new(file); + for name in ["oci-layout", "index.json"] { + tar.append_path_with_name(self.layout.join(name), name)?; + } + for blob in fs::read_dir(self.layout.join("blobs/sha256"))? { + let blob = blob?; + tar.append_path_with_name(blob.path(), Path::new("blobs/sha256").join(blob.file_name()))?; + } + tar.finish()?; + } + _ => anyhow::bail!("unexpected fake image tool"), + } + Ok(process::Output { stdout: vec![] }) + } +} + +#[test] +fn image_source_is_exclusive_and_builder_defaults_are_strict() { + for image in [ + json!({}), + json!({"build":{},"oci_ref":"example/image"}), + json!({"build":{"builder":"unknown"}}), + json!({"build":{"builder":"railpack","dockerfile":"Dockerfile"}}), + ] { + let mut config = serde_json::to_value(declaration(json!({"oci_ref":"example/image"}))).unwrap(); + config["image"] = image; + assert!(serde_json::from_value::(config).is_err()); + } + assert!(matches!( + declaration(json!({"build":{}})).image, + ImageSource::Build(config::BuildImage { + build: SourceBuild::Dockerfile { .. } + }) + )); +} + +#[test] +fn container_declarations_never_inherit_through_children_or_overrides() { + let a = serde_json::to_value(declaration(json!({"build":{}}))).unwrap(); + let b = serde_json::to_value(declaration(json!({"oci_ref":"example/child"}))).unwrap(); + let config: SpacetimeConfig = serde_json::from_value(json!({"database":"root","module-path":"shared","container":a,"children":[{"database":"plain","children":[{"database":"grandchild"}]},{"database":"own","container":b,"children":[{"database":"own-grandchild"}]}]})).unwrap(); + let targets = config.collect_all_targets_with_inheritance(); + let declarations: BTreeMap<_, _> = targets + .iter() + .map(|target| (target.fields["database"].as_str().unwrap(), target.container.is_some())) + .collect(); + assert_eq!( + declarations, + BTreeMap::from([ + ("root", true), + ("plain", false), + ("grandchild", false), + ("own", true), + ("own-grandchild", false) + ]) + ); + assert!(targets.iter().all(|target| !target.fields.contains_key("container"))); + assert!(crate::subcommands::container::select(&config, Some("plain")).is_err()); + assert!(crate::subcommands::container::select(&config, Some("not-a-local-target")).is_err()); + assert!(crate::subcommands::container::select(&config, None).is_err()); + assert!(crate::subcommands::container::select(&config, Some("own")).is_ok()); +} + +#[tokio::test] +async fn local_prebuilt_output_is_owned_verified_and_keeps_image_defaults() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("prebuilt image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let token = CancellationToken::new(); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"oci:prebuilt image"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + token.clone(), + ) + .await + .unwrap(); + assert!(!token.is_cancelled()); + assert!(runner.calls.lock().unwrap().is_empty()); + assert_eq!(prepared.metadata.manifest.digest, manifest.digest); + assert_eq!(prepared.metadata.container.argv, ["/usr/bin/env", "node", "app.js"]); + assert_eq!(prepared.metadata.container.user, "1000:1000"); + assert_eq!(prepared.metadata.container.working_directory, "/app"); + assert!(!serde_json::to_string(&prepared.metadata) + .unwrap() + .contains("private-image-value")); + assert_eq!(prepared.metadata.objects.len(), 3); + let temporary = prepared.layout(); + let output = root.path().join("reusable"); + prepared.persist(&output).unwrap(); + assert!(!temporary.exists()); + assert!(output.join("prepared.json").is_file()); + let reread = prepare_container( + &declaration(json!({"oci_ref":"oci:reusable"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + token, + ) + .await + .unwrap(); + assert_eq!(reread.metadata.manifest.digest, manifest.digest); + let temporary = reread.layout(); + drop(reread); + assert!(!temporary.exists()); +} + +#[tokio::test] +async fn command_override_replaces_image_argv_and_invalid_bytes_never_persist() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let mut config = declaration(json!({"oci_ref":"oci:image"})); + config.command = Some(vec!["/bin/custom".into()]); + let prepared = prepare_container( + &config, + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(prepared.metadata.container.argv, ["/bin/custom"]); + drop(prepared); + fs::write( + input + .join("blobs/sha256") + .join(manifest.digest.to_string().strip_prefix("sha256:").unwrap()), + b"tampered", + ) + .unwrap(); + assert!(prepare_container( + &config, + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert!(fs::read_dir(root.path()).unwrap().all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".spacetime-image-"))); +} + +#[tokio::test] +async fn dockerfile_and_railpack_have_explicit_local_endpoint_and_separate_secrets() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("fixture"); + fixture(&input); + let context = root.path().join("source with spaces"); + fs::create_dir(&context).unwrap(); + fs::write(context.join("Dockerfile"), b"FROM scratch").unwrap(); + let secret = root.path().join("secret file"); + fs::write(&secret, b"sensitive-build-value").unwrap(); + for builder in ["dockerfile", "railpack"] { + let runner = FakeRunner::new(&input); + let tools = BuildTools { + buildkit_host: Some("unix:///disposable/fake-buildkit.sock".into()), + secrets: vec![BuildSecret { + name: "BUILD_KEY".into(), + file: secret.clone(), + }], + ..Default::default() + }; + let config = declaration(json!({"build":{"builder":builder,"context":"source with spaces"}})); + let prepared = prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + let calls = runner.calls.lock().unwrap(); + let (_, argv, environment) = calls + .iter() + .find(|(label, _, _)| label == "BuildKit OCI build") + .unwrap(); + assert_eq!(&argv[..2], ["--addr", "unix:///disposable/fake-buildkit.sock"]); + assert!(argv.contains(&format!("context={}", context.canonicalize().unwrap().display()).into())); + assert!(argv.contains(&"--no-cache".into())); + assert!(environment.contains(&"DOCKER_CONFIG".into())); + assert!(!format!("{argv:?}").contains("sensitive-build-value")); + assert!(!serde_json::to_string(&prepared.metadata).unwrap().contains("BUILD_KEY")); + if builder == "railpack" { + assert!(argv.contains(&format!("source={RAILPACK_FRONTEND}").into())); + let (_, args, _) = calls + .iter() + .find(|(label, _, _)| label == "Railpack detection") + .unwrap(); + assert!(args.contains(&"BUILD_KEY=".into())); + } else { + assert_eq!(calls.len(), 1); + } + } +} + +#[tokio::test] +async fn unsupported_railpack_and_remote_builder_never_fall_back() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let config = declaration(json!({"build":{"builder":"railpack"}})); + let tools = BuildTools { + buildkit_host: Some("unix:///disposable/fake-buildkit.sock".into()), + ..Default::default() + }; + let mut runner = FakeRunner::new(&input); + runner.fail_detection = true; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert_eq!(runner.calls.lock().unwrap().len(), 2); + runner.calls.lock().unwrap().clear(); + runner.version = "railpack 99.0.0"; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert_eq!(runner.calls.lock().unwrap().len(), 1); + runner.calls.lock().unwrap().clear(); + let tools = BuildTools { + buildkit_host: Some("tcp://untrusted:1234".into()), + ..Default::default() + }; + assert!(prepare_container( + &config, + root.path(), + platform(), + &tools, + root.path(), + &runner, + CancellationToken::new() + ) + .await + .is_err()); + assert!(runner.calls.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn registry_import_uses_explicit_anonymous_auth_and_records_selected_digest() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + let manifest = fixture(&input); + let runner = FakeRunner::new(&input); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"example.invalid/team/image:tag"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &runner, + CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(prepared.metadata.manifest.digest, manifest.digest); + let calls = runner.calls.lock().unwrap(); + assert_eq!(calls.len(), 1); + let args = &calls[0].1; + assert!(args.contains(&"--preserve-digests".into())); + assert!(args.contains(&"docker://example.invalid/team/image:tag".into())); + let auth = PathBuf::from(&args[args.iter().position(|arg| arg == "--authfile").unwrap() + 1]); + assert_eq!(fs::read(auth).unwrap(), br#"{"auths":{}}"#); +} + +#[test] +fn archive_rejects_links_and_cancelled_reads() { + let root = tempfile::tempdir().unwrap(); + for kind in [tar::EntryType::Symlink, tar::EntryType::Link] { + let path = root.path().join("input.tar"); + let mut tar = tar::Builder::new(fs::File::create(&path).unwrap()); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(kind); + header.set_size(0); + header.set_link_name("/outside").unwrap(); + header.set_cksum(); + tar.append_data(&mut header, "index.json", std::io::empty()).unwrap(); + tar.finish().unwrap(); + let output = tempfile::tempdir().unwrap(); + assert!(oci::extract_archive( + &path, + output.path(), + &CancellationToken::new(), + Instant::now() + Duration::from_secs(1) + ) + .is_err()); + } + let token = CancellationToken::new(); + token.cancel(); + assert!(oci::check(&token, Instant::now() + Duration::from_secs(1)).is_err()); +} + +#[test] +fn container_build_command_requires_explicit_platform_and_output() { + crate::subcommands::container::cli().debug_assert(); + assert!(crate::subcommands::container::cli() + .try_get_matches_from(["container", "build"]) + .is_err()); + assert!(crate::subcommands::container::cli() + .try_get_matches_from([ + "container", + "build", + "local-target", + "--platform", + "linux/amd64", + "--out-dir", + "output" + ]) + .is_ok()); +} + +#[test] +fn legacy_publish_rejects_container_targets_without_affecting_plain_children() { + use crate::subcommands::publish::{build_publish_schema, get_filtered_publish_configs}; + let config: SpacetimeConfig = serde_json::from_value(json!({ + "database":"container-db", "container":declaration(json!({"build":{}})), + "children":[{"database":"ordinary-db"}] + })) + .unwrap(); + let command = crate::subcommands::publish::cli(); + let schema = build_publish_schema(&command).unwrap(); + for selected in ["container-db", "*"] { + let args = command.clone().try_get_matches_from(["publish", selected]).unwrap(); + let error = get_filtered_publish_configs(&config, &command, &schema, &args) + .unwrap_err() + .to_string(); + assert!(error.contains("does not yet publish container")); + } + let args = command + .clone() + .try_get_matches_from(["publish", "ordinary-db"]) + .unwrap(); + assert_eq!( + get_filtered_publish_configs(&config, &command, &schema, &args) + .unwrap() + .len(), + 1 + ); +} + +#[tokio::test] +async fn local_index_defaults_media_type_but_never_overwrites_existing_output() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let mut index: serde_json::Value = serde_json::from_slice(&fs::read(input.join("index.json")).unwrap()).unwrap(); + index.as_object_mut().unwrap().remove("mediaType"); + fs::write(input.join("index.json"), serde_json::to_vec(&index).unwrap()).unwrap(); + let prepared = prepare_container( + &declaration(json!({"oci_ref":"oci:image"})), + root.path(), + platform(), + &BuildTools::default(), + root.path(), + &FakeRunner::new(&input), + CancellationToken::new(), + ) + .await + .unwrap(); + let output = root.path().join("existing-empty-output"); + fs::create_dir(&output).unwrap(); + assert!(prepared.persist(&output).is_err()); + assert!(fs::read_dir(&output).unwrap().next().is_none()); +} + +#[test] +fn wrong_platform_and_duplicate_archive_object_are_rejected() { + let root = tempfile::tempdir().unwrap(); + let input = root.path().join("image"); + fixture(&input); + let wrong = ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }; + assert!(oci::verify_layout( + &input, + &root.path().join("output"), + &wrong, + &CancellationToken::new(), + Instant::now() + Duration::from_secs(2) + ) + .is_err()); + let archive = root.path().join("duplicate.tar"); + let mut tar = tar::Builder::new(fs::File::create(&archive).unwrap()); + for _ in 0..2 { + tar.append_path_with_name(input.join("index.json"), "index.json") + .unwrap(); + } + tar.finish().unwrap(); + assert!(oci::extract_archive( + &archive, + &root.path().join("duplicate"), + &CancellationToken::new(), + Instant::now() + Duration::from_secs(2) + ) + .is_err()); +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[tokio::test] +async fn subprocess_exit_cancellation_caller_drop_and_output_overflow_reap_before_workspace_release() { + use process::LocalRunner; + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + // These scripts exercise only owned local subprocesses. No Docker, server, + // network, saved configuration, or user credentials are involved. + for mode in ["exit", "cancel", "drop", "overflow", "timeout"] { + let workspace = Arc::new( + tempfile::Builder::new() + .prefix("fake-tool-") + .tempdir_in(root.path()) + .unwrap(), + ); + let path = workspace.path().to_path_buf(); + let pid_file = root.path().join(format!("{mode}.pid")); + let script = match mode { + "exit" => "echo $$ > \"$1\"; sleep 60 & exit 0", + "overflow" => "echo $$ > \"$1\"; yes x", + _ => "echo $$ > \"$1\"; sleep 60 & wait", + }; + let token = CancellationToken::new(); + let task = tokio::spawn(LocalRunner.run(Invocation { + tool: "/bin/sh".into(), + label: "owned fake builder", + args: vec![ + "-c".into(), + script.into(), + "fake-builder".into(), + pid_file.clone().into_os_string(), + ], + env: vec![], + cwd: root.path().into(), + workspace, + timeout: if mode == "timeout" { + Duration::from_millis(250) + } else { + Duration::from_secs(4) + }, + cancel: token.clone(), + })); + tokio::time::timeout(Duration::from_secs(2), async { + while !pid_file.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let pid = fs::read_to_string(&pid_file).unwrap().trim().parse::().unwrap(); + let pid = rustix::process::Pid::from_raw(pid).unwrap(); + if mode == "cancel" { + token.cancel(); + } + if mode == "drop" { + task.abort(); + let _ = task.await; + } else { + let result = tokio::time::timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap(); + assert_eq!(result.is_ok(), mode == "exit"); + } + tokio::time::timeout(Duration::from_secs(3), async { + while path.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!( + matches!( + rustix::process::waitpid(Some(pid), rustix::process::WaitOptions::NOHANG), + Err(rustix::io::Errno::CHILD) + ), + "builder leader must already be reaped" + ); + } +} + +#[test] +fn configuration_overlay_replaces_only_the_selected_container_declaration() { + let root = tempfile::tempdir().unwrap(); + let parent = declaration(json!({"build":{}})); + let child = declaration(json!({"oci_ref":"example.invalid/child:v1"})); + fs::write(root.path().join("spacetime.json"),serde_json::to_vec(&json!({"database":"parent","container":parent,"children":[{"database":"plain"},{"database":"own","container":child}]})).unwrap()).unwrap(); + let replacement = declaration(json!({"build":{"builder":"railpack"}})); + fs::write( + root.path().join("spacetime.dev.json"), + serde_json::to_vec(&json!({"container":replacement})).unwrap(), + ) + .unwrap(); + let loaded = crate::spacetime_config::find_and_load_with_env_from(Some("dev"), root.path().into()) + .unwrap() + .unwrap(); + assert!(matches!( + crate::subcommands::container::select(&loaded.config, Some("parent")) + .unwrap() + .image, + ImageSource::Build(config::BuildImage { + build: SourceBuild::Railpack { .. } + }) + )); + assert!(crate::subcommands::container::select(&loaded.config, Some("plain")).is_err()); + assert!(matches!( + crate::subcommands::container::select(&loaded.config, Some("own")) + .unwrap() + .image, + ImageSource::Prebuilt(_) + )); +} + +#[tokio::test] +async fn build_command_executes_local_import_with_only_project_configuration() { + let _serial = TEST_PREPARATIONS.lock().await; + let root = tempfile::tempdir().unwrap(); + fixture(&root.path().join("input")); + fs::write( + root.path().join("spacetime.json"), + serde_json::to_vec(&json!({ + "database":"local-selection", "server":"https://must-not-be-contacted.invalid", + "container":declaration(json!({"oci_ref":"oci:input"})) + })) + .unwrap(), + ) + .unwrap(); + let output = root.path().join("output"); + let args = crate::subcommands::container::cli() + .try_get_matches_from([ + OsString::from("container"), + "build".into(), + "local-selection".into(), + "--project-path".into(), + root.path().into(), + "--out-dir".into(), + output.clone().into_os_string(), + "--platform".into(), + "linux/amd64".into(), + ]) + .unwrap(); + crate::exec_local_subcommand("container", &args).await.unwrap().unwrap(); + let metadata: PreparedMetadata = serde_json::from_slice(&fs::read(output.join("prepared.json")).unwrap()).unwrap(); + assert_eq!(metadata.container.argv, ["/usr/bin/env", "node", "app.js"]); + assert_eq!(metadata.objects.len(), 3); +} + +#[test] +fn omitted_image_working_directory_normalizes_to_linux_root() { + let declaration = declaration(json!({"oci_ref":"example.invalid/no-working-dir"})); + let image = spacetimedb_oci::ContainerConfig { + cmd: Some(vec!["/app".into()]), + ..Default::default() + }; + let normalized = declaration + .normalize(spacetimedb_oci::sha256(b"fixture"), platform(), &image) + .unwrap(); + assert_eq!(normalized.working_directory, "/"); + let mut explicit = declaration; + explicit.working_directory = Some(String::new()); + assert!(explicit + .normalize(spacetimedb_oci::sha256(b"fixture"), platform(), &image) + .is_err()); +} diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index c44b3d2a41e..9340d6cc66e 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,6 +1,7 @@ pub mod api; mod common_args; mod config; +pub mod container; pub(crate) mod detect; mod edit_distance; mod errors; @@ -35,6 +36,7 @@ pub fn get_subcommands() -> Vec { logout::cli(), init::cli(), build::cli(), + subcommands::container::cli(), server::cli(), subscribe::cli(), start::cli(), @@ -42,6 +44,17 @@ pub fn get_subcommands() -> Vec { ] } +/// Dispatch commands that need only project files before opening saved CLI +/// server settings or credentials. Future container network commands use the +/// ordinary authenticated dispatcher below. +pub async fn exec_local_subcommand(cmd: &str, args: &ArgMatches) -> Option> { + if cmd == "container" && args.subcommand_name() == Some("build") { + Some(subcommands::container::exec(args).await.map(|()| ExitCode::SUCCESS)) + } else { + None + } +} + pub async fn exec_subcommand( config: Config, paths: &SpacetimePaths, @@ -62,6 +75,7 @@ pub async fn exec_subcommand( "list" => list::exec(config, args).await, "init" => init::exec(config, args).await.map(|_| ()), "build" => build::exec(config, args).await.map(drop), + "container" => subcommands::container::exec(args).await, "server" => server::exec(config, paths, args).await, "subscribe" => subscribe::exec(config, args).await, "start" => return start::exec(config, paths, args).await, diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 6f2e4d23f08..35c1a2bfed8 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -32,6 +32,10 @@ async fn main() -> anyhow::Result { let matches = get_command().get_matches(); let (cmd, subcommand_args) = matches.subcommand().unwrap(); + if let Some(result) = exec_local_subcommand(cmd, subcommand_args).await { + return result; + } + let root_dir = matches.get_one::("root_dir"); let paths = match root_dir { Some(dir) => SpacetimePaths::from_root_dir(dir), diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index b3316f2c318..51a31fd7ec8 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -108,6 +108,9 @@ pub enum CommandConfigError { #[derive(Debug, Clone, Serialize, Deserialize, Default)] #[serde(rename_all = "kebab-case")] pub struct SpacetimeConfig { + /// Container declaration belongs only to this database, never its children. + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, /// Configuration for the dev command. Root-level only, not inherited. #[serde(skip_serializing_if = "Option::is_none")] pub dev: Option, @@ -144,6 +147,7 @@ pub struct DevConfig { /// Contains all fields needed for both publish and generate operations. #[derive(Debug, Clone)] pub struct FlatTarget { + pub container: Option, /// All entity-level fields (database, module-path, server, etc.) pub fields: HashMap, /// Name of the config file from which this target's `database` value was merged. @@ -201,6 +205,7 @@ impl SpacetimeConfig { let effective_generate = self.generate.clone(); let target = FlatTarget { + container: self.container.clone(), fields: fields.clone(), source_config: self.source_config.clone(), generate: effective_generate, diff --git a/crates/cli/src/subcommands/container.rs b/crates/cli/src/subcommands/container.rs new file mode 100644 index 00000000000..4248996798b --- /dev/null +++ b/crates/cli/src/subcommands/container.rs @@ -0,0 +1,174 @@ +//! Local container tooling. Database selection here never contacts a server. +use crate::{ + container::{config::ContainerConfig, prepare_container, process::LocalRunner, BuildSecret, BuildTools}, + spacetime_config::{find_and_load_with_env_from, SpacetimeConfig}, +}; +use anyhow::{ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use std::path::{Path, PathBuf}; +use tokio_util::sync::CancellationToken; + +pub fn cli() -> Command { + Command::new("container") + .about("Build and manage a database's container") + .subcommand_required(true) + .subcommand( + Command::new("build") + .about("Prepare verified OCI artifacts locally without publishing") + .arg(Arg::new("database").help("Database target in local spacetime.json; no server lookup")) + .arg( + Arg::new("project_path") + .long("project-path") + .default_value(".") + .value_parser(clap::value_parser!(PathBuf)) + .help("Directory in which to find spacetime.json"), + ) + .arg( + Arg::new("out_dir") + .long("out-dir") + .required(true) + .value_parser(clap::value_parser!(PathBuf)) + .help("New directory for verified OCI artifacts and prepared.json"), + ) + .arg( + Arg::new("platform") + .long("platform") + .required(true) + .value_parser(["linux/amd64", "linux/arm64"]) + .help("Target Linux platform, independent of this computer's architecture"), + ) + .arg(Arg::new("env").long("env").help("Local configuration overlay name")) + .arg( + Arg::new("buildkit_host") + .long("buildkit-host") + .help("Explicit local BuildKit Unix socket, required for source builds"), + ) + .arg( + Arg::new("buildctl") + .long("buildctl") + .default_value("buildctl") + .value_parser(clap::value_parser!(PathBuf)) + .help("BuildKit client executable"), + ) + .arg( + Arg::new("railpack") + .long("railpack") + .default_value("railpack") + .value_parser(clap::value_parser!(PathBuf)) + .help("Pinned Railpack executable for explicitly selected Railpack builds"), + ) + .arg( + Arg::new("skopeo") + .long("skopeo") + .default_value("skopeo") + .value_parser(clap::value_parser!(PathBuf)) + .help("Skopeo executable for prebuilt registry images"), + ) + .arg( + Arg::new("registry_auth_file") + .long("registry-auth-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Explicit registry auth JSON; omitted means anonymous, never saved Docker credentials"), + ) + .arg( + Arg::new("build_secret") + .long("build-secret") + .action(ArgAction::Append) + .value_name("NAME=FILE") + .help("Explicit build secret file; separate from runtime env_keys"), + ), + ) +} + +pub(crate) fn select(config: &SpacetimeConfig, database: Option<&str>) -> Result { + let targets = config.collect_all_targets_with_inheritance(); + let selected = if let Some(database) = database { + let mut matches = targets + .iter() + .filter(|target| target.fields.get("database").and_then(|v| v.as_str()) == Some(database)); + let target = matches + .next() + .context("database target is not in local spacetime.json")?; + ensure!( + matches.next().is_none(), + "database target is ambiguous in local spacetime.json" + ); + target + } else { + let mut matches = targets.iter().filter(|target| target.container.is_some()); + let target = matches + .next() + .context("no container declaration in local spacetime.json")?; + ensure!( + matches.next().is_none(), + "several container targets exist; select a DATABASE from local spacetime.json" + ); + target + }; + selected + .container + .clone() + .context("selected database has no container declaration; containers are not inherited") +} + +pub async fn exec(args: &ArgMatches) -> Result<()> { + let ("build", args) = args.subcommand().context("missing container command")? else { + anyhow::bail!("unsupported container command"); + }; + let project = args.get_one::("project_path").unwrap().canonicalize()?; + let loaded = find_and_load_with_env_from(args.get_one::("env").map(String::as_str), project)? + .context("spacetime.json not found")?; + let declaration = select(&loaded.config, args.get_one::("database").map(String::as_str))?; + let output = std::env::current_dir()?.join(args.get_one::("out_dir").unwrap()); + ensure!(!output.exists(), "output already exists: {}", output.display()); + let parent = output.parent().unwrap_or(Path::new(".")); + ensure!(parent.is_dir(), "output parent directory does not exist"); + let mut tools = BuildTools { + buildctl: args.get_one::("buildctl").unwrap().clone(), + railpack: args.get_one::("railpack").unwrap().clone(), + skopeo: args.get_one::("skopeo").unwrap().clone(), + buildkit_host: args.get_one::("buildkit_host").cloned(), + registry_auth_file: args.get_one::("registry_auth_file").cloned(), + secrets: vec![], + }; + for secret in args.get_many::("build_secret").into_iter().flatten() { + let (name, file) = secret + .split_once('=') + .context("build secret must be NAME=FILE, not a value")?; + ensure!(!file.is_empty(), "build secret file is missing"); + tools.secrets.push(BuildSecret { + name: name.into(), + file: file.into(), + }); + } + let (os, architecture) = args.get_one::("platform").unwrap().split_once('/').unwrap(); + let cancel = CancellationToken::new(); + let prepared = { + let prepare = prepare_container( + &declaration, + &loaded.config_dir, + spacetimedb_lib::container::ImagePlatform { + os: os.into(), + architecture: architecture.into(), + }, + &tools, + parent, + &LocalRunner, + cancel.clone(), + ); + tokio::pin!(prepare); + tokio::select! { + result = &mut prepare => result?, + signal = tokio::signal::ctrl_c() => { + signal?; + cancel.cancel(); + let _ = prepare.await; + anyhow::bail!("container build cancelled; no output was published"); + } + } + }; + let digest = prepared.metadata.manifest.digest; + prepared.persist(&output)?; + println!("Prepared {digest} at {}", output.display()); + Ok(()) +} diff --git a/crates/cli/src/subcommands/mod.rs b/crates/cli/src/subcommands/mod.rs index 58456274469..677a5e9ce9b 100644 --- a/crates/cli/src/subcommands/mod.rs +++ b/crates/cli/src/subcommands/mod.rs @@ -1,5 +1,6 @@ pub mod build; pub mod call; +pub mod container; pub mod db_arg_resolution; pub mod delete; pub mod describe; diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index dd710da2832..6e66ac17ff9 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -171,6 +171,8 @@ pub fn get_filtered_publish_configs<'a>( let configs: Vec = filtered_targets .into_iter() .map(|target| { + anyhow::ensure!(target.container.is_none(), + "This CLI does not yet publish container declarations. Use `spacetime container build` to prepare the image; managed publication support is required before publishing this target."); let config = CommandConfig::new(schema, target.fields, args)?; config.validate()?; Ok(config) From d05b60ef27cb16944bee75b415b2174df33711c4 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 06:05:47 -0400 Subject: [PATCH 11/23] Reconcile restored container authority before receiving admission --- crates/core/src/auth/invocation.rs | 8 +- crates/core/src/auth/invocation/tests.rs | 57 +- crates/core/src/client/client_connection.rs | 25 +- crates/core/src/db/deployment.rs | 68 ++ crates/core/src/db/hosted_admission.rs | 282 +++++++ crates/core/src/db/mod.rs | 1 + crates/core/src/db/relational_db.rs | 15 +- crates/core/src/host/container_fence.rs | 204 ++++++ crates/core/src/host/container_fence/tests.rs | 389 ++++++++++ crates/core/src/host/empty_module.rs | 2 +- crates/core/src/host/empty_module/tests.rs | 9 + crates/core/src/host/host_controller.rs | 606 ++++++++------- .../core/src/host/host_controller/registry.rs | 339 +++++++++ .../core/src/host/host_controller/retained.rs | 251 +++++++ .../host/host_controller/retained_tests.rs | 689 ++++++++++++++++++ crates/core/src/host/instance_env.rs | 5 +- crates/core/src/host/mod.rs | 1 + crates/core/src/host/module_host.rs | 6 +- .../src/host/wasm_common/module_host_actor.rs | 4 +- crates/core/src/sql/execute.rs | 2 +- .../subscription/module_subscription_actor.rs | 40 +- crates/lib/src/deployment.rs | 12 + .../lib/src/deployment/system_empty_v1.wasm | Bin 0 -> 250 bytes 23 files changed, 2652 insertions(+), 363 deletions(-) create mode 100644 crates/core/src/db/hosted_admission.rs create mode 100644 crates/core/src/host/container_fence.rs create mode 100644 crates/core/src/host/container_fence/tests.rs create mode 100644 crates/core/src/host/host_controller/registry.rs create mode 100644 crates/core/src/host/host_controller/retained.rs create mode 100644 crates/core/src/host/host_controller/retained_tests.rs create mode 100644 crates/lib/src/deployment/system_empty_v1.wasm diff --git a/crates/core/src/auth/invocation.rs b/crates/core/src/auth/invocation.rs index 3c5fdf6e82f..dc9004effaa 100644 --- a/crates/core/src/auth/invocation.rs +++ b/crates/core/src/auth/invocation.rs @@ -101,12 +101,16 @@ impl InvocationCaller { /// A check before queueing does not serialize with generation revocation. pub(crate) fn check_hosted_admission( state: &S, - target: Identity, + database: &crate::db::relational_db::RelationalDB, proof: Option<&VerifiedHostedAuth>, ) -> anyhow::Result<()> { let Some(proof) = proof else { return Ok(()) }; anyhow::ensure!( - proof.target_database() == target, + database.hosted_admission().is_open(), + "receiving database has not reconciled hosted admission" + ); + anyhow::ensure!( + proof.target_database() == database.database_identity(), "hosted credential targets another database" ); proof.check_at(SystemTime::now())?; diff --git a/crates/core/src/auth/invocation/tests.rs b/crates/core/src/auth/invocation/tests.rs index 5d1ed1f1537..0507ca32bdb 100644 --- a/crates/core/src/auth/invocation/tests.rs +++ b/crates/core/src/auth/invocation/tests.rs @@ -112,13 +112,17 @@ fn transaction_admission_rechecks_persisted_fences_after_initial_authentication( let source = Identity::ONE; let auth = authenticate(source, target, SystemTime::now()); let proof = auth.hosted.as_ref(); + let wrong_target = authenticate(source, Identity::from_u256(99u64.into()), SystemTime::now()); + db.hosted_admission().begin().unwrap().complete().unwrap(); let caller = InvocationCaller::from(&auth); assert_eq!(caller.flags_for(target, &module(true)).unwrap(), 0); db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { - assert!(check_hosted_admission(tx, target, proof).is_err()); + assert!(check_hosted_admission(tx, &db, proof).is_err()); install_container_fence(&db, tx, &fence(source, 3, 7, true))?; - check_hosted_admission(tx, target, proof)?; - assert!(check_hosted_admission(tx, Identity::from_u256(99u64.into()), proof).is_err()); + assert!(!db.hosted_admission().is_open()); + db.hosted_admission().begin()?.complete()?; + check_hosted_admission(tx, &db, proof)?; + assert!(check_hosted_admission(tx, &db, wrong_target.hosted.as_ref()).is_err()); Ok(()) }) .unwrap(); @@ -129,28 +133,67 @@ fn transaction_admission_rechecks_persisted_fences_after_initial_authentication( }) .unwrap(); db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { - assert!(check_hosted_admission(tx, target, proof).is_err()); - check_hosted_admission(tx, target, None)?; + assert!(check_hosted_admission(tx, &db, proof).is_err()); + check_hosted_admission(tx, &db, None)?; // Another generation does not reactivate a copied credential. install_container_fence(&db, tx, &fence(source, 5, 9, true))?; - assert!(check_hosted_admission(tx, target, proof).is_err()); + assert!(check_hosted_admission(tx, &db, proof).is_err()); Ok(()) }) .unwrap(); } +#[test] +fn reopened_database_requires_a_fresh_sweep_despite_a_replayed_allowed_fence() { + let db = TestDB::durable().unwrap(); + let source = Identity::ONE; + let target = db.database_identity(); + let auth = authenticate(source, target, SystemTime::now()); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &fence(source, 3, 7, true)) + }) + .unwrap(); + let old_sweep = db.hosted_admission().begin().unwrap(); + let db = db.reopen().unwrap(); + // Shutdown seals the old object, including any late coordinator ticket. + assert!(old_sweep.complete().is_err()); + assert!(!db.hosted_admission().is_open()); + db.with_read_only(Workload::ForTests, |tx| { + crate::db::deployment::check_container_fence(tx, source, 3, 7).unwrap(); + let error = check_hosted_admission(tx, &db, auth.hosted.as_ref()).unwrap_err(); + assert!(error.to_string().contains("has not reconciled")); + check_hosted_admission(tx, &db, None).unwrap(); + }); + db.hosted_admission().begin().unwrap().complete().unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + check_hosted_admission(tx, &db, auth.hosted.as_ref()).unwrap(); + }); +} + #[test] fn expired_verified_proof_is_rejected_at_both_call_and_transaction_admission() { let db = TestDB::in_memory().unwrap(); let target = db.database_identity(); let source = Identity::ONE; + db.hosted_admission().begin().unwrap().complete().unwrap(); // Valid when received, expired before execution, without sleeps or forged proofs. let auth = authenticate(source, target, SystemTime::now() - Duration::from_secs(60)); assert!(InvocationCaller::from(&auth).flags_for(target, &module(true)).is_err()); db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { install_container_fence(&db, tx, &fence(source, 3, 7, true))?; - assert!(check_hosted_admission(tx, target, auth.hosted.as_ref()).is_err()); + assert!(check_hosted_admission(tx, &db, auth.hosted.as_ref()).is_err()); Ok(()) }) .unwrap(); } + +#[tokio::test] +async fn shutdown_seals_hosted_admission_even_for_retained_memory_database_handles() { + let db = TestDB::in_memory().unwrap(); + let pending = db.hosted_admission().begin().unwrap(); + assert_eq!(db.shutdown().await, None); + assert!(pending.complete().is_err()); + assert!(!db.hosted_admission().is_open()); + assert!(db.hosted_admission().begin().is_err()); + assert_eq!(db.shutdown().await, None); +} diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index af45a3ea748..85edfece555 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -170,9 +170,7 @@ impl DurableOffsetSupply for Arc { let proof = proof.clone(); Box::pin(async move { tokio::task::spawn_blocking(move || { - db.with_read_only(Workload::Internal, |tx| { - check_hosted_admission(tx, db.database_identity(), Some(&proof)) - }) + db.with_read_only(Workload::Internal, |tx| check_hosted_admission(tx, &db, Some(&proof))) }) .await? }) @@ -1495,6 +1493,11 @@ mod tests { } fn hosted_auth(db: &RelationalDB, lifetime: std::time::Duration) -> ConnectionAuthCtx { + // These fixtures model an already reconciled receiving host. Tests of + // startup closure explicitly close the gate after constructing it. + if !db.hosted_admission().is_open() { + db.hosted_admission().begin().unwrap().complete().unwrap(); + } use crate::auth::{ hosted_tokens::{sign_hosted_token, HostedTokenBinding, HostedTokenValidator}, JwtKeys, @@ -1596,6 +1599,22 @@ mod tests { assert_received_update(ordinary_rx.recv()).await; } + #[tokio::test] + async fn hosted_queued_delivery_rejects_closed_startup_gate_with_unchanged_fence() { + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (sender, mut receiver, actor) = + hosted_client(&db, db.db.clone(), false, std::time::Duration::from_secs(20)); + sender.send_message(None, empty_tx_update()).unwrap(); + db.hosted_admission().close(); + assert_receiver_closed(receiver.recv()).await; + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + let (ordinary, mut ordinary_rx) = default_client(db.db.clone()); + ordinary.send_message(None, empty_tx_update()).unwrap(); + assert_received_update(ordinary_rx.recv()).await; + } + #[tokio::test] async fn hosted_confirmed_delivery_rechecks_fence_after_durability_wait() { let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); diff --git a/crates/core/src/db/deployment.rs b/crates/core/src/db/deployment.rs index 53d0e800b77..8ee66d385c8 100644 --- a/crates/core/src/db/deployment.rs +++ b/crates/core/src/db/deployment.rs @@ -38,6 +38,8 @@ pub enum DeploymentError { ContainerFenced, #[error("a conflicting or older container fence cannot replace current authority")] FenceConflict, + #[error("the receiving host fence revision is exhausted")] + FenceRevisionExhausted, #[error("deployment metadata is inconsistent")] CorruptMetadata, #[error(transparent)] @@ -446,12 +448,78 @@ pub fn install_container_fence( if next.generation <= previous.generation || next.target_grant_revision < previous.target_grant_revision { return Err(DeploymentError::FenceConflict); } + db.hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + } else { + db.hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; } tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, next)?; Ok(()) } +/// An exact denial of a previously observed allowed fence. Revision counters +/// belong to this live database open, not to the durable generation namespace. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FenceDenial { + pub row: StContainerFenceRow, + pub revision_before: u64, + pub revision_after: u64, +} + +/// Deny an orphan without lowering or inventing a control generation. +/// +/// The trusted coordinator must establish the source's absence from current +/// control inventory, then recheck that inventory revision while holding this +/// same serializable transaction. This function does not confer authority to +/// any external caller. Only the exact allowed tuple, or its already-denied +/// form, is accepted. Ordinary installation cannot reopen this generation. +pub fn deny_container_fence( + db: &RelationalDB, + tx: &mut MutTx, + expected: &StContainerFenceRow, +) -> Result { + if !expected.allowed || expected.generation == 0 { + return Err(DeploymentError::FenceConflict); + } + let key: AlgebraicValue = expected.source_identity.into(); + let (pointer, current) = tx + .iter_by_col_eq(ST_CONTAINER_FENCE_ID, ColId(0), &key)? + .next() + .map(|row| StContainerFenceRow::try_from(row).map(|value| (row.pointer(), value))) + .transpose()? + .ok_or(DeploymentError::FenceConflict)?; + let denied = StContainerFenceRow { + allowed: false, + ..expected.clone() + }; + let (revision_before, revision_after) = if current == denied { + let revision = db.hosted_admission().fence_revision(); + (revision, revision) + } else { + if current != *expected { + return Err(DeploymentError::FenceConflict); + } + // The gate is closed before the row changes. A rollback still + // conservatively invalidates any concurrently progressing page scan. + let revisions = db + .hosted_admission() + .fences_changed() + .map_err(|_| DeploymentError::FenceRevisionExhausted)?; + db.delete(tx, ST_CONTAINER_FENCE_ID, [pointer]); + tx.insert_via_serialize_bsatn(ST_CONTAINER_FENCE_ID, &denied)?; + revisions + }; + Ok(FenceDenial { + row: denied, + revision_before, + revision_after, + }) +} + /// Called with verified hosted credentials inside every admitted transaction, /// including each later transaction of a procedure. Signature, audience, expiry, /// capability, and interface checks are additional receiving-host requirements. diff --git a/crates/core/src/db/hosted_admission.rs b/crates/core/src/db/hosted_admission.rs new file mode 100644 index 00000000000..59b6f30ef1c --- /dev/null +++ b/crates/core/src/db/hosted_admission.rs @@ -0,0 +1,282 @@ +//! Transient receiving-host admission, separate from durable generation fences. +//! +//! A replayed fence can be older than current control authority. Every database +//! open starts closed, including cold maintenance opens and restored databases. +//! Trusted platform code must reconcile the complete incoming fence inventory +//! before completing a sweep. This state is never serialized or restored. + +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; + +use anyhow::ensure; +use parking_lot::Mutex; + +#[derive(Default)] +struct State { + revision: u64, + fence_revision: u64, + sweeping: bool, +} + +#[derive(Default)] +struct Inner { + open: AtomicBool, + state: Mutex, +} + +/// Owned by one live RelationalDB. Opening another copy of the same database +/// Identity creates an independent, closed gate. +#[derive(Default)] +pub struct HostedAdmission(Arc); + +impl HostedAdmission { + pub fn is_open(&self) -> bool { + self.0.open.load(Ordering::Acquire) + } + + /// Start one bounded reconciliation. A competing sweep fails immediately. + /// Dropping its ticket keeps admission closed and permits a later retry. + pub fn begin(&self) -> anyhow::Result { + let mut state = self.0.state.lock(); + ensure!(!state.sweeping, "hosted admission reconciliation is already running"); + self.0.open.store(false, Ordering::Release); + let revision = state + .revision + .checked_add(1) + .filter(|value| *value != u64::MAX) + .ok_or_else(|| anyhow::anyhow!("hosted admission revision exhausted"))?; + state.revision = revision; + state.sweeping = true; + Ok(HostedAdmissionSweep { + inner: self.0.clone(), + revision, + fence_revision: state.fence_revision, + }) + } + + /// Read while holding the database transaction that observes fence rows. + pub fn fence_revision(&self) -> u64 { + self.0.state.lock().fence_revision + } + + /// Call before changing a durable fence, within its mutation transaction. + /// Even a later rollback conservatively invalidates an inventory scan. A + /// delayed installation after startup closes admission again, so it cannot + /// insert an unexamined allowed row behind a completed scan's cursor. + pub fn fences_changed(&self) -> anyhow::Result<(u64, u64)> { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + let previous = state.fence_revision; + let Some(next) = previous.checked_add(1).filter(|next| *next != u64::MAX) else { + state.fence_revision = u64::MAX; + state.revision = u64::MAX; + state.sweeping = false; + anyhow::bail!("hosted fence revision exhausted"); + }; + state.fence_revision = next; + Ok((previous, next)) + } + + /// Invalidate any outstanding sweep. This prevents new admission; it is + /// not a substitute for a transactional fence and positive actor drainage. + pub fn close(&self) { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + state.revision = state.revision.saturating_add(1); + state.sweeping = false; + } + + /// Permanently close a database whose storage writer is being shut down. + /// A retained handle cannot start a new sweep on that obsolete object. + pub fn seal(&self) { + let mut state = self.0.state.lock(); + self.0.open.store(false, Ordering::Release); + state.revision = u64::MAX; + state.sweeping = false; + } +} + +/// A ticket is tied to the exact database-open state that issued it. Completing +/// one cannot open a replacement database, or undo a later close operation. +#[must_use = "dropping the sweep leaves hosted admission closed"] +pub struct HostedAdmissionSweep { + inner: Arc, + revision: u64, + fence_revision: u64, +} + +impl HostedAdmissionSweep { + /// Retain this guard in the async owner when moving the ticket to physical + /// blocking work. Cancelling that owner invalidates this exact sweep, even + /// when dropping the blocking task's JoinHandle cannot stop its execution. + pub fn cancellation_guard(&self) -> HostedAdmissionCancellation { + HostedAdmissionCancellation { + inner: Some(self.inner.clone()), + revision: self.revision, + } + } + + /// Trusted platform code calls this only after durable fence installation, + /// actor completion and current control confirmation of the entire sweep. + pub fn complete(self) -> anyhow::Result<()> { + let revision = self.fence_revision; + self.complete_with_fence_revision(revision) + } + + /// Complete a scan which made its own confirmed fence mutations. The + /// caller holds the actual DB transaction and has verified every stored + /// fence against current authority at this exact physical revision. + pub fn complete_with_fence_revision(self, fence_revision: u64) -> anyhow::Result<()> { + let mut state = self.inner.state.lock(); + ensure!( + state.sweeping && state.revision == self.revision, + "hosted admission reconciliation was invalidated" + ); + ensure!( + fence_revision != u64::MAX && state.fence_revision == fence_revision, + "hosted fence inventory changed during reconciliation" + ); + state.sweeping = false; + self.inner.open.store(true, Ordering::Release); + Ok(()) + } +} + +/// Cancellation belongs to one sweep revision. A late guard cannot close a +/// newer retry, and completion and cancellation serialize on the same mutex. +#[must_use = "retain until physical completion and disarm only after success"] +pub struct HostedAdmissionCancellation { + inner: Option>, + revision: u64, +} + +impl HostedAdmissionCancellation { + pub fn disarm(mut self) { + self.inner = None; + } +} + +impl Drop for HostedAdmissionCancellation { + fn drop(&mut self) { + let Some(inner) = &self.inner else { return }; + let mut state = inner.state.lock(); + if state.revision == self.revision { + inner.open.store(false, Ordering::Release); + state.revision = state.revision.saturating_add(1); + state.sweeping = false; + } + } +} + +impl Drop for HostedAdmissionSweep { + fn drop(&mut self) { + let mut state = self.inner.state.lock(); + if state.revision == self.revision { + state.sweeping = false; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cancelled_sweep_retries_and_cannot_open_another_database() { + let original = HostedAdmission::default(); + let replacement = HostedAdmission::default(); + assert!(!original.is_open()); + let sweep = original.begin().unwrap(); + assert!(original.begin().is_err()); + drop(sweep); + assert!(!original.is_open()); + original.begin().unwrap().complete().unwrap(); + assert!(original.is_open()); + assert!(!replacement.is_open()); + } + + #[test] + fn stale_completion_and_drop_cannot_open_or_cancel_a_newer_sweep() { + let gate = HostedAdmission::default(); + let stale = gate.begin().unwrap(); + gate.close(); + let current = gate.begin().unwrap(); + assert!(stale.complete().is_err()); + assert!(!gate.is_open()); + assert!(gate.begin().is_err()); + current.complete().unwrap(); + assert!(gate.is_open()); + gate.close(); + assert!(!gate.is_open()); + } + + #[test] + fn revision_exhaustion_remains_closed() { + let gate = HostedAdmission::default(); + gate.0.state.lock().revision = u64::MAX - 2; + gate.begin().unwrap().complete().unwrap(); + assert!(gate.is_open()); + assert!(gate.begin().is_err()); + assert!(!gate.is_open()); + gate.close(); + assert!(gate.begin().is_err()); + assert!(!gate.is_open()); + } + + #[test] + fn owner_cancellation_invalidates_detached_completion_without_closing_a_newer_retry() { + let gate = HostedAdmission::default(); + let stale = gate.begin().unwrap(); + let cancellation = stale.cancellation_guard(); + drop(cancellation); + let current = gate.begin().unwrap(); + assert!(stale.complete().is_err()); + current.complete().unwrap(); + assert!(gate.is_open()); + + gate.close(); + let stale = gate.begin().unwrap(); + let cancellation = stale.cancellation_guard(); + gate.close(); + gate.begin().unwrap().complete().unwrap(); + drop(cancellation); + assert!(gate.is_open()); + assert!(stale.complete().is_err()); + } + + #[test] + fn only_an_acknowledged_completion_survives_owner_drop() { + let gate = HostedAdmission::default(); + let ticket = gate.begin().unwrap(); + let cancellation = ticket.cancellation_guard(); + ticket.complete().unwrap(); + drop(cancellation); + assert!(!gate.is_open()); + let ticket = gate.begin().unwrap(); + let cancellation = ticket.cancellation_guard(); + ticket.complete().unwrap(); + cancellation.disarm(); + assert!(gate.is_open()); + } + + #[test] + fn changed_fences_invalidate_scans_and_close_a_previously_open_gate() { + let gate = HostedAdmission::default(); + let ticket = gate.begin().unwrap(); + assert_eq!(gate.fences_changed().unwrap(), (0, 1)); + assert!(ticket.complete().is_err()); + let ticket = gate.begin().unwrap(); + assert_eq!(gate.fences_changed().unwrap(), (1, 2)); + ticket.complete_with_fence_revision(2).unwrap(); + assert!(gate.is_open()); + gate.fences_changed().unwrap(); + assert!(!gate.is_open()); + let ticket = gate.begin().unwrap(); + gate.0.state.lock().fence_revision = u64::MAX - 1; + assert!(gate.fences_changed().is_err()); + assert!(ticket.complete_with_fence_revision(u64::MAX).is_err()); + assert!(gate.begin().is_err()); + } +} diff --git a/crates/core/src/db/mod.rs b/crates/core/src/db/mod.rs index 13b6b10f7f0..d2f06a453b5 100644 --- a/crates/core/src/db/mod.rs +++ b/crates/core/src/db/mod.rs @@ -13,6 +13,7 @@ pub mod container_environment; pub mod deployment; mod durability; pub mod environment; +pub mod hosted_admission; pub mod persistence; pub mod relational_db; pub mod snapshot; diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index e5e2ef57373..26bb800ff9d 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -95,6 +95,7 @@ pub type ConnectedClients = HashSet<(Identity, ConnectionId)>; pub struct RelationalDB { database_identity: Identity, owner_identity: Identity, + hosted_admission: super::hosted_admission::HostedAdmission, inner: Locking, durability: Option>, @@ -160,6 +161,7 @@ impl RelationalDB { database_identity, owner_identity, + hosted_admission: Default::default(), row_count_fn: default_row_count_fn(database_identity), disk_size_fn, @@ -169,6 +171,12 @@ impl RelationalDB { } } + /// Transient container admission for this exact database open. The trusted + /// receiving host opens it only after reconciling current incoming fences. + pub fn hosted_admission(&self) -> &super::hosted_admission::HostedAdmission { + &self.hosted_admission + } + /// Open a database, which may or may not already exist. /// /// # Initialization @@ -339,8 +347,8 @@ impl RelationalDB { /// Shut down the database, without dropping it. /// - /// If the database is in-memory only, this does nothing. - /// Otherwise, it instructs the durability layer to shut down + /// Permanently closes hosted admission on this database object. + /// For a disk database, it also instructs the durability layer to shut down /// and waits until all outstanding transactions are reported as durable. /// /// After calling this method, calling [Self::commit_tx_downgrade] or @@ -351,6 +359,9 @@ impl RelationalDB { /// /// Returns the durable [TxOffset] in a `Some` otherwise. pub async fn shutdown(&self) -> Option { + // Idle module handles may retain this database after its writer stops. + // They must not retain admission or complete an earlier startup sweep. + self.hosted_admission.seal(); if let Some(durability) = &self.durability { return durability.close().await; } diff --git a/crates/core/src/host/container_fence.rs b/crates/core/src/host/container_fence.rs new file mode 100644 index 00000000000..4b872fbcea1 --- /dev/null +++ b/crates/core/src/host/container_fence.rs @@ -0,0 +1,204 @@ +//! Bounded host-only inspection and durable denial of retained source fences. +//! +//! The authenticated platform coordinator owns control membership checks and +//! actor drainage. These APIs do not authorize an external client or expose the +//! protected fence table through SQL, subscriptions, or module syscalls. + +use crate::db::deployment::{deny_container_fence, DeploymentError, FenceDenial}; +use crate::db::relational_db::RelationalDB; +use crate::host::container_environment::Durable; +use spacetimedb_datastore::execution_context::Workload; +use spacetimedb_datastore::locking_tx_datastore::state_view::ScanOrIndex; +use spacetimedb_datastore::system_tables::{StContainerFenceRow, ST_CONTAINER_FENCE_ID}; +use spacetimedb_datastore::traits::IsolationLevel; +use spacetimedb_lib::Identity; +use spacetimedb_primitives::ColId; +use spacetimedb_sats::AlgebraicValue; +use std::ops::Bound; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Semaphore; + +pub const FENCE_PAGE_SIZE: usize = 64; +static OPERATIONS: LazyLock> = LazyLock::new(|| Arc::new(Semaphore::new(8))); + +#[derive(Debug, thiserror::Error)] +pub enum FenceOperationError { + #[error("receiving host fence operation capacity exhausted")] + Capacity, + #[error("receiving host fence storage is unavailable")] + Storage, + #[error("receiving host fence durability is unavailable")] + DurabilityUnavailable, + #[error("receiving host fence durability failed")] + DurabilityFailed, + #[error("receiving host fence index is unavailable")] + IndexUnavailable, + #[error("receiving host fence inventory changed")] + RevisionChanged, + #[error(transparent)] + Deployment(#[from] DeploymentError), +} + +#[derive(Debug)] +pub struct FencePage { + /// Includes denied rows so every bounded physical page advances its cursor. + pub rows: Vec, + /// Last source in this page, or the input cursor if the page is empty. + pub next_source: Option, + pub complete: bool, + /// Read under the same database transaction as the rows. Restart the scan + /// if this differs from the previous page or a known local denial result. + pub fence_revision: u64, +} + +/// Read at most 64 rows using the protected source Identity B-tree. We refuse +/// the datastore's scan fallback: row order and physical work must be bounded. +pub async fn page(db: Arc, after: Option) -> Result { + let permit = OPERATIONS + .clone() + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + tokio::task::spawn_blocking(move || { + let _permit = permit; + let tx = db.begin_tx(Workload::Internal); + let result = (|| { + let lower = after.map_or(Bound::Unbounded, |identity| { + Bound::Excluded(AlgebraicValue::U256(identity.to_u256().into())) + }); + let range = db + .iter_by_col_range(&tx, ST_CONTAINER_FENCE_ID, ColId(0), (lower, Bound::Unbounded)) + .map_err(|_| FenceOperationError::Storage)?; + let ScanOrIndex::Index(mut range) = range else { + return Err(FenceOperationError::IndexUnavailable); + }; + let rows = range + .by_ref() + .take(FENCE_PAGE_SIZE) + .map(|row| StContainerFenceRow::try_from(row).map_err(|_| FenceOperationError::Storage)) + .collect::, _>>()?; + let complete = range.next().is_none(); + let next_source = rows.last().map(|row| row.source_identity.into()).or(after); + Ok(FencePage { + rows, + next_source, + complete, + fence_revision: db.hosted_admission().fence_revision(), + }) + })(); + let (_, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + result + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +/// The coordinator must have excluded this exact source from current control +/// authority. For a check that must serialize with an inventory lock, use the +/// synchronous `deployment::deny_container_fence` inside the caller's own +/// serializable transaction instead. Neither API performs that control check. +pub async fn deny_orphan( + db: Arc, + expected: StContainerFenceRow, +) -> Result, FenceOperationError> { + deny_with_capacity(db, expected, OPERATIONS.clone()).await +} + +/// Confirm durability of all fences visible at one exact physical revision. +/// In particular, a retry that sees a previous owner's committed denial must +/// still wait for that denial's storage acknowledgment. The coordinator must +/// recheck the revision under its final database transaction before admission; +/// this receipt does not freeze future mutations or confer control authority. +pub async fn confirm_revision( + db: Arc, + expected_physical: u64, +) -> Result, FenceOperationError> { + confirm_with_capacity(db, expected_physical, OPERATIONS.clone()).await +} + +async fn confirm_with_capacity( + db: Arc, + expected_physical: u64, + capacity: Arc, +) -> Result, FenceOperationError> { + let mut durability = db + .durable_tx_offset() + .ok_or(FenceOperationError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let durable_through = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_tx(Workload::Internal); + let physical = action_db.hosted_admission().fence_revision(); + let (offset, metrics, reducer) = action_db.release_tx(tx); + action_db.report_read_tx_metrics(reducer, metrics); + if physical != expected_physical || physical == u64::MAX { + return Err(FenceOperationError::RevisionChanged); + } + Ok(offset) + }) + .await + .map_err(|_| FenceOperationError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| FenceOperationError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt: (), + durable_through, + }) + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +async fn deny_with_capacity( + db: Arc, + expected: StContainerFenceRow, + capacity: Arc, +) -> Result, FenceOperationError> { + let mut durability = db + .durable_tx_offset() + .ok_or(FenceOperationError::DurabilityUnavailable)?; + let permit = capacity + .try_acquire_owned() + .map_err(|_| FenceOperationError::Capacity)?; + // A caller's cancellation drops only this JoinHandle. The owned task keeps + // both the permit and database alive through physical commit and durability. + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, receipt) = + action_db.with_auto_rollback(tx, |tx| deny_container_fence(&action_db, tx, &expected))?; + let (offset, data, metrics, reducer) = action_db + .commit_tx(tx) + .map_err(|_| FenceOperationError::Storage)? + .ok_or(FenceOperationError::Storage)?; + action_db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, FenceOperationError>((offset, receipt)) + }) + .await + .map_err(|_| FenceOperationError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| FenceOperationError::DurabilityFailed)?; + // Do not drop the physical database before the owned durability wait. + drop(db); + Ok(Durable { + receipt, + durable_through, + }) + }) + .await + .map_err(|_| FenceOperationError::Storage)? +} + +#[cfg(test)] +mod tests; diff --git a/crates/core/src/host/container_fence/tests.rs b/crates/core/src/host/container_fence/tests.rs new file mode 100644 index 00000000000..fdd641fef88 --- /dev/null +++ b/crates/core/src/host/container_fence/tests.rs @@ -0,0 +1,389 @@ +use super::*; +use crate::db::deployment::{check_container_fence, install_container_fence}; +use crate::db::relational_db::tests_utils::TestDB; +use spacetimedb_lib::hash_bytes; +use std::time::Duration; + +fn fence(source: u64) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: Identity::from_u256(source.into()).into(), + generation: 7, + target_grant_revision: 3, + target_set_hash: hash_bytes(b"retained target set"), + allowed: true, + } +} + +fn install(db: &RelationalDB, row: &StContainerFenceRow) { + db.with_auto_commit(Workload::ForTests, |tx| install_container_fence(db, tx, row)) + .unwrap(); +} + +#[tokio::test] +async fn container_fence_pages_follow_identity_index_and_include_denied_rows() { + let db = TestDB::in_memory().unwrap(); + // Reverse insertion order deliberately differs from source index order. + for source in (0..130).rev() { + install( + &db, + &StContainerFenceRow { + allowed: source % 3 == 0, + ..fence(source) + }, + ); + } + let revision = db.hosted_admission().fence_revision(); + let mut after = None; + let mut identities = Vec::new(); + for expected_size in [64, 64, 2] { + let page = page(db.db.clone(), after).await.unwrap(); + assert_eq!(page.rows.len(), expected_size); + assert_eq!(page.complete, expected_size == 2); + assert_eq!(page.fence_revision, revision); + identities.extend(page.rows.iter().map(|row| Identity::from(row.source_identity))); + after = page.next_source; + assert_eq!(after, identities.last().copied()); + } + assert_eq!( + identities, + (0..130u64).map(|n| Identity::from_u256(n.into())).collect::>() + ); + let empty = page(db.db.clone(), after).await.unwrap(); + assert!(empty.rows.is_empty() && empty.complete); + assert_eq!(empty.next_source, after); +} + +#[tokio::test] +async fn container_fence_missing_index_refuses_unbounded_scan_and_memory_refuses_denial() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(17); + install(&db, &expected); + assert!(matches!( + deny_orphan(db.db.clone(), expected).await, + Err(FenceOperationError::DurabilityUnavailable) + )); + db.with_auto_commit(Workload::ForTests, |tx| { + db.drop_index(tx, spacetimedb_primitives::IndexId(34)) + }) + .unwrap(); + assert!(matches!( + page(db.db.clone(), None).await, + Err(FenceOperationError::IndexUnavailable) + )); +} + +#[test] +fn container_fence_orphan_denial_is_durable_exact_and_cannot_reopen_generation() { + let db = TestDB::durable_without_snapshot_repo().unwrap(); + let expected = fence(19); + install(&db, &expected); + db.hosted_admission().begin().unwrap().complete().unwrap(); + let before = db.hosted_admission().fence_revision(); + let first = db + .runtime() + .unwrap() + .block_on(deny_orphan(db.db.clone(), expected.clone())) + .unwrap(); + assert_eq!(first.receipt.revision_before, before); + assert_eq!(first.receipt.revision_after, before + 1); + assert!(!db.hosted_admission().is_open()); + assert_eq!( + first.receipt.row, + StContainerFenceRow { + allowed: false, + ..expected.clone() + } + ); + assert!(db.durable_tx_offset().unwrap().get().unwrap().unwrap() >= first.durable_through); + let retry = db + .runtime() + .unwrap() + .block_on(deny_orphan(db.db.clone(), expected.clone())) + .unwrap(); + assert_eq!(retry.receipt.revision_before, before + 1); + assert_eq!(retry.receipt.revision_after, before + 1); + let db = db.reopen().unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + assert!(matches!( + check_container_fence( + tx, + expected.source_identity.into(), + expected.generation, + expected.target_grant_revision + ), + Err(DeploymentError::ContainerFenced) + )); + assert!(matches!( + install_container_fence(&db, tx, &expected), + Err(DeploymentError::FenceConflict) + )); + let retry = deny_container_fence(&db, tx, &expected)?; + assert_eq!(retry.row, first.receipt.row); + assert_eq!(retry.revision_before, retry.revision_after); + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + install( + &db, + &StContainerFenceRow { + generation: expected.generation + 1, + ..expected + }, + ); +} + +#[test] +fn container_fence_orphan_cas_rejects_missing_changed_and_invalid_observations() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(20); + db.with_auto_commit(Workload::ForTests, |tx| { + assert!(matches!( + deny_container_fence(&db, tx, &expected), + Err(DeploymentError::FenceConflict) + )); + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + install(&db, &expected); + let next = StContainerFenceRow { + generation: expected.generation + 1, + ..expected.clone() + }; + install(&db, &next); + let revision = db.hosted_admission().fence_revision(); + db.with_auto_commit(Workload::ForTests, |tx| { + for stale in [ + expected, + StContainerFenceRow { + allowed: false, + ..next.clone() + }, + StContainerFenceRow { + target_set_hash: hash_bytes(b"different"), + ..next.clone() + }, + StContainerFenceRow { + target_grant_revision: 4, + ..next.clone() + }, + ] { + assert!(matches!( + deny_container_fence(&db, tx, &stale), + Err(DeploymentError::FenceConflict) + )); + } + check_container_fence( + tx, + next.source_identity.into(), + next.generation, + next.target_grant_revision, + )?; + Ok::<_, DeploymentError>(()) + }) + .unwrap(); + assert_eq!(db.hosted_admission().fence_revision(), revision); +} + +#[tokio::test] +async fn container_fence_physical_change_behind_cursor_invalidates_completion() { + let db = TestDB::in_memory().unwrap(); + install(&db, &fence(100)); + let ticket = db.hosted_admission().begin().unwrap(); + let first = page(db.db.clone(), None).await.unwrap(); + assert!(first.complete); + install(&db, &fence(1)); + assert!(ticket.complete_with_fence_revision(first.fence_revision).is_err()); + assert!(!db.hosted_admission().is_open()); + let restarted = page(db.db.clone(), None).await.unwrap(); + assert_eq!(restarted.rows.len(), 2); + assert_ne!(first.fence_revision, restarted.fence_revision); +} + +#[test] +fn container_fence_orphan_rollback_keeps_row_and_invalidates_physical_scan() { + let db = TestDB::in_memory().unwrap(); + let expected = fence(21); + install(&db, &expected); + let before = db.hosted_admission().fence_revision(); + let ticket = db.hosted_admission().begin().unwrap(); + let result = db.with_auto_commit(Workload::ForTests, |tx| { + deny_container_fence(&db, tx, &expected)?; + Err::<(), _>(DeploymentError::CorruptMetadata) + }); + assert!(result.is_err()); + assert!(ticket.complete_with_fence_revision(before).is_err()); + db.with_auto_commit(Workload::ForTests, |tx| { + check_container_fence( + tx, + expected.source_identity.into(), + expected.generation, + expected.target_grant_revision, + ) + }) + .unwrap(); +} + +/// Real local storage with its durable acknowledgment deliberately withheld. +/// This models a lagging quorum without replacing the transaction or writer. +struct DelayedAcknowledgment { + writer: Arc>, + acknowledged: spacetimedb_durability::DurableOffset, +} + +impl spacetimedb_durability::Durability for DelayedAcknowledgment { + type TxData = crate::db::relational_db::Txdata; + fn append_tx(&self, tx: spacetimedb_durability::PreparedTx) { + self.writer.append_tx(tx); + } + fn durable_tx_offset(&self) -> spacetimedb_durability::DurableOffset { + self.acknowledged.clone() + } + fn close(&self) -> spacetimedb_durability::Close { + self.writer.close() + } +} + +async fn delayed_storage() -> ( + crate::db::relational_db::tests_utils::TempReplicaDir, + Arc, + crate::db::relational_db::LocalDurability, + tokio::sync::watch::Sender>, +) { + use crate::db::persistence::Persistence; + use crate::db::relational_db::{local_durability, tests_utils::TempReplicaDir}; + let directory = TempReplicaDir::new().unwrap(); + let (writer, disk_size) = local_durability((*directory).clone(), None).await.unwrap(); + let (acknowledge, acknowledged) = tokio::sync::watch::channel(None); + let db = Arc::new( + TestDB::open_db( + writer.as_history(), + Some(Persistence { + durability: Arc::new(DelayedAcknowledgment { + writer: writer.clone(), + acknowledged: acknowledged.into(), + }), + disk_size, + snapshots: None, + runtime: tokio::runtime::Handle::current(), + }), + None, + 0, + ) + .unwrap(), + ); + (directory, db, writer, acknowledge) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_fence_cancelled_denial_retains_capacity_through_actual_durability_wait() { + use spacetimedb_durability::Durability; + let (_directory, db, writer, acknowledge) = delayed_storage().await; + let expected = fence(22); + install(&db, &expected); + let capacity = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(deny_with_capacity(db.clone(), expected.clone(), capacity.clone())); + let row = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let page = page(db.clone(), None).await.unwrap(); + if let Some(row) = page.rows.first().filter(|row| !row.allowed) { + break row.clone(); + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!row.allowed); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + deny_with_capacity(db.clone(), expected.clone(), capacity.clone()).await, + Err(FenceOperationError::Capacity) + )); + let tx = db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + let mut actual = writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + acknowledge.send_replace(Some(durable)); + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + // The detached owner finished only once its physical acknowledgment arrived. + db.shutdown().await; + drop(db); + writer.close().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_fence_retry_barrier_waits_for_prior_commit_and_retains_cancelled_capacity() { + use spacetimedb_durability::Durability; + let (_directory, db, writer, acknowledge) = delayed_storage().await; + let expected = fence(23); + install(&db, &expected); + // A previous coordinator has committed but never confirmed durability. + let denied = db + .with_auto_commit(Workload::ForTests, |tx| deny_container_fence(&db, tx, &expected)) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + // Changed inventory fails immediately, even while acknowledgments lag. + assert!(matches!( + tokio::time::timeout( + Duration::from_secs(5), + confirm_with_capacity(db.clone(), denied.revision_before, capacity.clone()) + ) + .await + .unwrap(), + Err(FenceOperationError::RevisionChanged) + )); + let caller = tokio::spawn(confirm_with_capacity( + db.clone(), + denied.revision_after, + capacity.clone(), + )); + tokio::time::timeout(Duration::from_secs(5), async { + while capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(!caller.is_finished()); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert!(matches!( + confirm_with_capacity(db.clone(), denied.revision_after, capacity.clone()).await, + Err(FenceOperationError::Capacity) + )); + let tx = db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = db.release_tx(tx); + db.report_read_tx_metrics(reducer, metrics); + let mut actual = writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + acknowledge.send_replace(Some(durable)); + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); + let proof = confirm_revision(db.clone(), denied.revision_after).await.unwrap(); + assert_eq!(proof.durable_through, offset); + // The proof is only a barrier, so finalization still checks current revision. + install(&db, &fence(24)); + assert!(matches!( + confirm_revision(db.clone(), denied.revision_after).await, + Err(FenceOperationError::RevisionChanged) + )); + db.shutdown().await; + drop(db); + writer.close().await; +} diff --git a/crates/core/src/host/empty_module.rs b/crates/core/src/host/empty_module.rs index fff546cf83c..c0310ac8628 100644 --- a/crates/core/src/host/empty_module.rs +++ b/crates/core/src/host/empty_module.rs @@ -16,7 +16,7 @@ use std::sync::OnceLock; pub const VERSION_1: u32 = 1; pub use spacetimedb_lib::deployment::SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH as VERSION_1_PROGRAM_HASH; -pub const VERSION_1_BYTES: &[u8] = include_bytes!("empty_module/v1.wasm"); +pub use spacetimedb_lib::deployment::SYSTEM_EMPTY_MODULE_V1_BYTES as VERSION_1_BYTES; /// Return the exact bundled program for a recognized system module version. /// Unknown versions fail closed instead of silently selecting the latest one. diff --git a/crates/core/src/host/empty_module/tests.rs b/crates/core/src/host/empty_module/tests.rs index 5d3220d585e..dca7c58fc52 100644 --- a/crates/core/src/host/empty_module/tests.rs +++ b/crates/core/src/host/empty_module/tests.rs @@ -19,6 +19,15 @@ fn bundled_bytes_match_current_v10_schema_wire_format() { let program = program(VERSION_1).unwrap(); assert_eq!(program.hash, hash_bytes(include_bytes!("v1.wasm"))); assert_eq!(program.hash, VERSION_1_PROGRAM_HASH); + let artifact = spacetimedb_lib::deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT; + assert_eq!(artifact.size_bytes, program.bytes.len() as u64); + assert_eq!( + artifact.digest.to_string(), + format!( + "sha256:{}", + include_str!("v1.sha256").split_whitespace().next().unwrap() + ) + ); assert!(matches_program(VERSION_1, &program)); } diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index ddda547492d..b4e8486317d 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -25,12 +25,14 @@ use crate::worker_metrics::WORKER_METRICS; use anyhow::{anyhow, bail, Context}; use async_trait::async_trait; use durability::{Durability, EmptyHistory}; +use futures::FutureExt as _; use log::{info, trace, warn}; +#[cfg(test)] use parking_lot::Mutex; use scopeguard::defer; use spacetimedb_commitlog::SizeOnDisk; use spacetimedb_data_structures::error_stream::ErrorStream; -use spacetimedb_data_structures::map::{IntMap, IntSet}; +use spacetimedb_data_structures::map::IntSet; use spacetimedb_datastore::db_metrics::data_size::DATA_SIZE_METRICS; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::execution_context::Workload; @@ -47,10 +49,9 @@ use std::future::Future; use std::ops::Deref; use std::sync::Arc; use std::time::Duration; -use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, RwLock as AsyncRwLock}; +use tokio::sync::{watch, RwLock as AsyncRwLock}; use tokio::task::AbortHandle; -use tokio::time::error::Elapsed; -use tokio::time::{interval_at, timeout, Instant}; +use tokio::time::{timeout, Instant}; // TODO: // @@ -67,8 +68,12 @@ const IN_MEMORY_DATABASE_LOGGER_MAX_SIZE: u64 = 0x1_000_000; /// A shared mutable cell containing a module host and associated database. type HostCell = Arc>>; -/// The registry of all running hosts. -type Hosts = Arc>>; +mod registry; +mod retained; +use registry::{Hosts, Registration}; + +#[cfg(test)] +mod retained_tests; #[cfg(test)] mod deployment_tests; @@ -118,6 +123,8 @@ pub struct HostController { /// Map of all hosts managed by this controller, /// keyed by replica id. hosts: Hosts, + /// Held by physical cold operations through their joined writer shutdown. + retained_capacity: Arc, /// The root directory for database data. pub data_dir: Arc, /// The default configuration to use for databases created by this @@ -251,6 +258,7 @@ impl HostController { ) -> Self { Self { hosts: <_>::default(), + retained_capacity: retained::capacity(), default_config, program_storage, energy_monitor, @@ -352,10 +360,10 @@ impl HostController { // Note that `tokio::spawn` only cancels its tasks when the runtime shuts down, // at which point we won't be calling `try_init_host` again anyways. let rx = tokio::spawn(async move { - let host = this.try_init_host(database, replica_id).await?; + let host = this.try_init_host(database, replica_id, &guard).await?; let rx = host.module.subscribe(); - *guard = Some(host); + guard.install(host); Ok::<_, anyhow::Error>(rx) }) @@ -467,7 +475,7 @@ impl HostController { let mut host = match guard.take() { None => { trace!("host not running, try_init"); - this.try_init_host(database, replica_id).await? + this.try_init_host(database, replica_id, &guard).await? } Some(host) => { trace!("host found, updating"); @@ -475,30 +483,46 @@ impl HostController { } }; let mut database_committed = false; - let update_result = host - .update_module( - this.runtimes.clone(), - program, - policy, - deployment, - this.energy_monitor.clone(), - this.unregister_fn(replica_id), - this.db_cores.take(), - &mut database_committed, - ) - .await; + let registration = guard.registration(); + let update_result = std::panic::AssertUnwindSafe(host.update_module( + this.runtimes.clone(), + program, + policy, + deployment, + this.energy_monitor.clone(), + this.unregister_fn(registration.clone()), + registration, + this.db_cores.take(), + &mut database_committed, + )) + .catch_unwind() + .await; + let update_result = match update_result { + Ok(result) => result, + Err(panic) => { + if let Err(error) = retained::close_host(host).await { + if matches!(error, retained::CloseFailure::WriterUnconfirmed) { + guard.quarantine(None); + } + return Err(error.into()); + } + std::panic::resume_unwind(panic); + } + }; if update_result.is_err() && database_committed { // Schema/program/receipt already committed. The previous // executable cannot be retained after activation failure. // Close clients/scheduler and reconstruct from stored program // on the next leader lookup or reconciliation attempt. - let module = host.module.borrow().clone(); - module.exit().await; - host.replica_ctx.relational_db().shutdown().await; - drop(host); + if let Err(error) = retained::close_host(host).await { + if matches!(error, retained::CloseFailure::WriterUnconfirmed) { + guard.quarantine(None); + } + return Err(error.into()); + } } else { - *guard = Some(host); + guard.install(host); } update_result @@ -546,58 +570,22 @@ impl HostController { /// Release all resources of the [`ModuleHost`] identified by `replica_id`, /// and deregister it from the controller. + /// + /// A timeout returns an error while the owned close continues. Only success + /// confirms the configured storage writer has completed shutdown. #[tracing::instrument(level = "trace", skip_all)] pub async fn exit_module_host(&self, replica_id: u64, timeout: Duration) -> Result<(), anyhow::Error> { - let Some(lock) = self.hosts.lock().remove(&replica_id) else { + let Some(request) = registry::close(&self.hosts, replica_id) else { return Ok(()); }; - // To debug the potential deadlock issue reported in - // https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - // we'll log a warning every 5s if we can't acquire an exclusive lock. - let start = Instant::now(); - let mut t = interval_at(start + Duration::from_secs(5), Duration::from_secs(5)); - let warn_blocked = tokio::spawn(async move { - loop { - t.tick().await; - warn!( - "blocked waiting to exit module for replica {} since {}s", - replica_id, - start.elapsed().as_secs_f32() - ); - } - }); - defer!(warn_blocked.abort()); - - let shutdown = tokio::time::timeout(timeout, async { - let mut guard = lock.write_owned().await; - let Some(host) = guard.take() else { - return; - }; - let module = host.module.borrow().clone(); - let info = module.info(); - - let database_identity = info.database_identity; - let table_names = info.module_def.tables().map(|t| t.name.deref()); - - // Ensure we clear the metrics even if the future is cancelled. - defer!(remove_database_gauges(&database_identity, table_names)); - - info!("replica={replica_id} database={database_identity} exiting module"); - module.exit().await; - info!("replica={replica_id} database={database_identity} exiting database"); - module.relational_db().shutdown().await; - info!("replica={replica_id} database={database_identity} module host exited"); - }) - .await; - - if shutdown.is_err() { - warn!( - "replica={replica_id} shutdown timed out after {}s", - start.elapsed().as_secs_f32() - ); + if let Some(owner) = request.owner { + // The close owns its registry pin independently of this waiter. + // Cancellation or timeout cannot release a still-running writer. + tokio::spawn(owner.run()); } - - Ok(()) + tokio::time::timeout(timeout, request.completion.wait()) + .await + .map_err(|_| anyhow!("replica {replica_id} shutdown is still pending after {timeout:?}"))? } /// Get the [`ModuleHost`] identified by `replica_id` or return an error @@ -654,41 +642,40 @@ impl HostController { self.hosts.lock().keys().copied().collect() } - /// On-panic callback passed to [`ModuleHost`]s created by this controller. - /// - /// Removes the module with the given `replica_id` from this controller. - fn unregister_fn(&self, replica_id: u64) -> impl Fn() + Send + Sync + 'static + use<> { - let hosts = Arc::downgrade(&self.hosts); + /// On-panic callbacks are scoped to one installed executable and cell. + /// A stale callback cannot unregister an updated or reopened database. + fn unregister_fn(&self, registration: Registration) -> impl Fn() + Send + Sync + 'static + use<> { + let runtime = tokio::runtime::Handle::current(); move || { - if let Some(hosts) = hosts.upgrade() { - hosts.lock().remove(&replica_id); + if let Some(request) = registration.close_if_current() + && let Some(owner) = request.owner + { + runtime.spawn(owner.run()); } } } - /// Acquire a write lock on the [HostCell] for `replica_id`. - /// - /// This will time out after 5s to aid debugging of - /// https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - async fn acquire_write_lock(&self, replica_id: u64) -> Result>, Elapsed> { - let lock = self.hosts.lock().entry(replica_id).or_default().clone(); - timeout(Duration::from_secs(5), lock.write_owned()).await + /// The total wait includes any ongoing close and cell reacquisition. + async fn acquire_write_lock(&self, replica_id: u64) -> anyhow::Result { + timeout(Duration::from_secs(5), registry::Pin::write(&self.hosts, replica_id)).await? } - /// Acquire a read lock on the [HostCell] for `replica_id`. - /// - /// This will time out after 5s to aid debugging of - /// https://github.com/clockworklabs/SpacetimeDBPrivate/issues/2337 - async fn acquire_read_lock(&self, replica_id: u64) -> Result>, Elapsed> { - let lock = self.hosts.lock().entry(replica_id).or_default().clone(); - timeout(Duration::from_secs(5), lock.read_owned()).await + async fn acquire_read_lock(&self, replica_id: u64) -> anyhow::Result { + timeout(Duration::from_secs(5), registry::Pin::read(&self.hosts, replica_id)).await? } - async fn try_init_host(&self, database: Database, replica_id: u64) -> anyhow::Result { + async fn try_init_host( + &self, + database: Database, + replica_id: u64, + guard: ®istry::WriteGuard, + ) -> anyhow::Result { let database_identity = database.database_identity; - Host::try_init(self, database, replica_id) - .await - .with_context(|| format!("failed to init replica {} for {}", replica_id, database_identity)) + let result = Host::try_init(self, database, replica_id, guard.registration()).await; + if result.as_ref().is_err_and(retained::writer_unconfirmed) { + guard.quarantine(None); + } + result.with_context(|| format!("failed to init replica {} for {}", replica_id, database_identity)) } } @@ -887,6 +874,7 @@ async fn update_module( /// Encapsulates a database, associated module, and auxiliary state. struct Host { + registration: Registration, /// The [`ModuleHost`], providing the callable reducer API. /// /// Modules may be updated via [`Host::update_module`]. @@ -917,240 +905,239 @@ struct Host { impl Host { /// Attempt to instantiate a [`Host`] from persistent storage. /// - /// Note that this does **not** run module initialization routines, but may - /// create on-disk artifacts if the host / database did not exist. + /// This executes the stored module and initializes a new database when + /// necessary. It may create on-disk artifacts. Retained cleanup uses the + /// separate metadata-only open path instead. #[tracing::instrument(level = "debug", skip_all)] - async fn try_init(host_controller: &HostController, database: Database, replica_id: u64) -> anyhow::Result { + async fn try_init( + host_controller: &HostController, + database: Database, + replica_id: u64, + registration: Registration, + ) -> anyhow::Result { let HostController { data_dir, default_config: config, program_storage, energy_monitor, runtimes, - persistence, - page_pool, bsatn_rlb_pool, .. } = host_controller; let replica_dir = data_dir.replica(replica_id); let (tx_metrics_queue, tx_metrics_recorder_task) = spawn_tx_metrics_recorder(); - let (db, connected_clients) = match config.storage { - db::Storage::Memory => RelationalDB::open( - database.database_identity, - database.owner_identity, - EmptyHistory::new(), - None, - Some(tx_metrics_queue), - page_pool.clone(), - )?, - db::Storage::Disk => { - // Replay from the local state. - let history = relational_db::local_history(&replica_dir).await?; - let persistence = persistence.persistence(&database, replica_id).await?; - // Loading a database from persistent storage involves heavy - // blocking I/O. `asyncify` to avoid blocking the async worker. - let (db, clients) = asyncify({ - let database_identity = database.database_identity; - let owner_identity = database.owner_identity; - let page_pool = page_pool.clone(); - move || { - RelationalDB::open( - database_identity, - owner_identity, - history, - Some(persistence), - Some(tx_metrics_queue), - page_pool, - ) - } - }) - .await - // Make sure we log the source chain of the error - // as a single line, with the help of `anyhow`. - .map_err(anyhow::Error::from) - .inspect_err(|e| { - tracing::error!( - database = %database.database_identity, - replica = replica_id, - "Failed to open database: {e:#}" + let metrics_cleanup = scopeguard::guard(tx_metrics_recorder_task.clone(), |task| task.abort()); + let (db, connected_clients, joined) = + retained::open_database(host_controller, &database, replica_id, false, Some(tx_metrics_queue)).await?; + let initialized = std::panic::AssertUnwindSafe(async { + let (mut program, program_needs_init, initial_deployment) = match db.program()? { + // Launch module with program from existing database. + Some(program) => { + info!( + "loaded program {} from the database host-type={}", + program.hash, + HostType::from(program.kind) ); - })?; - - (db, clients) - } - }; - let (mut program, program_needs_init, initial_deployment) = match db.program()? { - // Launch module with program from existing database. - Some(program) => { - info!( - "loaded program {} from the database host-type={}", - program.hash, - HostType::from(program.kind) - ); - (program, false, None) - } - // Database is empty, load program from external storage and run - // initialization. - None => { - info!( - "loading program {} from external storage host-type={}", - database.initial_program, database.host_type - ); - let initial_deployment = program_storage.initial_deployment(&database).await?; - let program_bytes = load_program(program_storage, database.initial_program).await?; - let program = Program { - hash: database.initial_program, - bytes: program_bytes, - kind: database.host_type.into(), - }; - (program, true, initial_deployment) - } - }; - - let relational_db = Arc::new(db); - let (program, launched) = match HostType::from(program.kind) { - HostType::Js => { - ModuleLauncher { - database, - replica_id, - program, - on_panic: host_controller.unregister_fn(replica_id), - relational_db, - energy_monitor: energy_monitor.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), + (program, false, None) } - .launch_module() - .await? - } - HostType::Wasm => { - // Prior to https://github.com/clockworklabs/SpacetimeDB/pull/4549 - // the host type in `st_module` was always set to wasm. - // We now correctly use the host type from the database, but the - // module may in fact be a JS module. - // Retry JS only for an existing stored module whose database - // declaration explicitly identifies it as JS. A new publication - // or declared Wasm module must preserve its Wasm validation error. - // If the legacy retry succeeds, repair `st_module`. - // - // TODO: This code should eventually be removed once all - // databases have been repaired. - let launch_wasm_result = ModuleLauncher { - database: database.clone(), - replica_id, - program: program.clone(), - on_panic: host_controller.unregister_fn(replica_id), - relational_db: relational_db.clone(), - energy_monitor: energy_monitor.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.clone().module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), + // Database is empty, load program from external storage and run + // initialization. + None => { + info!( + "loading program {} from external storage host-type={}", + database.initial_program, database.host_type + ); + let initial_deployment = program_storage.initial_deployment(&database).await?; + let program_bytes = load_program(program_storage, database.initial_program).await?; + let program = Program { + hash: database.initial_program, + bytes: program_bytes, + kind: database.host_type.into(), + }; + (program, true, initial_deployment) } - .launch_module() - .await; - match launch_wasm_result { - Ok(program_and_module_host) => program_and_module_host, - Err(e) => { - if program_needs_init || database.host_type != HostType::Js { - return Err(e); - } - warn!("failed to launch wasm module, trying js: {e:#}"); - - program.kind = ModuleKind::JS; - let res = ModuleLauncher { - database, - replica_id, - program: program.clone(), - on_panic: host_controller.unregister_fn(replica_id), - relational_db: relational_db.clone(), - energy_monitor: energy_monitor.clone(), - module_logs: match config.storage { - db::Storage::Memory => None, - db::Storage::Disk => Some(replica_dir.module_logs()), - }, - runtimes: runtimes.clone(), - core: host_controller.db_cores.take(), - bsatn_rlb_pool: bsatn_rlb_pool.clone(), - } - .launch_module() - .await; + }; - if res.is_ok() { - let _ = relational_db - .with_auto_commit(Workload::Internal, |tx| relational_db.update_program(tx, program)); + let relational_db = db.clone(); + let (program, launched) = match HostType::from(program.kind) { + HostType::Js => { + ModuleLauncher { + database, + replica_id, + program, + on_panic: host_controller.unregister_fn(registration.clone()), + relational_db, + energy_monitor: energy_monitor.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await? + } + HostType::Wasm => { + // Prior to https://github.com/clockworklabs/SpacetimeDB/pull/4549 + // the host type in `st_module` was always set to wasm. + // We now correctly use the host type from the database, but the + // module may in fact be a JS module. + // Retry JS only for an existing stored module whose database + // declaration explicitly identifies it as JS. A new publication + // or declared Wasm module must preserve its Wasm validation error. + // If the legacy retry succeeds, repair `st_module`. + // + // TODO: This code should eventually be removed once all + // databases have been repaired. + let launch_wasm_result = ModuleLauncher { + database: database.clone(), + replica_id, + program: program.clone(), + on_panic: host_controller.unregister_fn(registration.clone()), + relational_db: relational_db.clone(), + energy_monitor: energy_monitor.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.clone().module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await; + match launch_wasm_result { + Ok(program_and_module_host) => program_and_module_host, + Err(e) => { + if program_needs_init || database.host_type != HostType::Js { + return Err(e); + } + warn!("failed to launch wasm module, trying js: {e:#}"); + + program.kind = ModuleKind::JS; + let res = ModuleLauncher { + database, + replica_id, + program: program.clone(), + on_panic: host_controller.unregister_fn(registration.clone()), + relational_db: relational_db.clone(), + energy_monitor: energy_monitor.clone(), + module_logs: match config.storage { + db::Storage::Memory => None, + db::Storage::Disk => Some(replica_dir.module_logs()), + }, + runtimes: runtimes.clone(), + core: host_controller.db_cores.take(), + bsatn_rlb_pool: bsatn_rlb_pool.clone(), + } + .launch_module() + .await; + + if res.is_ok() { + let _ = relational_db.with_auto_commit(Workload::Internal, |tx| { + relational_db.update_program(tx, program) + }); + } + + res.map_err(|js_error| { + e.context(format!("legacy JS host-type repair also failed: {js_error:#}")) + })? } - - res.map_err(|js_error| { - e.context(format!("legacy JS host-type repair also failed: {js_error:#}")) - })? } } + }; + + if program_needs_init { + let call_result = launched + .module_host + .init_database_with_deployment(program, initial_deployment) + .await?; + if let Some(call_result) = call_result { + Result::from(call_result)?; + } + } else { + drop(program) } - }; - if program_needs_init { - let call_result = launched - .module_host - .init_database_with_deployment(program, initial_deployment) - .await?; - if let Some(call_result) = call_result { - Result::from(call_result)?; + let LaunchedModule { + replica_ctx, + module_host, + scheduler, + scheduler_starter, + } = launched; + + // Disconnect dangling clients. + // No need to clear view tables here since we do it in `clear_all_clients`. + for (identity, connection_id) in connected_clients { + module_host + .call_identity_disconnected(identity, connection_id) + .await + .with_context(|| { + format!( + "Error calling disconnect for {} {} on {}", + identity, connection_id, replica_ctx.database_identity + ) + })?; } - } else { - drop(program) - } + // We should have no clients left, but we do this just in case. + // This should only matter if we crashed with something in st_client_credentials, + // then restarted with an older version of the code that doesn't use st_client_credentials. + // That case would cause some permanently dangling st_client_credentials. + // Since we have no clients on startup, this should be safe to do regardless. + module_host.clear_all_clients().await?; + + // Scheduled work can run before this Host is installed in its cell. + // Its callback must already identify this executable; the close + // owner will wait for our pinned guard before taking the Host. + registration.activate(); + scheduler_starter.start(&module_host)?; + #[cfg(test)] + if retained::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + .lock() + .remove(&replica_ctx.database_identity) + { + host_controller.unregister_fn(registration.clone())(); + } + let disk_metrics_recorder_task = tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle(); + let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone()); - let LaunchedModule { - replica_ctx, - module_host, - scheduler, - scheduler_starter, - } = launched; - - // Disconnect dangling clients. - // No need to clear view tables here since we do it in `clear_all_clients`. - for (identity, connection_id) in connected_clients { - module_host - .call_identity_disconnected(identity, connection_id) - .await - .with_context(|| { - format!( - "Error calling disconnect for {} {} on {}", - identity, connection_id, replica_ctx.database_identity - ) - })?; - } - // We should have no clients left, but we do this just in case. - // This should only matter if we crashed with something in st_client_credentials, - // then restarted with an older version of the code that doesn't use st_client_credentials. - // That case would cause some permanently dangling st_client_credentials. - // Since we have no clients on startup, this should be safe to do regardless. - module_host.clear_all_clients().await?; - - scheduler_starter.start(&module_host)?; - let disk_metrics_recorder_task = tokio::spawn(metric_reporter(replica_ctx.clone())).abort_handle(); - let view_cleanup_task = spawn_view_cleanup_loop(replica_ctx.relational_db().clone()); - - let module = watch::Sender::new(module_host); - - Ok(Host { - module, - replica_ctx, - scheduler, - disk_metrics_recorder_task, - tx_metrics_recorder_task, - view_cleanup_task, + let module = watch::Sender::new(module_host); + + Ok(Host { + registration, + module, + replica_ctx, + scheduler, + disk_metrics_recorder_task, + tx_metrics_recorder_task, + view_cleanup_task, + }) }) + .catch_unwind() + .await; + match initialized { + Ok(Ok(host)) => { + let _ = scopeguard::ScopeGuard::into_inner(metrics_cleanup); + Ok(host) + } + Ok(Err(error)) => { + if let Some(joined) = joined { + joined.join().await?; + } + drop(db); + Err(error) + } + Err(panic) => { + if let Some(joined) = joined { + joined.join().await?; + } + drop(db); + std::panic::resume_unwind(panic) + } + } } /// Construct an in-memory instance of `database` running `program`. @@ -1219,6 +1206,7 @@ impl Host { deployment: Option, energy_monitor: Arc, on_panic: impl Fn() + Send + Sync + 'static, + registration: Registration, core: AllocatedJobCore, database_committed: &mut bool, ) -> anyhow::Result { @@ -1268,14 +1256,18 @@ impl Host { // Otherwise, we want the database to continue running with the old state. match update_result { UpdateDatabaseResult::NoUpdateNeeded | UpdateDatabaseResult::UpdatePerformed { .. } => { + registration.activate(); scheduler_starter.start(&module)?; self.scheduler = scheduler; + self.registration = registration.clone(); let old_module = self.module.send_replace(module); old_module.exit().await; } // In this case, we need to disconnect all clients connected to the old module UpdateDatabaseResult::UpdatePerformedWithClientDisconnect { .. } => { + self.registration = registration; + self.registration.activate(); // Replace the module first, so that new clients get the new module. let old_watcher = std::mem::replace(&mut self.module, watch::Sender::new(module.clone())); let old_module = old_watcher.borrow().clone(); diff --git a/crates/core/src/host/host_controller/registry.rs b/crates/core/src/host/host_controller/registry.rs new file mode 100644 index 00000000000..3b4ababd3f6 --- /dev/null +++ b/crates/core/src/host/host_controller/registry.rs @@ -0,0 +1,339 @@ +//! Canonical host cells, pinned by queued and active operations. +//! +//! Lock order: the registry mutex is never held across an await. A cell guard +//! may briefly acquire the registry mutex to publish its resident-host state. +//! It releases the cell guard before releasing its operation pin. The last pin +//! removes an empty, non-closing entry without waiting for unrelated Arc owners. + +use super::{Host, HostCell}; +use parking_lot::Mutex; +use spacetimedb_data_structures::map::IntMap; +use std::ops::{Deref, DerefMut}; +use std::sync::{Arc, Weak}; +use tokio::sync::{watch, OwnedRwLockReadGuard, OwnedRwLockWriteGuard}; + +pub(super) type Hosts = Arc>>; + +pub(super) struct Entry { + cell: HostCell, + pins: usize, + resident: bool, + registration: Option>, + closing: Option>, +} + +/// Completion means the storage writer has actually closed, not that the +/// caller's wait expired. Provider-owned snapshot/archival services are separate. +pub(super) struct Closing { + result: watch::Sender>>>, + /// An unconfirmed cold writer remains charged to the finite capacity. + /// Recovery requires positive external repair or process termination; + /// recreating a controller is not proof that an old writer has stopped. + _retained_capacity: Option, +} + +impl Closing { + pub async fn wait(&self) -> anyhow::Result<()> { + let mut receiver = self.result.subscribe(); + let result = receiver.wait_for(Option::is_some).await?.clone().unwrap(); + result.map_err(|message| anyhow::anyhow!(message.to_string())) + } +} + +/// A pin counts an operation, including time queued for the cell lock. Idle +/// ModuleHost references are not pins. Raw cells never authorize an operation. +pub(super) struct Pin { + hosts: Weak>>, + replica: u64, + cell: HostCell, +} + +impl Pin { + pub fn acquire(hosts: &Hosts, replica: u64) -> Self { + let mut entries = hosts.lock(); + let entry = entries.entry(replica).or_insert_with(|| Entry { + cell: HostCell::default(), + pins: 0, + resident: false, + registration: None, + closing: None, + }); + entry.pins += 1; + Self { + hosts: Arc::downgrade(hosts), + replica, + cell: entry.cell.clone(), + } + } + + fn closing(&self) -> Option> { + let hosts = self.hosts.upgrade()?; + let entries = hosts.lock(); + let entry = entries.get(&self.replica)?; + debug_assert!(Arc::ptr_eq(&entry.cell, &self.cell)); + entry.closing.clone() + } + + pub async fn read(hosts: &Hosts, replica: u64) -> anyhow::Result { + loop { + let pin = Self::acquire(hosts, replica); + let guard = pin.cell.clone().read_owned().await; + if let Some(closing) = pin.closing() { + drop(guard); + drop(pin); + closing.wait().await?; + continue; + } + return Ok(ReadGuard { + guard: Some(guard), + pin: Some(pin), + }); + } + } + + pub async fn write(hosts: &Hosts, replica: u64) -> anyhow::Result { + loop { + let pin = Self::acquire(hosts, replica); + let guard = pin.cell.clone().write_owned().await; + if let Some(closing) = pin.closing() { + drop(guard); + drop(pin); + closing.wait().await?; + continue; + } + return Ok(WriteGuard { + guard: Some(guard), + pin: Some(pin), + }); + } + } + + fn publish(&self, host: Option<&Host>) { + let Some(hosts) = self.hosts.upgrade() else { return }; + let mut entries = hosts.lock(); + let Some(entry) = entries.get_mut(&self.replica) else { + return; + }; + debug_assert!(Arc::ptr_eq(&entry.cell, &self.cell)); + entry.resident = host.is_some(); + entry.registration = host.map(|host| host.registration.token.clone()); + } + + fn registration(&self) -> Registration { + Registration { + hosts: self.hosts.clone(), + replica: self.replica, + cell: Arc::downgrade(&self.cell), + token: Arc::new(()), + } + } +} + +impl Drop for Pin { + fn drop(&mut self) { + let Some(hosts) = self.hosts.upgrade() else { return }; + let removed = { + let mut entries = hosts.lock(); + let Some(entry) = entries.get_mut(&self.replica) else { + return; + }; + assert!(Arc::ptr_eq(&entry.cell, &self.cell), "replaced a pinned host cell"); + entry.pins -= 1; + if entry.pins == 0 && !entry.resident && entry.closing.is_none() { + entries.remove(&self.replica) + } else { + None + } + }; + drop(removed); + } +} + +pub(super) struct ReadGuard { + guard: Option>>, + pin: Option, +} + +impl Deref for ReadGuard { + type Target = Option; + fn deref(&self) -> &Self::Target { + self.guard.as_ref().unwrap() + } +} + +impl Drop for ReadGuard { + fn drop(&mut self) { + drop(self.guard.take()); + drop(self.pin.take()); + } +} + +pub(super) struct WriteGuard { + guard: Option>>, + pin: Option, +} + +impl WriteGuard { + pub fn registration(&self) -> Registration { + self.pin.as_ref().unwrap().registration() + } + + pub fn install(&mut self, host: Host) { + **self = Some(host); + self.pin.as_ref().unwrap().publish(self.as_ref()); + } + + pub fn quarantine(&self, capacity: Option) { + let pin = self.pin.as_ref().unwrap(); + let Some(hosts) = pin.hosts.upgrade() else { return }; + let mut entries = hosts.lock(); + let entry = entries.get_mut(&pin.replica).expect("pinned cell exists"); + if let Some(closing) = &entry.closing { + closing + .result + .send_replace(Some(Err("storage writer close is unconfirmed".into()))); + } + entry.closing = Some(Arc::new(Closing { + result: watch::Sender::new(Some(Err( + "storage writer close is unconfirmed; replica is quarantined".into() + ))), + _retained_capacity: capacity, + })); + } +} + +impl Deref for WriteGuard { + type Target = Option; + fn deref(&self) -> &Self::Target { + self.guard.as_ref().unwrap() + } +} +impl DerefMut for WriteGuard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.guard.as_mut().unwrap() + } +} +impl Drop for WriteGuard { + fn drop(&mut self) { + self.pin.as_ref().unwrap().publish(self.as_ref()); + drop(self.guard.take()); + drop(self.pin.take()); + } +} + +/// The token changes with each installed executable, even when its cell and +/// database stay the same. A late panic from an old executable is not authority +/// to close its replacement. +#[derive(Clone)] +pub(super) struct Registration { + hosts: Weak>>, + replica: u64, + cell: Weak>>, + token: Arc<()>, +} + +impl Registration { + pub fn activate(&self) { + let (Some(hosts), Some(cell)) = (self.hosts.upgrade(), self.cell.upgrade()) else { + return; + }; + let mut entries = hosts.lock(); + if let Some(entry) = entries.get_mut(&self.replica) + && Arc::ptr_eq(&entry.cell, &cell) + { + entry.registration = Some(self.token.clone()); + } + } + + pub fn close_if_current(&self) -> Option { + let (Some(hosts), Some(cell)) = (self.hosts.upgrade(), self.cell.upgrade()) else { + return None; + }; + let mut entries = hosts.lock(); + let entry = entries.get_mut(&self.replica)?; + if !Arc::ptr_eq(&entry.cell, &cell) + || !entry + .registration + .as_ref() + .is_some_and(|token| Arc::ptr_eq(token, &self.token)) + { + return None; + } + Some(request_close(&hosts, self.replica, entry)) + } +} + +pub(super) struct CloseOwner { + pin: Pin, + completion: Arc, +} + +pub(super) struct CloseRequest { + pub completion: Arc, + pub owner: Option, +} + +pub(super) fn close(hosts: &Hosts, replica: u64) -> Option { + let mut entries = hosts.lock(); + let entry = entries.get_mut(&replica)?; + Some(request_close(hosts, replica, entry)) +} + +fn request_close(hosts: &Hosts, replica: u64, entry: &mut Entry) -> CloseRequest { + if let Some(completion) = &entry.closing { + return CloseRequest { + completion: completion.clone(), + owner: None, + }; + } + let completion = Arc::new(Closing { + result: watch::Sender::new(None), + _retained_capacity: None, + }); + entry.closing = Some(completion.clone()); + entry.pins += 1; + CloseRequest { + completion: completion.clone(), + owner: Some(CloseOwner { + pin: Pin { + hosts: Arc::downgrade(hosts), + replica, + cell: entry.cell.clone(), + }, + completion, + }), + } +} + +impl CloseOwner { + pub async fn run(self) { + let mut guard = self.pin.cell.clone().write_owned().await; + // A prior cold owner can report an unconfirmed writer before this + // queued close obtains its guard. An empty cell is not proof of closure. + if self.completion.result.borrow().is_some() { + return; + } + let result = match guard.take() { + Some(host) => super::retained::close_host(host).await, + None => Ok(()), + }; + let writer_closed = !matches!(result, Err(super::retained::CloseFailure::WriterUnconfirmed)); + // Publish under the cell lock, then release it before final registry + // unpin/removal. Closing stays set during both steps, so reopen waits. + self.pin.publish(None); + drop(guard); + if let Some(hosts) = self.pin.hosts.upgrade() { + let mut entries = hosts.lock(); + if let Some(entry) = entries.get_mut(&self.pin.replica) { + debug_assert!(Arc::ptr_eq(&entry.cell, &self.pin.cell)); + if writer_closed { + entry.closing = None; + } + } + } + self.completion + .result + .send_replace(Some(result.map_err(|error| Arc::from(error.to_string())))); + // self.pin drops last. An unrelated idle Arc cannot delay completion. + } +} diff --git a/crates/core/src/host/host_controller/retained.rs b/crates/core/src/host/host_controller/retained.rs new file mode 100644 index 00000000000..e661012a716 --- /dev/null +++ b/crates/core/src/host/host_controller/retained.rs @@ -0,0 +1,251 @@ +//! Operation-scoped access to initialized retained storage, without starting a +//! module. Only the configured durability writer is joined on close; shared +//! provider snapshot and archival services retain their existing ownership. + +use super::*; +use crate::db::persistence::Persistence; +use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; +use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; +use std::panic::{resume_unwind, AssertUnwindSafe}; +use std::sync::OnceLock; +use tokio::sync::Semaphore; + +#[derive(Debug, thiserror::Error)] +pub(super) enum CloseFailure { + #[error("module exit panicked; the storage writer was positively closed")] + ModuleExit, + #[error("storage writer close panicked; the replica remains quarantined")] + WriterUnconfirmed, +} + +#[cfg(test)] +pub(super) static FAIL_NEXT_MODULE_EXIT: parking_lot::Mutex> = + parking_lot::Mutex::new(std::collections::BTreeSet::new()); + +#[cfg(test)] +pub(super) static PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START: parking_lot::Mutex> = + parking_lot::Mutex::new(std::collections::BTreeSet::new()); + +pub(super) fn writer_unconfirmed(error: &anyhow::Error) -> bool { + matches!( + error.downcast_ref::(), + Some(CloseFailure::WriterUnconfirmed) + ) +} + +/// Some providers hand the first close caller the writer's JoinHandle and let +/// later calls finish immediately. Share that *first* physical close instead. +/// In particular, partial RelationalDB replay may start close from Drop before +/// our owning task sees its error. That task must join the same close future. +pub(super) struct JoinedDurability { + inner: Arc>, + close: OnceLock>>>, +} + +impl JoinedDurability { + pub fn wrap(persistence: &mut Persistence) -> Arc { + let joined = Arc::new(Self { + inner: persistence.durability.clone(), + close: OnceLock::new(), + }); + persistence.durability = joined.clone(); + joined + } + + pub async fn join(&self) -> Result, CloseFailure> { + AssertUnwindSafe(async { self.close().await }) + .catch_unwind() + .await + .map_err(|_| CloseFailure::WriterUnconfirmed) + } +} + +impl Durability for JoinedDurability { + type TxData = Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.inner.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.inner.durable_tx_offset() + } + fn close(&self) -> Close { + self.close.get_or_init(|| self.inner.close().shared()).clone().boxed() + } +} + +/// Shared with ordinary initialization, so a failed normal replay cannot leave +/// its writer shutting down behind a later retained-storage operation. +pub(super) async fn open_database( + controller: &HostController, + database: &Database, + replica_id: u64, + retained_only: bool, + tx_metrics_queue: Option, +) -> anyhow::Result<( + Arc, + relational_db::ConnectedClients, + Option>, +)> { + if matches!(controller.default_config.storage, db::Storage::Memory) { + anyhow::ensure!(!retained_only, "retained storage requires disk persistence"); + let (db, clients) = RelationalDB::open( + database.database_identity, + database.owner_identity, + EmptyHistory::new(), + None, + tx_metrics_queue, + controller.page_pool.clone(), + )?; + return Ok((Arc::new(db), clients, None)); + } + + let replica_dir = controller.data_dir.replica(replica_id); + if retained_only { + let commit_log = replica_dir.commit_log(); + // Fs::new can create a missing directory. Establish existing history + // before calling either that helper or the configured provider. + asyncify(move || { + anyhow::ensure!(commit_log.is_dir(), "retained database history is absent"); + anyhow::ensure!( + spacetimedb_commitlog::committed_meta(commit_log)?.is_some(), + "retained database history is empty" + ); + Ok::<_, anyhow::Error>(()) + }) + .await?; + } + let history = relational_db::local_history(&replica_dir).await?; + let mut persistence = controller.persistence.persistence(database, replica_id).await?; + let joined = JoinedDurability::wrap(&mut persistence); + let identity = database.database_identity; + let owner = database.owner_identity; + let page_pool = controller.page_pool.clone(); + let opened = AssertUnwindSafe(asyncify(move || { + RelationalDB::open(identity, owner, history, Some(persistence), tx_metrics_queue, page_pool) + })) + .catch_unwind() + .await; + let (db, clients) = match opened { + Ok(Ok(opened)) => opened, + Ok(Err(error)) => { + joined.join().await?; + return Err(error.into()); + } + Err(panic) => { + joined.join().await?; + resume_unwind(panic); + } + }; + let db = Arc::new(db); + if retained_only { + let validation = db + .metadata() + .and_then(|metadata| metadata.ok_or_else(|| anyhow!("retained database is not initialized").into())); + if let Err(error) = validation { + joined.join().await?; + drop(db); + return Err(error.into()); + } + } + Ok((db, clients, Some(joined))) +} + +impl HostController { + /// Access retained initialized database storage without launching user code. + /// + /// This is a trusted host API, not an external authorization endpoint. The + /// caller must confirm current leadership and operation authority. It must + /// not retain or return the supplied database/module handles, spawn work + /// that outlives the returned future, or reenter this replica's controller. + /// A cold open keeps hosted admission closed and never runs initialization, + /// lifecycle reducers, scheduled functions, or module compilation. + /// + /// Caller cancellation does not stop the owned operation. Its finite + /// capacity permit and registry pin remain until its writer has closed. + pub async fn with_retained_database( + &self, + database: Database, + replica_id: u64, + operation: F, + ) -> anyhow::Result + where + T: Send + 'static, + F: FnOnce(Arc, Option) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let permit = self + .retained_capacity + .clone() + .try_acquire_owned() + .map_err(|_| anyhow!("retained database operation capacity exhausted"))?; + let controller = self.clone(); + tokio::spawn(async move { + let _permit = permit; + let guard = controller + .acquire_write_lock(replica_id) + .await + .map_err(|_| anyhow!("unable to lock retained database"))?; + if let Some(host) = guard.as_ref() { + anyhow::ensure!( + host.replica_ctx.database.database_identity == database.database_identity + && host.replica_ctx.database.owner_identity == database.owner_identity, + "retained database identity mismatch" + ); + let module = host.module.borrow().clone(); + let db = host.replica_ctx.relational_db().clone(); + return operation(db, Some(module)).await; + } + let (db, _clients, joined) = match open_database(&controller, &database, replica_id, true, None).await { + Ok(opened) => opened, + Err(error) => { + if writer_unconfirmed(&error) { + guard.quarantine(Some(_permit)); + } + return Err(error); + } + }; + let result = AssertUnwindSafe(async { operation(db.clone(), None).await }) + .catch_unwind() + .await; + // Keep the guard and permit while waiting for the actual first close + // even if an inner helper or RelationalDB::Drop also requests close. + if let Err(error) = joined.expect("retained open always uses disk persistence").join().await { + guard.quarantine(Some(_permit)); + return Err(error.into()); + } + drop(db); + drop(guard); + match result { + Ok(result) => result, + Err(panic) => resume_unwind(panic), + } + }) + .await? + } +} + +pub(super) async fn close_host(host: Host) -> Result<(), CloseFailure> { + let module = host.module.borrow().clone(); + let info = module.info(); + let identity = info.database_identity; + let table_names = info.module_def.tables().map(|table| table.name.deref()); + defer!(remove_database_gauges(&identity, table_names)); + let exited = AssertUnwindSafe(async { + module.exit().await; + #[cfg(test)] + if FAIL_NEXT_MODULE_EXIT.lock().remove(&identity) { + panic!("injected module exit failure"); + } + }) + .catch_unwind() + .await; + let writer = AssertUnwindSafe(module.relational_db().shutdown()).catch_unwind().await; + drop(host); + writer.map_err(|_| CloseFailure::WriterUnconfirmed)?; + exited.map_err(|_| CloseFailure::ModuleExit) +} + +pub(super) fn capacity() -> Arc { + Arc::new(Semaphore::new(2)) +} diff --git a/crates/core/src/host/host_controller/retained_tests.rs b/crates/core/src/host/host_controller/retained_tests.rs new file mode 100644 index 00000000000..ef7fda18bb7 --- /dev/null +++ b/crates/core/src/host/host_controller/retained_tests.rs @@ -0,0 +1,689 @@ +//! Real local durability writers and module hosts. No network or external +//! configuration. The probe counts writers until their first close completes. +use super::*; +use crate::db::persistence::{LocalPersistenceProvider, Persistence}; +use crate::host::empty_module; +use futures::FutureExt; +use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; +use spacetimedb_paths::FromPathUnchecked; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use tokio::sync::Semaphore; + +struct Probe { + opened: AtomicUsize, + active: AtomicUsize, + maximum: AtomicUsize, + block_close: AtomicBool, + panic_close: AtomicBool, + close_started: Semaphore, + release_close: Semaphore, +} + +impl Default for Probe { + fn default() -> Self { + Self { + opened: AtomicUsize::new(0), + active: AtomicUsize::new(0), + maximum: AtomicUsize::new(0), + block_close: AtomicBool::new(false), + panic_close: AtomicBool::new(false), + close_started: Semaphore::new(0), + release_close: Semaphore::new(0), + } + } +} + +struct ProbedProvider { + local: LocalPersistenceProvider, + probe: Arc, +} + +#[async_trait] +impl PersistenceProvider for ProbedProvider { + async fn persistence(&self, database: &Database, replica: u64) -> anyhow::Result { + let mut persistence = self.local.persistence(database, replica).await?; + let active = self.probe.active.fetch_add(1, Ordering::SeqCst) + 1; + self.probe.maximum.fetch_max(active, Ordering::SeqCst); + self.probe.opened.fetch_add(1, Ordering::SeqCst); + persistence.durability = Arc::new(ProbedWriter { + inner: persistence.durability, + probe: self.probe.clone(), + closing: AtomicBool::new(false), + }); + Ok(persistence) + } +} + +struct ProbedWriter { + inner: Arc>, + probe: Arc, + closing: AtomicBool, +} + +impl Durability for ProbedWriter { + type TxData = Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.inner.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.inner.durable_tx_offset() + } + fn close(&self) -> Close { + // Reproduce the actual first-caller-owns-join behavior deliberately. + // A second close must not let another database open before this one. + if self.closing.swap(true, Ordering::SeqCst) { + let offset = self.inner.durable_tx_offset().last_seen(); + return async move { offset }.boxed(); + } + let close = self.inner.close(); + let probe = self.probe.clone(); + async move { + probe.close_started.add_permits(1); + if probe.block_close.load(Ordering::SeqCst) { + probe.release_close.acquire().await.unwrap().forget(); + } + assert!( + !probe.panic_close.load(Ordering::SeqCst), + "injected writer close failure" + ); + let offset = close.await; + probe.active.fetch_sub(1, Ordering::SeqCst); + offset + } + .boxed() + } +} + +fn fixture( + id: u64, +) -> ( + tempfile::TempDir, + HostController, + Database, + Arc, + Arc, +) { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let program = empty_module::program(1).unwrap(); + let initial = program.clone(); + let lookups = Arc::new(AtomicUsize::new(0)); + let lookup_count = lookups.clone(); + let storage = move |hash| { + let initial = initial.clone(); + lookup_count.fetch_add(1, Ordering::SeqCst); + async move { Ok((hash == initial.hash).then_some(initial.bytes)) } + }; + let probe = Arc::new(Probe::default()); + let controller = HostController::new( + data.clone(), + db::Config { + storage: db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(storage), + Arc::new(NullEnergyMonitor), + Arc::new(ProbedProvider { + local: LocalPersistenceProvider::new(data), + probe: probe.clone(), + }), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: HostType::Wasm, + initial_program: program.hash, + }; + (directory, controller, database, probe, lookups) +} + +async fn closed_seed(controller: &HostController, database: &Database, probe: &Probe) { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + probe.close_started.acquire().await.unwrap().forget(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); +} + +async fn wait_for_close(probe: &Probe) { + timeout(Duration::from_secs(5), probe.close_started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); +} + +async fn wait_for_capacity(controller: &HostController) { + timeout(Duration::from_secs(5), async { + while controller.retained_capacity.available_permits() != 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_missing_history_does_not_create_a_replica_or_lookup_program() { + let (_directory, controller, database, probe, lookups) = fixture(0xc001); + assert!(controller + .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) + .await + .is_err()); + assert!(!controller.data_dir.replica(database.id).0.exists()); + assert_eq!(probe.opened.load(Ordering::SeqCst), 0); + assert_eq!(lookups.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_idle_module_and_queued_relookup_do_not_break_positive_close() { + let (_directory, controller, database, probe, _) = fixture(0xc002); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + probe.block_close.store(true, Ordering::SeqCst); + let exiting = { + let controller = controller.clone(); + tokio::spawn(async move { controller.exit_module_host(0xc002, Duration::from_secs(5)).await }) + }; + wait_for_close(&probe).await; + let lookup = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + }) + }; + assert!( + controller + .exit_module_host(database.id, Duration::from_millis(10)) + .await + .is_err(), + "a timeout is not positive closure" + ); + exiting.abort(); + assert!(exiting.await.unwrap_err().is_cancelled()); + assert_eq!(probe.opened.load(Ordering::SeqCst), 1); + assert_eq!(probe.active.load(Ordering::SeqCst), 1); + probe.block_close.store(false, Ordering::SeqCst); + probe.release_close.add_permits(1); + let current = timeout(Duration::from_secs(5), lookup).await.unwrap().unwrap().unwrap(); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + assert!(!Arc::ptr_eq(idle.relational_db(), current.relational_db())); + let seen_live = controller + .with_retained_database(database.clone(), database.id, |db, module| async move { + Ok(module.is_some() && db.metadata()?.is_some()) + }) + .await + .unwrap(); + assert!(seen_live); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); + drop((idle, current)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_cancelled_call_keeps_capacity_and_writer_until_positive_close() { + let (_directory, controller, database, probe, _) = fixture(0xc003); + closed_seed(&controller, &database, &probe).await; + probe.block_close.store(true, Ordering::SeqCst); + let first = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .with_retained_database(database.clone(), database.id, |_, module| async move { + assert!(module.is_none()); + Ok(()) + }) + .await + }) + }; + wait_for_close(&probe).await; + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + assert_eq!(controller.retained_capacity.available_permits(), 1); + let second = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) + .await + }) + }; + timeout(Duration::from_secs(5), async { + while controller.retained_capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(controller + .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) + .await + .is_err()); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + probe.release_close.add_permits(1); + wait_for_close(&probe).await; + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + probe.release_close.add_permits(1); + second.await.unwrap().unwrap(); + wait_for_capacity(&controller).await; + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_failed_replay_joins_the_first_close_before_retry() { + let (_directory, controller, database, probe, _) = fixture(0xc004); + closed_seed(&controller, &database, &probe).await; + let mut wrong_owner = database.clone(); + wrong_owner.owner_identity = Identity::from_u256(123456_u64.into()); + probe.block_close.store(true, Ordering::SeqCst); + let failed = { + let controller = controller.clone(); + tokio::spawn(async move { + controller + .with_retained_database(wrong_owner.clone(), wrong_owner.id, |_, _| async { Ok(()) }) + .await + }) + }; + wait_for_close(&probe).await; + assert!(!failed.is_finished()); + let next = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .with_retained_database(database.clone(), database.id, |db, _| async move { + Ok(db.metadata()?.is_some()) + }) + .await + }) + }; + tokio::task::yield_now().await; + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + probe.release_close.add_permits(1); + assert!(failed.await.unwrap().is_err()); + wait_for_close(&probe).await; + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + probe.release_close.add_permits(1); + assert!(next.await.unwrap().unwrap()); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_operation_panic_closes_writer_and_releases_its_pin() { + let (_directory, controller, database, probe, _) = fixture(0xc005); + closed_seed(&controller, &database, &probe).await; + assert!(controller + .with_retained_database(database.clone(), database.id, |_, _| async { + panic!("injected trusted operation panic"); + #[allow(unreachable_code)] + Ok::<_, anyhow::Error>(()) + }) + .await + .is_err()); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + controller + .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + assert!(controller.managed_replicas().is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_failed_normal_initialization_closes_writer_before_cold_retry() { + let (_directory, controller, database, probe, _) = fixture(0xc008); + closed_seed(&controller, &database, &probe).await; + let mut wrong_owner = database.clone(); + wrong_owner.owner_identity = Identity::from_u256(123456_u64.into()); + probe.block_close.store(true, Ordering::SeqCst); + let failed = { + let controller = controller.clone(); + tokio::spawn(async move { + controller + .get_or_launch_module_host(wrong_owner.clone(), wrong_owner.id) + .await + }) + }; + wait_for_close(&probe).await; + let next = { + let controller = controller.clone(); + let database = database.clone(); + tokio::spawn(async move { + controller + .with_retained_database(database.clone(), database.id, |_, module| async move { + assert!(module.is_none()); + Ok(()) + }) + .await + }) + }; + assert!(!failed.is_finished()); + probe.release_close.add_permits(1); + assert!(failed.await.unwrap().is_err()); + wait_for_close(&probe).await; + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + probe.release_close.add_permits(1); + next.await.unwrap().unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_nonempty_history_without_initialized_module_is_rejected() { + let (_directory, controller, database, probe, lookups) = fixture(0xc009); + let (db, _, joined) = retained::open_database(&controller, &database, database.id, false, None) + .await + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + crate::db::environment::set(&db, tx, "PENDING", "value") + }) + .unwrap(); + assert!(db.metadata().unwrap().is_none()); + joined.unwrap().close().await; + drop(db); + let result = controller + .with_retained_database(database.clone(), database.id, |_, _| async { + panic!("uninitialized storage must never reach the operation"); + #[allow(unreachable_code)] + Ok::<_, anyhow::Error>(()) + }) + .await; + assert!(result.unwrap_err().to_string().contains("not initialized")); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + assert_eq!(lookups.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_module_exit_panic_reports_error_after_positive_writer_close_and_allows_reopen() { + let (_directory, controller, database, probe, _) = fixture(0xc00a); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + retained::FAIL_NEXT_MODULE_EXIT + .lock() + .insert(database.database_identity); + let error = controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap_err(); + assert!(error.to_string().contains("module exit panicked")); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.managed_replicas().is_empty()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + drop(idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_unconfirmed_writer_close_reports_error_and_quarantines_capacity_and_replica() { + let (_directory, controller, database, probe, _) = fixture(0xc00b); + closed_seed(&controller, &database, &probe).await; + probe.panic_close.store(true, Ordering::SeqCst); + let error = controller + .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) + .await + .unwrap_err(); + assert!(error.to_string().contains("writer close panicked")); + assert_eq!(controller.retained_capacity.available_permits(), 1); + let error = timeout( + Duration::from_secs(1), + controller.get_or_launch_module_host(database.clone(), database.id), + ) + .await + .unwrap() + .unwrap_err(); + assert!(error.to_string().contains("unable to lock")); + assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + assert!(controller + .exit_module_host(database.id, Duration::from_secs(1)) + .await + .unwrap_err() + .to_string() + .contains("quarantined")); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_panic_callback_before_initial_host_install_still_owns_cleanup() { + let (_directory, controller, database, probe, _) = fixture(0xc00c); + retained::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + .lock() + .insert(database.database_identity); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + // No explicit exit is requested until the injected scheduler interleaving + // has independently started the actual storage writer close. + wait_for_close(&probe).await; + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert!(controller.get_module_host(database.id).await.is_err()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + drop(idle); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_stale_panic_callback_does_not_unregister_updated_or_reopened_host() { + let (_directory, controller, database, probe, _) = fixture(0xc006); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let old_callback = { + let guard = controller.acquire_read_lock(database.id).await.unwrap(); + controller.unregister_fn(guard.as_ref().unwrap().registration.clone()) + }; + let mut newer = empty_module::VERSION_1_BYTES.to_vec(); + newer.extend_from_slice(&[0, 3, 1, b'x', 1]); + let newer_hash = spacetimedb_lib::hash_bytes(&newer); + controller + .update_module_host( + database.clone(), + HostType::Wasm, + database.id, + newer.into(), + MigrationPolicy::Compatible, + ) + .await + .unwrap(); + old_callback(); + assert_eq!( + controller.get_module_host(database.id).await.unwrap().info.module_hash, + newer_hash + ); + assert_eq!(probe.active.load(Ordering::SeqCst), 1); + let current_callback = { + let guard = controller.acquire_read_lock(database.id).await.unwrap(); + controller.unregister_fn(guard.as_ref().unwrap().registration.clone()) + }; + current_callback(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert!(controller.get_module_host(database.id).await.is_err()); + controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + current_callback(); + assert!(controller.get_module_host(database.id).await.is_ok()); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn retained_cold_snapshot_cleanup_never_executes_stored_javascript_or_opens_admission() { + use crate::db::deployment::{install_container_fence, install_publication_fence, record_deployment_commit}; + use crate::host::container_environment; + use spacetimedb_datastore::system_tables::StContainerFenceRow; + use spacetimedb_lib::container::*; + use spacetimedb_lib::container_environment::EnvironmentSnapshotScope; + use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; + use spacetimedb_lib::{hash_bytes, Uuid}; + + let (_directory, controller, database, probe, lookups) = fixture(0xc007); + let idle = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + let hostile = Program::from_bytes( + ModuleKind::JS, + b"throw new Error('cold storage must never execute this module');".to_vec(), + ); + let uuid = || Uuid::from_u128(uuid::Uuid::now_v7().as_u128()); + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: vec!["SECRET".into()], + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + }; + let publication = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: database.owner_identity, + expected_revision: None, + prepared_manifest_hash: hash_bytes(b"cold cleanup fixture"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::User(UserModule { + kind: UserModuleKind::Js, + program_hash: hostile.hash, + }), + container: Some(spec.clone()), + }), + }; + let fence = StContainerFenceRow { + source_identity: database.database_identity.into(), + generation: 1, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed: true, + }; + idle.relational_db() + .with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { + idle.relational_db().update_program(tx, hostile)?; + install_publication_fence(tx, publication.publication_epoch, publication.operation_id)?; + record_deployment_commit(tx, &publication, Timestamp::now(), &Default::default())?; + install_container_fence(idle.relational_db(), tx, &fence)?; + crate::db::environment::set(idle.relational_db(), tx, "SECRET", "retained-fixture-secret")?; + Ok(()) + }) + .unwrap(); + let scope = EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: database.id, + database_identity: database.database_identity, + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: publication.deployment.revision().unwrap(), + start_request: publication.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + }; + let receipt = container_environment::capture(idle.relational_db().clone(), scope.clone()) + .await + .unwrap(); + idle.relational_db() + .hosted_admission() + .begin() + .unwrap() + .complete() + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let expected_scope = scope.clone(); + controller + .with_retained_database(database.clone(), database.id, move |db, module| async move { + assert!(module.is_none()); + assert!(!db.hosted_admission().is_open()); + let values = container_environment::read(db.clone(), receipt.receipt).await?; + assert_eq!(values.receipt.selected_values["SECRET"], "retained-fixture-secret"); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence( + &db, + tx, + &StContainerFenceRow { + generation: 2, + allowed: false, + ..fence + }, + ) + })?; + container_environment::close(db, expected_scope, 2).await?; + Ok(()) + }) + .await + .unwrap(); + assert!(controller.get_module_host(database.id).await.is_err()); + assert_eq!(lookups.load(Ordering::SeqCst), 1); + assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + assert_eq!(probe.active.load(Ordering::SeqCst), 0); + assert_eq!(controller.retained_capacity.available_permits(), 2); + drop(idle); +} diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 888fe5558c0..c0d90ac2333 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -319,7 +319,7 @@ impl InstanceEnv { return Err(NodesError::NotInTransaction); } self.relational_db().with_read_only(Workload::Internal, |tx| { - check_hosted_admission(tx, *self.database_identity(), self.hosted_auth.as_deref()) + check_hosted_admission(tx, self.relational_db(), self.hosted_auth.as_deref()) .map_err(|err| NodesError::HostedInvocationRejected(err.to_string()))?; environment::get(tx, key).map_err(|err| NodesError::from(DBError::Other(err.into()))) }) @@ -814,7 +814,7 @@ impl InstanceEnv { let tx = self .relational_db() .begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - if let Err(err) = check_hosted_admission(&tx, *self.database_identity(), self.hosted_auth.as_deref()) { + if let Err(err) = check_hosted_admission(&tx, self.relational_db(), self.hosted_auth.as_deref()) { let _ = tx.rollback(); return Err(NodesError::HostedInvocationRejected(err.to_string())); } @@ -1522,6 +1522,7 @@ mod test { use spacetimedb_datastore::system_tables::StContainerFenceRow; use std::time::SystemTime; let db = relational_db()?; + db.hosted_admission().begin()?.complete()?; let (mut env, _runtime) = instance_env(db.clone())?; env.func_type = FuncCallType::Procedure; let keys = JwtKeys::generate()?; diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index e3401bbb0ab..97acb630a86 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -11,6 +11,7 @@ use spacetimedb_schema::def::deserialize::{ArgsSeed, FunctionDef}; use spacetimedb_schema::def::ModuleDef; pub mod container_environment; +pub mod container_fence; mod disk_storage; pub mod empty_module; mod host_controller; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 91c7a1dd091..b41d8ef8a34 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -745,7 +745,7 @@ pub fn call_identity_connected( let flags = caller .flags_for(module.database_identity, &module.module_def) .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; - check_hosted_admission(&*mut_tx, module.database_identity, caller.hosted.as_deref()) + check_hosted_admission(&*mut_tx, stdb, caller.hosted.as_deref()) .map_err(|e| ClientConnectedError::Rejected(e.to_string().into()))?; mut_tx @@ -3539,7 +3539,7 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = check_hosted_admission(&*tx, db.database_identity(), client.auth.hosted.as_ref()).and_then(|()| { + let result = check_hosted_admission(&*tx, &db, client.auth.hosted.as_ref()).and_then(|()| { Self::execute_one_off_query(&db, &tx, &auth, &query, &rlb_pool, |table_name, rows| { ws_v1::OneOffTable { table_name, rows } }) @@ -3620,7 +3620,7 @@ impl ModuleHost { db.report_read_tx_metrics(reducer, tx_metrics); }); - let result = check_hosted_admission(&*tx, db.database_identity(), client.auth.hosted.as_ref()).and_then(|()| { + let result = check_hosted_admission(&*tx, &db, client.auth.hosted.as_ref()).and_then(|()| { Self::execute_one_off_query::(&db, &tx, &auth, &query, &rlb_pool, |table, rows| { ws_v2::SingleTableRows { table, rows } }) diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 736965b8710..0b1569177d4 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -867,7 +867,7 @@ impl InstanceCommon { .replica_ctx() .relational_db() .with_read_only(Workload::Internal, |tx| { - check_hosted_admission(tx, self.info.database_identity, hosted_auth.as_deref()) + check_hosted_admission(tx, inst.replica_ctx().relational_db(), hosted_auth.as_deref()) }); if let Err(err) = admission { return ( @@ -1105,7 +1105,7 @@ impl InstanceCommon { let workload = Workload::Reducer(ReducerContext::from(op.clone())); let tx = tx.unwrap_or_else(|| stdb.begin_mut_tx(IsolationLevel::Serializable, workload)); - if let Err(err) = check_hosted_admission(&tx, info.database_identity, op.hosted_auth.as_deref()) { + if let Err(err) = check_hosted_admission(&tx, stdb, op.hosted_auth.as_deref()) { let event = ModuleEvent { timestamp, caller_identity, diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 6c89e6adc3e..5c3c8fea4b2 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -94,7 +94,7 @@ fn run_inner( // We parse the sql statement in a mutable transaction. // If it turns out to be a query, we downgrade the tx. let (tx, stmt) = db.with_auto_rollback(db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql), |tx| { - check_hosted_admission(tx, db.database_identity(), auth.hosted.as_deref())?; + check_hosted_admission(tx, &db, auth.hosted.as_deref())?; let stmt = compile_sql_stmt(&sql_text, &SchemaViewer::new(tx, &auth), &auth)?; // Check mutation authority while the automatic rollback guard owns // the transaction, including rejected administrative statements. diff --git a/crates/core/src/subscription/module_subscription_actor.rs b/crates/core/src/subscription/module_subscription_actor.rs index ac1c34fc2f4..642c32531fd 100644 --- a/crates/core/src/subscription/module_subscription_actor.rs +++ b/crates/core/src/subscription/module_subscription_actor.rs @@ -323,7 +323,7 @@ impl ModuleSubscriptions { /// database transaction before the registry lock serializes with fencing. pub(crate) fn register_hosted_connection(&self, sender: &Arc) -> anyhow::Result<()> { self.relational_db.with_read_only(Workload::Internal, |tx| { - check_hosted_admission(tx, self.relational_db.database_identity(), sender.auth.hosted.as_ref())?; + check_hosted_admission(tx, &self.relational_db, sender.auth.hosted.as_ref())?; let mut connections = self.hosted_connections.write(); connections.retain(|connection| connection.strong_count() != 0); connections.push(Arc::downgrade(sender)); @@ -351,13 +351,7 @@ impl ModuleSubscriptions { let Some(connection) = connection.upgrade() else { return false; }; - if check_hosted_admission( - tx, - self.relational_db.database_identity(), - connection.auth.hosted.as_ref(), - ) - .is_err() - { + if check_hosted_admission(tx, &self.relational_db, connection.auth.hosted.as_ref()).is_err() { cancelled.push(connection.cancel_hosted_connection()); } true @@ -686,11 +680,7 @@ impl ModuleSubscriptions { let hash_with_param = QueryHash::from_string(&sql, auth.caller(), true); let (mut_tx, _) = self.begin_mut_tx(Workload::Subscribe); - check_hosted_admission( - &*mut_tx, - self.relational_db.database_identity(), - sender.auth.hosted.as_ref(), - )?; + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let existing_query = { let guard = self.subscriptions.read(); @@ -794,11 +784,7 @@ impl ModuleSubscriptions { }; let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); - check_hosted_admission( - &*mut_tx, - self.relational_db.database_identity(), - sender.auth.hosted.as_ref(), - )?; + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let mut subscriptions = self.subscriptions.write(); let queries = return_on_err!( @@ -879,11 +865,7 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); - check_hosted_admission( - &*mut_tx, - self.relational_db.database_identity(), - sender.auth.hosted.as_ref(), - )?; + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -1001,11 +983,7 @@ impl ModuleSubscriptions { // Always lock the db before the subscription lock to avoid deadlocks. let (mut_tx, _) = self.begin_mut_tx(Workload::Unsubscribe); - check_hosted_admission( - &*mut_tx, - self.relational_db.database_identity(), - sender.auth.hosted.as_ref(), - )?; + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let removed_queries = { let _compile_timer = subscription_metrics.compilation_time.start_timer(); @@ -1109,11 +1087,7 @@ impl ModuleSubscriptions { // We always get the db lock before the subscription lock to avoid deadlocks. let (mut_tx, _tx_offset) = self.begin_mut_tx(Workload::Subscribe); - check_hosted_admission( - &*mut_tx, - self.relational_db.database_identity(), - sender.auth.hosted.as_ref(), - )?; + check_hosted_admission(&*mut_tx, &self.relational_db, sender.auth.hosted.as_ref())?; let compile_timer = metrics.compilation_time.start_timer(); diff --git a/crates/lib/src/deployment.rs b/crates/lib/src/deployment.rs index d05b4b0cd05..c1fc5e211da 100644 --- a/crates/lib/src/deployment.rs +++ b/crates/lib/src/deployment.rs @@ -12,12 +12,24 @@ pub mod manifest; pub const PUBLISH_PROTOCOL_VERSION: u32 = 1; pub const SYSTEM_EMPTY_MODULE_VERSION: u32 = 1; +/// Immutable built-in program, also used by clients for authorized migration +/// preflight. Replacing these bytes requires a new system-module version. +pub const SYSTEM_EMPTY_MODULE_V1_BYTES: &[u8] = include_bytes!("deployment/system_empty_v1.wasm"); /// Immutable Keccak-256 program identity of the version-1 bundled empty Wasm /// module. Control can verify initial program bytes without linking the host. pub const SYSTEM_EMPTY_MODULE_V1_PROGRAM_HASH: Hash = Hash::from_byte_array([ 0x08, 0x36, 0x50, 0x97, 0xd1, 0xef, 0x20, 0x26, 0x53, 0x61, 0x5f, 0x02, 0xcc, 0x46, 0xbe, 0x16, 0x08, 0x80, 0x44, 0xc6, 0xb9, 0xcd, 0x96, 0x60, 0xe3, 0xa0, 0xf3, 0x36, 0xe4, 0x79, 0x18, 0xf0, ]); +/// SHA-256 descriptor of the same immutable bundled version-1 bytes. Clients +/// can name SystemEmpty in a prepared manifest without linking the host. +pub const SYSTEM_EMPTY_MODULE_V1_ARTIFACT: manifest::ModuleArtifact = manifest::ModuleArtifact { + digest: crate::container::OciDigest::sha256([ + 0x90, 0x37, 0x89, 0x67, 0xf4, 0xf5, 0xdb, 0x99, 0x37, 0x30, 0xe3, 0x67, 0x11, 0x67, 0x1a, 0x94, 0xdc, 0x5e, + 0xc6, 0x36, 0x75, 0xf6, 0x83, 0x60, 0xa2, 0x1d, 0xd1, 0x52, 0x4b, 0xcd, 0x7a, 0xa3, + ]), + size_bytes: 250, +}; pub const MAX_DEPLOYMENT_BYTES: usize = 256 * 1024; pub const PUBLISH_RETRY_WINDOW_MS: u64 = 7 * 24 * 60 * 60 * 1000; pub const MAX_OPERATION_CLOCK_SKEW_MS: u64 = 5 * 60 * 1000; diff --git a/crates/lib/src/deployment/system_empty_v1.wasm b/crates/lib/src/deployment/system_empty_v1.wasm new file mode 100644 index 0000000000000000000000000000000000000000..811462677fb5faed967a654acb2d5f5a610edb47 GIT binary patch literal 250 zcmY+9L2JV>427TNY=edcLw6qSA1En Date: Tue, 8 Sep 2026 06:18:10 -0400 Subject: [PATCH 12/23] Retain environment operations through durability confirmation --- crates/core/src/host/container_environment.rs | 97 +++--- .../container_environment/durability_tests.rs | 288 ++++++++++++++++++ 2 files changed, 350 insertions(+), 35 deletions(-) create mode 100644 crates/core/src/host/container_environment/durability_tests.rs diff --git a/crates/core/src/host/container_environment.rs b/crates/core/src/host/container_environment.rs index e3afd2da0e4..45535df1a09 100644 --- a/crates/core/src/host/container_environment.rs +++ b/crates/core/src/host/container_environment.rs @@ -40,31 +40,45 @@ pub async fn capture( pub async fn read( db: Arc, receipt: EnvironmentSnapshotReceipt, +) -> Result { + read_with_capacity(db, receipt, OPERATIONS.clone()).await +} + +async fn read_with_capacity( + db: Arc, + receipt: EnvironmentSnapshotReceipt, + capacity: Arc, ) -> Result { let mut durability = db .durable_tx_offset() .ok_or(EnvironmentSnapshotError::DurabilityUnavailable)?; - let permit = OPERATIONS - .clone() + let permit = capacity .try_acquire_owned() .map_err(|_| EnvironmentSnapshotError::Capacity)?; - let (_permit, durable_through, receipt) = tokio::task::spawn_blocking(move || { - let tx = db.begin_tx(Workload::Internal); - let result = storage::read(&db, &tx, &receipt); - let (offset, metrics, reducer) = db.release_tx(tx); - db.report_read_tx_metrics(reducer, metrics); - result.map(|result| (permit, offset, result)) - }) - .await - .map_err(|_| EnvironmentSnapshotError::Storage)??; - durability - .wait_for(durable_through) + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_tx(Workload::Internal); + let result = storage::read(&action_db, &tx, &receipt); + let (offset, metrics, reducer) = action_db.release_tx(tx); + action_db.report_read_tx_metrics(reducer, metrics); + result.map(|result| (offset, result)) + }) .await - .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; - Ok(Durable { - receipt, - durable_through, + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt, + durable_through, + }) }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)? } pub async fn close( @@ -97,28 +111,41 @@ async fn mutate_with_capacity( let permit = capacity .try_acquire_owned() .map_err(|_| EnvironmentSnapshotError::Capacity)?; - let (_permit, durable_through, receipt) = tokio::task::spawn_blocking(move || { - let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); - let (tx, result) = db.with_auto_rollback(tx, action)?; - let (offset, data, metrics, reducer) = db - .commit_tx(tx) - .map_err(|_| EnvironmentSnapshotError::Storage)? - .ok_or(EnvironmentSnapshotError::Storage)?; - db.report_mut_tx_metrics(reducer, metrics, Some(data)); - Ok::<_, EnvironmentSnapshotError>((permit, offset, result)) - }) - .await - .map_err(|_| EnvironmentSnapshotError::Storage)??; - durability - .wait_for(durable_through) + // A cancelled waiter drops only this JoinHandle. The actual operation + // keeps its database and finite slot until commit and durability finish. + tokio::spawn(async move { + let _permit = permit; + let action_db = db.clone(); + let (durable_through, receipt) = tokio::task::spawn_blocking(move || { + let tx = action_db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + let (tx, result) = action_db.with_auto_rollback(tx, action)?; + let (offset, data, metrics, reducer) = action_db + .commit_tx(tx) + .map_err(|_| EnvironmentSnapshotError::Storage)? + .ok_or(EnvironmentSnapshotError::Storage)?; + action_db.report_mut_tx_metrics(reducer, metrics, Some(data)); + Ok::<_, EnvironmentSnapshotError>((offset, result)) + }) .await - .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; - Ok(Durable { - receipt, - durable_through, + .map_err(|_| EnvironmentSnapshotError::Storage)??; + durability + .wait_for(durable_through) + .await + .map_err(|_| EnvironmentSnapshotError::DurabilityFailed)?; + drop(db); + Ok(Durable { + receipt, + durable_through, + }) }) + .await + .map_err(|_| EnvironmentSnapshotError::Storage)? } +#[cfg(test)] +#[path = "container_environment/durability_tests.rs"] +mod durability_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/core/src/host/container_environment/durability_tests.rs b/crates/core/src/host/container_environment/durability_tests.rs new file mode 100644 index 00000000000..c72ab9435db --- /dev/null +++ b/crates/core/src/host/container_environment/durability_tests.rs @@ -0,0 +1,288 @@ +//! Real local writes with only their durability acknowledgment delayed. +use super::*; +use crate::db::deployment::{ + install_container_fence, install_publication_fence, record_deployment_commit, DeploymentCommit, +}; +use crate::db::relational_db::{ + local_durability, + tests_utils::{TempReplicaDir, TestDB}, + LocalDurability, +}; +use crate::db::{environment, persistence::Persistence}; +use spacetimedb_datastore::system_tables::StContainerFenceRow; +use spacetimedb_durability::{Close, Durability, DurableOffset, PreparedTx}; +use spacetimedb_lib::container::*; +use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, SYSTEM_EMPTY_MODULE_VERSION}; +use spacetimedb_lib::{hash_bytes, Timestamp, Uuid}; +use std::time::Duration; + +fn uuid() -> Uuid { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) +} + +fn setup(db: &RelationalDB, keys: Vec) -> EnvironmentSnapshotScope { + let spec = ContainerSpec { + image_manifest: OciDigest::sha256([7; 32]), + image_platform: ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + argv: vec!["/app/agent".into()], + user: "1000:1000".into(), + working_directory: "/app".into(), + mode: ContainerMode::Job, + restart: RestartPolicy::Never, + env_keys: keys, + resources: ContainerResources { + cpu_millicores: 1000, + memory_bytes: 64 * 1024 * 1024, + scratch_bytes: 64 * 1024 * 1024, + pids_max: 64, + }, + ports: vec![], + mounts: vec![], + stop_grace_ms: DEFAULT_STOP_GRACE_MS, + } + .normalize(&Default::default()) + .unwrap(); + let request = DeploymentCommit { + operation_id: uuid(), + publication_epoch: 1, + publisher: db.owner_identity(), + expected_revision: None, + prepared_manifest_hash: hash_bytes(b"prepared"), + deployment: DeploymentSpec::V1(DeploymentSpecV1 { + module: ModuleComponent::SystemEmpty(SYSTEM_EMPTY_MODULE_VERSION), + container: Some(spec.clone()), + }), + }; + db.with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<_> { + install_publication_fence(tx, request.publication_epoch, request.operation_id)?; + record_deployment_commit(tx, &request, Timestamp::now(), &Default::default())?; + install_container_fence(db, tx, &self_fence(db, 1, true))?; + for key in &spec.env_keys { + environment::set(db, tx, key, "before")?; + } + Ok(()) + }) + .unwrap(); + EnvironmentSnapshotScope { + cluster: "local-test".into(), + database_id: 1, + database_identity: db.database_identity(), + node_id: 2, + node_incarnation: uuid(), + generation: 1, + deployment_revision: request.deployment.revision().unwrap(), + start_request: request.operation_id, + env_generation: uuid(), + env_keys: spec.env_keys, + } +} + +fn self_fence(db: &RelationalDB, generation: u64, allowed: bool) -> StContainerFenceRow { + StContainerFenceRow { + source_identity: db.database_identity().into(), + generation, + target_grant_revision: 1, + target_set_hash: hash_bytes(b"targets"), + allowed, + } +} + +struct DelayedAcknowledgment { + writer: LocalDurability, + acknowledged: DurableOffset, +} +impl Durability for DelayedAcknowledgment { + type TxData = crate::db::relational_db::Txdata; + fn append_tx(&self, tx: PreparedTx) { + self.writer.append_tx(tx); + } + fn durable_tx_offset(&self) -> DurableOffset { + self.acknowledged.clone() + } + fn close(&self) -> Close { + self.writer.close() + } +} + +struct Fixture { + db: Arc, + writer: LocalDurability, + acknowledge: tokio::sync::watch::Sender>, + _directory: TempReplicaDir, +} +impl Fixture { + async fn new() -> Self { + let directory = TempReplicaDir::new().unwrap(); + let (writer, disk_size) = local_durability((*directory).clone(), None).await.unwrap(); + let (acknowledge, acknowledged) = tokio::sync::watch::channel(None); + let db = Arc::new( + TestDB::open_db( + writer.as_history(), + Some(Persistence { + durability: Arc::new(DelayedAcknowledgment { + writer: writer.clone(), + acknowledged: acknowledged.into(), + }), + disk_size, + snapshots: None, + runtime: tokio::runtime::Handle::current(), + }), + None, + 0, + ) + .unwrap(), + ); + Self { + db, + writer, + acknowledge, + _directory: directory, + } + } + async fn acknowledge_current(&self) -> u64 { + let tx = self.db.begin_tx(Workload::ForTests); + let (offset, metrics, reducer) = self.db.release_tx(tx); + self.db.report_read_tx_metrics(reducer, metrics); + let mut actual = self.writer.durable_tx_offset(); + let durable = tokio::time::timeout(Duration::from_secs(5), actual.wait_for(offset)) + .await + .unwrap() + .unwrap(); + self.acknowledge.send_replace(Some(durable)); + offset + } + async fn finish(self) { + self.db.shutdown().await; + drop(self.db); + self.writer.close().await; + } +} + +async fn wait_for_slot(capacity: &Arc) { + tokio::time::timeout(Duration::from_secs(5), async { + while capacity.available_permits() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} +async fn released(capacity: &Arc) { + let permit = tokio::time::timeout(Duration::from_secs(5), capacity.clone().acquire_owned()) + .await + .unwrap() + .unwrap(); + drop(permit); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_capture_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let capacity = Arc::new(Semaphore::new(1)); + let action_db = db.clone(); + let (captured, result) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(mutate_with_capacity(db.clone(), capacity.clone(), move |tx| { + let receipt = storage::capture(&action_db, tx, &scope)?; + captured.send(receipt.clone()).unwrap(); + Ok(receipt) + })); + let captured = tokio::time::timeout(Duration::from_secs(5), result) + .await + .unwrap() + .unwrap(); + // This read waits for the actual capture transaction to commit. + db.with_read_only(Workload::ForTests, |tx| storage::read(&db, tx, &captured)) + .unwrap(); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + mutate_with_capacity(db.clone(), capacity.clone(), |_| Ok(())).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + let values = read(db.clone(), captured).await.unwrap(); + assert_eq!(values.receipt.selected_values["SECRET"], "before"); + drop(db); + fixture.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_read_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let receipt = db + .with_auto_commit(Workload::ForTests, |tx| storage::capture(&db, tx, &scope)) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let caller = tokio::spawn(read_with_capacity(db.clone(), receipt.clone(), capacity.clone())); + wait_for_slot(&capacity).await; + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + read_with_capacity(db.clone(), receipt.clone(), capacity.clone()).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + let values = read_with_capacity(db.clone(), receipt, capacity).await.unwrap(); + assert_eq!(values.receipt.selected_values["SECRET"], "before"); + drop(db); + fixture.finish().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn container_environment_close_cancellation_retains_slot_until_durable_acknowledgment() { + let fixture = Fixture::new().await; + let db = fixture.db.clone(); + let scope = setup(&db, vec!["SECRET".into()]); + let receipt = db + .with_auto_commit(Workload::ForTests, |tx| storage::capture(&db, tx, &scope)) + .unwrap(); + db.with_auto_commit(Workload::ForTests, |tx| { + install_container_fence(&db, tx, &self_fence(&db, 2, false)) + }) + .unwrap(); + let capacity = Arc::new(Semaphore::new(1)); + let action_db = db.clone(); + let (closed, result) = tokio::sync::oneshot::channel(); + let caller = tokio::spawn(mutate_with_capacity(db.clone(), capacity.clone(), move |tx| { + let receipt = storage::close(&action_db, tx, &scope, 2)?; + closed.send(()).unwrap(); + Ok(receipt) + })); + tokio::time::timeout(Duration::from_secs(5), result) + .await + .unwrap() + .unwrap(); + db.with_read_only(Workload::ForTests, |tx| { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + assert_eq!( + tx.table_row_count(spacetimedb_datastore::system_tables::ST_CONTAINER_ENVIRONMENT_ID), + Some(0) + ); + }); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + assert_eq!(capacity.available_permits(), 0); + assert!(matches!( + mutate_with_capacity(db.clone(), capacity.clone(), |_| Ok(())).await, + Err(EnvironmentSnapshotError::Capacity) + )); + fixture.acknowledge_current().await; + released(&capacity).await; + assert!(matches!( + read(db.clone(), receipt).await, + Err(EnvironmentSnapshotError::Fenced) + )); + drop(db); + fixture.finish().await; +} From 5cb24484585979166e9a040c61bef88d1bb24c30 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 06:38:56 -0400 Subject: [PATCH 13/23] Publish managed deployments with durable CLI recovery --- Cargo.lock | 4 + crates/cli/Cargo.toml | 3 + crates/cli/docs/container-build.md | 85 +- crates/cli/src/container/mod.rs | 3 +- crates/cli/src/container/publish/client.rs | 428 ++++++++ crates/cli/src/container/publish/journal.rs | 336 ++++++ crates/cli/src/container/publish/mod.rs | 315 ++++++ crates/cli/src/container/publish/tests.rs | 574 +++++++++++ crates/cli/src/container/tests.rs | 13 +- crates/cli/src/spacetime_config.rs | 16 + crates/cli/src/subcommands/dev.rs | 4 + crates/cli/src/subcommands/publish.rs | 60 +- crates/cli/src/subcommands/publish/managed.rs | 955 ++++++++++++++++++ 13 files changed, 2764 insertions(+), 32 deletions(-) create mode 100644 crates/cli/src/container/publish/client.rs create mode 100644 crates/cli/src/container/publish/journal.rs create mode 100644 crates/cli/src/container/publish/mod.rs create mode 100644 crates/cli/src/container/publish/tests.rs create mode 100644 crates/cli/src/subcommands/publish/managed.rs diff --git a/Cargo.lock b/Cargo.lock index bc61994dfc8..4a38b122d66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7820,6 +7820,7 @@ name = "spacetimedb-cli" version = "2.3.0" dependencies = [ "anyhow", + "axum", "base64 0.21.7", "bytes", "cargo_metadata", @@ -7890,6 +7891,8 @@ dependencies = [ "toml 0.8.23", "toml_edit 0.22.27", "tracing", + "url", + "uuid", "walkdir", "wasmbin", "webbrowser", @@ -10270,6 +10273,7 @@ checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.4", "js-sys", + "serde", "wasm-bindgen", ] diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index acd1414422d..60e000ed4c6 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -76,6 +76,8 @@ tokio-tungstenite.workspace = true toml.workspace = true toml_edit.workspace = true tracing = { workspace = true, features = ["release_max_level_off"] } +url.workspace = true +uuid = { workspace = true, features = ["std", "serde", "v7"] } walkdir.workspace = true wasmbin.workspace = true webbrowser.workspace = true @@ -94,6 +96,7 @@ notify.workspace = true path-clean = "1.0.1" [dev-dependencies] +axum.workspace = true pretty_assertions.workspace = true fs_extra.workspace = true diff --git a/crates/cli/docs/container-build.md b/crates/cli/docs/container-build.md index 643a488dc08..71749eda520 100644 --- a/crates/cli/docs/container-build.md +++ b/crates/cli/docs/container-build.md @@ -9,10 +9,8 @@ The command is dispatched before saved CLI server settings or credentials are opened; only project configuration and explicitly selected build credentials are read. -This slice implements local preparation. Managed publication and container -lifecycle commands are separate integration work. The existing `publish` command -rejects a selected container declaration so it cannot silently publish only the -module. +The same verified preparation feeds managed `publish`, described below. +Container lifecycle commands remain separate integration work. ## Configuration @@ -147,3 +145,82 @@ Tests use generated local OCI fixtures and fake tool invocations, including an owned shell fixture for process cleanup. They do not execute Docker, BuildKit, Railpack, Skopeo, or any server operation. Actual supported-builder acceptance remains a separate integration check. + +## Managed publication + +`spacetime publish` publishes a selected target's container declaration through +managed publication. Select the image platform explicitly. Server selection uses +the normal CLI URL, configured alias, or default: + +```sh +spacetime publish my-db --server https://your-test-server.example \ + --container-platform linux/amd64 \ + --artifact-endpoint https://your-test-artifacts.example +``` + +The URLs above are placeholders. Managed transport requires HTTPS for remote +servers and also supports HTTP loopback servers. The CLI does not send the publisher's Bearer credential to an +advertised artifact origin unless it is the same origin as the selected server +or the exact URL is explicitly approved with `--artifact-endpoint`. It does not +follow HTTP redirects or inherit HTTP proxy settings for managed publication. +Image registry credentials remain separate and require `--registry-auth-file`. + +A target with a container declaration and no module source preserves its existing +module. A new container-only database uses the immutable versioned empty module. +An explicit `module-path`, `bin-path`, or `js-path` replaces the module. An omitted +container preserves it; `--remove-container` removes it. `--remove-module` selects +the empty module and runs the existing authorized migration preflight. A selected +container declaration conflicts with `--remove-container`, and a configured +module source conflicts with `--remove-module`. Manual migrations and data-clear +publication are rejected. Precompiled NativeAOT modules can use `--bin-path`; +managed source compilation with `--native-aot` is not yet supported. + +Ordinary module-only publications use the legacy path. When deployment inspection +finds an existing managed revision, module-only updates use managed publication +with `ContainerAction::Keep`. `--managed` explicitly selects managed publication +for a new module-only deployment. A managed error never falls back to a raw +module publication. Existing databases use their exact current deployment +revision as a compare-and-set precondition. + +New managed databases first reserve a server-generated Identity under the +publisher and creation options. A caller cannot select an unreserved new Identity. +`--parent` resolves an accessible existing database; `--organization` currently +requires the organization's Identity. Requested database naming runs separately +after activation. If naming fails or its response is lost, the command reports +the successfully created Identity and does not repeat publication or overwrite +names on a later resume. + +Before staging artifacts, the CLI saves a private operation directory under +`.spacetime/publications/` beside the project configuration, or the explicit +`--publication-state-dir`. The directory retains exact request bytes, immutable +module/OCI artifacts and upload receipts. It contains no publisher credential or +resolved database environment values. Image blobs can contain environment +values baked into the image, so treat the directory as private build output. + +If a request fails or the command is interrupted, use the printed directory: + +```sh +spacetime publish --resume-publication .spacetime/publications/OPERATION_UUID \ + --server https://your-test-server.example \ + --artifact-endpoint https://your-test-artifacts.example +``` + +Resume authenticates the original publisher and server, observes accepted state, +and reuses the exact operation bytes. It does not read `spacetime.json`, rebuild, +or resolve a changed image tag. Ambiguous uploads query the same upload receipt +before continuing. If current database-read access was revoked, resume uses the +exact stored PUT to recover the original publisher's admitted result before +requiring local artifacts. A lost publication response never creates a second +operation. The coordinator PUT has a bounded 30-minute timeout covering its +separate schema, image, storage, and confirmation steps; interruption retains +the journal while the outcome is uncertain. Exact public PUT replay remains +limited to the operation's seven-day retry window; server-side recovery has its +own durable lifetime. +The default activation wait is 60 seconds; `--publication-wait 0` returns after +the first confirmed status, and a pending status prints its resume instruction. +Keep the directory while an outcome is uncertain; a confirmed terminal operation +can be removed locally when its artifacts are no longer needed. + +Container declarations remain local to their exact database target. They do not +inherit to nested targets. `spacetime dev` currently rejects selected container +targets because it does not yet supervise their local runtime. diff --git a/crates/cli/src/container/mod.rs b/crates/cli/src/container/mod.rs index 056f7d1f545..02af7005921 100644 --- a/crates/cli/src/container/mod.rs +++ b/crates/cli/src/container/mod.rs @@ -2,9 +2,10 @@ pub mod config; pub mod oci; pub mod process; +pub mod publish; #[cfg(test)] -mod tests; +pub(crate) mod tests; use anyhow::{ensure, Context, Result}; use config::{ContainerConfig, ImageSource, SourceBuild}; diff --git a/crates/cli/src/container/publish/client.rs b/crates/cli/src/container/publish/client.rs new file mode 100644 index 00000000000..819085c8613 --- /dev/null +++ b/crates/cli/src/container/publish/client.rs @@ -0,0 +1,428 @@ +//! Ordinary publisher transport. Credentials are held in memory and never +//! redirected, inherited from a proxy, or sent to an unapproved artifact URL. +use anyhow::{bail, ensure, Context, Result}; +use reqwest::{header::HeaderValue, Client, Method, StatusCode, Url}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use spacetimedb_client_api_messages::name::{DomainName, PrePublishResult, SetDomainsResult}; +use spacetimedb_lib::{container::OciDigest, deployment::api::*, Identity, Uuid}; +use std::time::Duration; + +pub const MAX_RESPONSE_BYTES: usize = 1024 * 1024; +// A submission composes the 120s schema worker, 300s image verification, +// two independently bounded 300s artifact writes, and read/pin/control calls. +// This caller deadline does not extend those server resource or credential +// bounds. Cancellation/timeout leaves the exact operation in its journal. +const COORDINATOR_REQUEST_TIMEOUT: Duration = Duration::from_secs(30 * 60); +pub const UPLOAD_CHUNK_BYTES: usize = 8 * 1024 * 1024; + +#[derive(Debug, thiserror::Error)] +#[error("{action} returned HTTP {status}")] +pub struct HttpFailure { + pub action: &'static str, + pub status: StatusCode, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UploadKind { + Manifest, + Config, + Layer, + Module, +} +impl UploadKind { + fn header(self) -> &'static str { + match self { + Self::Manifest => "manifest", + Self::Config => "config", + Self::Layer => "layer", + Self::Module => "module", + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObjectRef { + pub digest: OciDigest, + pub size: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UploadStatus { + pub id: uuid::Uuid, + pub object: ObjectRef, + pub offset: u64, + pub expires_at: u64, + pub complete: bool, +} +impl UploadStatus { + pub fn validate(&self, object: ObjectRef, id: Option) -> Result<()> { + ensure!( + self.id.get_version_num() == 4 && id.is_none_or(|id| self.id == id), + "artifact upload session changed" + ); + ensure!( + self.object == object && self.offset <= object.size && (!self.complete || self.offset == object.size), + "artifact upload descriptor or offset changed" + ); + Ok(()) + } +} + +pub struct ArtifactEndpoint(Url); +impl ArtifactEndpoint { + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +pub struct PublisherClient { + http: Client, + server: Url, + authorization: HeaderValue, +} +impl PublisherClient { + pub fn new(server: &str, mut authorization: HeaderValue) -> Result { + let server = endpoint(server)?; + ensure!( + authorization.as_bytes().starts_with(b"Bearer "), + "managed publication requires ordinary Bearer authentication" + ); + authorization.set_sensitive(true); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { + http, + server, + authorization, + }) + } + pub fn server(&self) -> &Url { + &self.server + } + fn request(&self, method: Method, url: Url) -> reqwest::RequestBuilder { + self.http + .request(method, url) + .header(reqwest::header::AUTHORIZATION, self.authorization.clone()) + } + /// A response cannot grant itself permission to receive this credential. + /// Same-origin paths are accepted; another origin requires an explicit URL. + pub fn artifact_endpoint(&self, advertised: &str, approved: Option<&str>) -> Result { + let artifact = endpoint(advertised)?; + if artifact.origin() != self.server.origin() { + let approved=approved.context("artifact service uses another origin; pass --artifact-endpoint with its exact trusted URL before sending publisher credentials")?; + ensure!( + endpoint(approved)? == artifact, + "advertised artifact endpoint differs from the explicitly approved endpoint" + ); + } else if let Some(approved) = approved { + ensure!( + endpoint(approved)? == artifact, + "advertised artifact endpoint differs from the explicitly approved endpoint" + ); + } + Ok(ArtifactEndpoint(artifact)) + } + pub async fn capabilities(&self) -> Result> { + let response = self + .http + .get(route(&self.server, &["v1", "containers", "capabilities"])?) + .send() + .await?; + optional_json(response, "publication capabilities").await + } + pub async fn permission(&self) -> Result { + json( + self.request( + Method::GET, + route(&self.server, &["v1", "containers", "publish-permission"])?, + ) + .send() + .await?, + "publication permission", + ) + .await + } + pub async fn deployment(&self, name: &str) -> Result> { + optional_json( + self.request( + Method::GET, + route(&self.server, &["v1", "database", name, "deployment"])?, + ) + .send() + .await?, + "deployment inspection", + ) + .await + } + pub async fn reserve(&self, request: &ReserveDatabaseRequest) -> Result { + json( + self.request( + Method::POST, + route(&self.server, &["v1", "containers", "reservations"])?, + ) + .json(request) + .send() + .await?, + "database reservation", + ) + .await + } + pub async fn submit(&self, database: Identity, request: &PublishRequest) -> Result { + let bytes = serde_json::to_vec(request)?; + self.submit_bytes(database, &bytes).await + } + /// Resume sends these same immutable bytes, without reconstructing a request + /// from a changed tag, project configuration, or current deployment. + pub async fn submit_bytes(&self, database: Identity, bytes: &[u8]) -> Result { + ensure!(bytes.len() <= 256 * 1024, "publication HTTP body exceeds 256 KiB"); + json( + self.request( + Method::PUT, + route(&self.server, &["v1", "database", &database.to_hex(), "deployment"])?, + ) + .timeout(COORDINATOR_REQUEST_TIMEOUT) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(bytes.to_vec()) + .send() + .await?, + "publication submission", + ) + .await + } + pub async fn status(&self, database: Identity, operation: Uuid) -> Result> { + optional_json( + self.request( + Method::GET, + route( + &self.server, + &[ + "v1", + "database", + &database.to_hex(), + "deployment", + "operations", + &operation.to_string(), + ], + )?, + ) + .send() + .await?, + "publication status", + ) + .await + } + pub async fn preflight(&self, database: Identity, kind: &str, bytes: &[u8]) -> Result { + json( + self.request( + Method::POST, + route(&self.server, &["v1", "database", &database.to_hex(), "pre_publish"])?, + ) + .query(&[("host_type", kind), ("pretty_print_style", "NoColor")]) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(bytes.to_vec()) + .send() + .await?, + "module migration preflight", + ) + .await + } + pub async fn set_name(&self, database: Identity, name: &str) -> Result<()> { + let name: DomainName = name.parse()?; + let result: SetDomainsResult = json( + self.request( + Method::PUT, + route(&self.server, &["v1", "database", &database.to_hex(), "names"])?, + ) + .json(&[name]) + .send() + .await?, + "database naming", + ) + .await?; + ensure!( + matches!(result, SetDomainsResult::Success), + "database was created, but assigning its requested name was not confirmed" + ); + Ok(()) + } + pub async fn begin_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + kind: UploadKind, + object: ObjectRef, + ) -> Result { + let status: UploadStatus = json( + self.request( + Method::POST, + route(&artifact.0, &["v1", "databases", &database.to_hex(), "uploads"])?, + ) + .header("x-spacetimedb-artifact-kind", kind.header()) + .json(&serde_json::json!({"kind":kind,"object":object})) + .send() + .await?, + "artifact upload creation", + ) + .await?; + status.validate(object, None)?; + Ok(status) + } + pub async fn upload_status( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + id: uuid::Uuid, + object: ObjectRef, + ) -> Result { + let status: UploadStatus = json( + self.request( + Method::GET, + route( + &artifact.0, + &["v1", "databases", &database.to_hex(), "uploads", &id.to_string()], + )?, + ) + .send() + .await?, + "artifact upload status", + ) + .await?; + status.validate(object, Some(id))?; + Ok(status) + } + pub async fn append_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + status: &UploadStatus, + chunk: Vec, + ) -> Result { + ensure!( + !chunk.is_empty() + && chunk.len() <= UPLOAD_CHUNK_BYTES + && status + .offset + .checked_add(chunk.len() as u64) + .is_some_and(|end| end <= status.object.size), + "invalid artifact chunk" + ); + let next: UploadStatus = json( + self.request( + Method::PATCH, + route( + &artifact.0, + &["v1", "databases", &database.to_hex(), "uploads", &status.id.to_string()], + )?, + ) + .query(&[("offset", status.offset)]) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(chunk) + .send() + .await?, + "artifact upload append", + ) + .await?; + next.validate(status.object, Some(status.id))?; + Ok(next) + } + pub async fn complete_upload( + &self, + artifact: &ArtifactEndpoint, + database: Identity, + status: &UploadStatus, + ) -> Result { + let next: UploadStatus = json( + self.request( + Method::POST, + route( + &artifact.0, + &[ + "v1", + "databases", + &database.to_hex(), + "uploads", + &status.id.to_string(), + "complete", + ], + )?, + ) + .send() + .await?, + "artifact upload completion", + ) + .await?; + next.validate(status.object, Some(status.id))?; + ensure!(next.complete, "artifact completion not confirmed"); + Ok(next) + } +} + +pub(crate) fn is_loopback(url: &Url) -> bool { + match url.host() { + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + Some(url::Host::Domain(domain)) => domain.eq_ignore_ascii_case("localhost"), + None => false, + } +} +pub fn endpoint(value: &str) -> Result { + let mut url = Url::parse(value).context("resolved server URL must use HTTP(S)")?; + ensure!( + url.username().is_empty() && url.password().is_none() && url.query().is_none() && url.fragment().is_none(), + "endpoint must not contain credentials, query, or fragment" + ); + let local = is_loopback(&url); + ensure!( + url.scheme() == "https" || (url.scheme() == "http" && local), + "use HTTPS, or HTTP for a local loopback server" + ); + ensure!(url.host_str().is_some(), "endpoint host is missing"); + if !url.path().ends_with('/') { + let path = format!("{}/", url.path()); + url.set_path(&path); + } + Ok(url) +} +fn route(base: &Url, segments: &[&str]) -> Result { + let mut url = base.clone(); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid base URL"))? + .pop_if_empty() + .extend(segments); + Ok(url) +} +async fn optional_json(response: reqwest::Response, action: &'static str) -> Result> { + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + json(response, action).await.map(Some) +} +async fn json(mut response: reqwest::Response, action: &'static str) -> Result { + if !response.status().is_success() { + return Err(HttpFailure { + action, + status: response.status(), + } + .into()); + } + if response + .content_length() + .is_some_and(|length| length > MAX_RESPONSE_BYTES as u64) + { + bail!("{action} response exceeds its bound"); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await? { + ensure!( + chunk.len() <= MAX_RESPONSE_BYTES.saturating_sub(bytes.len()), + "{action} response exceeds its bound" + ); + bytes.extend_from_slice(&chunk); + } + serde_json::from_slice(&bytes).with_context(|| format!("invalid {action} response")) +} diff --git a/crates/cli/src/container/publish/journal.rs b/crates/cli/src/container/publish/journal.rs new file mode 100644 index 00000000000..adb8f9521d8 --- /dev/null +++ b/crates/cli/src/container/publish/journal.rs @@ -0,0 +1,336 @@ +//! A publication resume record contains exact request bytes and object +//! descriptors, never credentials or resolved runtime environment values. +use super::client::{ObjectRef, UploadKind, UploadStatus}; +use crate::container::{oci, PreparedContainer}; +use anyhow::{ensure, Context, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use spacetimedb_lib::{ + container::OciDigest, + deployment::{api::*, PUBLISH_PROTOCOL_VERSION}, + Identity, Uuid, +}; +use std::{ + fs::{self, File}, + io::{Read, Write}, + path::{Path, PathBuf}, + time::Instant, +}; +use tokio_util::sync::CancellationToken; + +const MAX_RECORD_BYTES: usize = 1024 * 1024; +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct UploadRecord { + pub kind: UploadKind, + pub object: ObjectRef, + pub session: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Record { + pub version: u32, + pub server: String, + pub artifact_endpoint: String, + pub publisher: Identity, + pub database: Option, + pub reservation: Option, + pub requested_name: Option, + /// UTF-8 JSON sent verbatim on every submission attempt. + pub request_json: String, + pub request_digest: OciDigest, + pub uploads: Vec, + pub submitted: bool, + pub status: Option, + pub name_confirmed: bool, + pub naming_attempted: bool, +} +impl Record { + pub fn request(&self) -> Result { + ensure!( + self.version == 1 && self.request_json.len() <= 256 * 1024, + "unsupported or oversized publication resume record" + ); + ensure!( + spacetimedb_oci::sha256(self.request_json.as_bytes()) == self.request_digest, + "publication request bytes changed" + ); + let request: PublishRequest = serde_json::from_str(&self.request_json)?; + let envelope = &request.manifest.current().envelope; + ensure!( + envelope.version == PUBLISH_PROTOCOL_VERSION, + "unsupported publication protocol" + ); + ensure!( + envelope.operation_id.get_version() == Some(spacetimedb_lib::sats::uuid::Version::V7), + "publication operation must be UUIDv7" + ); + ensure!(self.uploads.len() <= 259, "too many retained publication objects"); + if let Some(reservation) = &self.reservation { + ensure!( + reservation.operation_id == envelope.operation_id + && reservation.version == PUBLISH_PROTOCOL_VERSION + && request.creation.as_ref() == Some(&reservation.options), + "reservation differs from the immutable publication request" + ); + } else { + ensure!( + self.database.is_some() && request.creation.is_none(), + "publication database binding is missing" + ); + } + for upload in &self.uploads { + ensure!(upload.object.size > 0, "empty publication artifact"); + if let Some(status) = &upload.session { + status.validate(upload.object, None)?; + } + } + if let Some(status) = &self.status { + self.check_status(&request, status)?; + } + Ok(request) + } + pub fn check_status(&self, request: &PublishRequest, status: &PublicationStatus) -> Result<()> { + let envelope = &request.manifest.current().envelope; + ensure!( + Some(status.database_identity) == self.database + && status.operation_id == envelope.operation_id + && status.expected_revision == envelope.expected_revision + && status.proposed_revision == request.manifest.current().deployment.revision()?, + "publication response belongs to another operation or deployment" + ); + Ok(()) + } +} + +pub struct Journal { + directory: PathBuf, + _lock: File, + pub record: Record, +} +struct IncompleteDirectory(Option); +impl Drop for IncompleteDirectory { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + let _ = fs::remove_dir_all(path); + } + } +} +impl Journal { + pub fn create( + base: &Path, + record: Record, + image: Option, + module: Option<&[u8]>, + ) -> Result { + let request = record.request()?; + let has_image = image.is_some(); + let directory = base.join(request.manifest.current().envelope.operation_id.to_string()); + fs::create_dir_all(base)?; + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder + .create(&directory) + .context("publication resume directory already exists; resume it rather than reusing its operation")?; + let mut incomplete = IncompleteDirectory(Some(directory.clone())); + let lock = Self::lock(&directory, true)?; + if let Some(image) = image { + image.persist(&directory.join("image"))?; + } + if let Some(module) = module { + let mut file = File::options() + .write(true) + .create_new(true) + .open(directory.join("module.blob"))?; + file.write_all(module)?; + file.sync_all()?; + } + let journal = Self { + directory, + _lock: lock, + record, + }; + journal.sync_retained_artifacts(has_image)?; + journal.save()?; + sync_directory_chain(&journal.directory)?; + incomplete.0 = None; + Ok(journal) + } + pub fn open(directory: &Path) -> Result { + let directory = directory + .canonicalize() + .context("publication resume directory not found")?; + let lock = Self::lock(&directory, false)?; + let path = directory.join("publication.json"); + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "publication resume record must be a regular file" + ); + let mut bytes = vec![]; + File::open(path)? + .take(MAX_RECORD_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + ensure!( + bytes.len() <= MAX_RECORD_BYTES, + "publication resume record is too large" + ); + let record: Record = serde_json::from_slice(&bytes)?; + record.request()?; + // Creation saves JSON only after artifact flush. Reconfirm the parent + // link if its creator stopped before finishing that last barrier. + sync_directory_chain(&directory)?; + Ok(Self { + directory, + _lock: lock, + record, + }) + } + fn sync_retained_artifacts(&self, has_image: bool) -> Result<()> { + let mut files = self + .record + .uploads + .iter() + .map(|upload| self.object_path(upload)) + .collect::>(); + if has_image { + files.extend( + ["oci-layout", "index.json", "prepared.json"].map(|name| self.directory.join("image").join(name)), + ); + } + let mut directories = std::collections::BTreeSet::new(); + for path in files { + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "retained publication artifact must be a regular file" + ); + // Write access also lets Windows flush an owned immutable artifact. + File::options().read(true).write(true).open(&path)?.sync_all()?; + let mut parent = path.parent(); + while let Some(path) = parent { + directories.insert(path.to_owned()); + if path == self.directory { + break; + } + parent = path.parent(); + } + } + // Flush children before the directories that link them. + for directory in directories.into_iter().rev() { + sync_directory(&directory)?; + } + Ok(()) + } + fn lock(directory: &Path, create: bool) -> Result { + let file = File::options() + .read(true) + .write(true) + .create_new(create) + .open(directory.join("publication.lock"))?; + file.try_lock() + .context("another process is using this publication resume directory")?; + Ok(file) + } + pub fn directory(&self) -> &Path { + &self.directory + } + pub fn operation(&self) -> Result { + Ok(self.record.request()?.manifest.current().envelope.operation_id) + } + pub fn save(&self) -> Result<()> { + self.record.request()?; + let bytes = serde_json::to_vec_pretty(&self.record)?; + ensure!( + bytes.len() <= MAX_RECORD_BYTES, + "publication resume record is too large" + ); + let mut file = tempfile::NamedTempFile::new_in(&self.directory)?; + file.write_all(&bytes)?; + file.as_file().sync_all()?; + file.persist(self.directory.join("publication.json"))?; + #[cfg(unix)] + File::open(&self.directory)?.sync_all()?; + Ok(()) + } + pub fn object_path(&self, upload: &UploadRecord) -> PathBuf { + match upload.kind { + UploadKind::Module => self.directory.join("module.blob"), + _ => self.directory.join("image/blobs/sha256").join( + upload + .object + .digest + .to_string() + .strip_prefix("sha256:") + .expect("SHA256"), + ), + } + } + /// Reopening state never trusts stale local object bytes. This repeats only + /// their digest check; server preparation remains the admission authority. + pub async fn verify_objects(&self, cancel: CancellationToken) -> Result<()> { + let permit = crate::container::VERIFIERS + .clone() + .try_acquire_owned() + .context("two OCI verifications are already running")?; + let objects = self + .record + .uploads + .iter() + .map(|upload| (self.object_path(upload), upload.object)) + .collect::>(); + tokio::task::spawn_blocking(move || { + let _permit = permit; + let deadline = Instant::now() + crate::container::VERIFY_TIMEOUT; + for (path, object) in objects { + oci::check(&cancel, deadline)?; + ensure!( + fs::symlink_metadata(&path)?.is_file(), + "retained publication artifact must be a regular file" + ); + let mut file = File::open(path)?; + ensure!(file.metadata()?.len() == object.size, "retained artifact size changed"); + let mut hash = Sha256::new(); + let mut total = 0u64; + let mut buffer = [0u8; 64 * 1024]; + loop { + oci::check(&cancel, deadline)?; + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + total = total.checked_add(read as u64).context("artifact size overflow")?; + ensure!(total <= object.size, "retained artifact grew"); + hash.update(&buffer[..read]); + } + ensure!( + total == object.size && OciDigest::sha256(hash.finalize().into()) == object.digest, + "retained publication artifact digest changed" + ); + } + Ok::<_, anyhow::Error>(()) + }) + .await + .context("publication verification worker stopped")??; + Ok(()) + } +} + +fn sync_directory(directory: &Path) -> Result<()> { + #[cfg(unix)] + File::open(directory)?.sync_all()?; + // As in paths::utils::write_atomic, Windows directory handles cannot be + // synced through std. File flushes still precede publishing the journal. + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} +fn sync_directory_chain(directory: &Path) -> Result<()> { + let canonical = directory.canonicalize()?; + for ancestor in canonical.ancestors() { + sync_directory(ancestor)?; + } + Ok(()) +} diff --git a/crates/cli/src/container/publish/mod.rs b/crates/cli/src/container/publish/mod.rs new file mode 100644 index 00000000000..35ac356322e --- /dev/null +++ b/crates/cli/src/container/publish/mod.rs @@ -0,0 +1,315 @@ +//! Managed publication uses one immutable operation and a durable local resume +//! directory. Ambiguous HTTP responses never select a new operation or fall +//! back to the legacy module publication endpoint. +pub mod client; +pub mod journal; + +#[cfg(test)] +pub(crate) mod tests; + +use anyhow::{bail, ensure, Context, Result}; +use client::{ArtifactEndpoint, PublisherClient, UploadStatus, UPLOAD_CHUNK_BYTES}; +use journal::Journal; +use spacetimedb_lib::deployment::{ + api::{PublicationPhase, PublicationStatus}, + operation_expiry_ms, +}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio_util::sync::CancellationToken; + +#[derive(Debug)] +pub enum Outcome { + Complete(PublicationStatus), + Pending(PublicationStatus), + Aborted(PublicationStatus), + /// Database creation succeeded. Never describe a naming failure as a + /// rollback or send another publication operation to compensate for it. + NamingUnconfirmed(PublicationStatus), +} + +pub async fn run( + client: &PublisherClient, + journal: &mut Journal, + approved_artifact: Option<&str>, + wait: Duration, + cancel: CancellationToken, +) -> Result { + let cancel = cancel.child_token(); + let _cancel = crate::container::CancelOnDrop(cancel.clone()); + ensure!( + client.server().as_str() == journal.record.server, + "resume server differs from the original publication endpoint" + ); + let artifact = client.artifact_endpoint(&journal.record.artifact_endpoint, approved_artifact)?; + let request = journal.record.request()?; + let operation = request.manifest.current().envelope.operation_id; + let permission = client.permission().await?; + ensure!( + permission.identity == journal.record.publisher, + "resume publisher differs from the original publication identity" + ); + + // Observe an already admitted operation before requiring original local + // object files or applying today's new-publication permission/limits. + if journal.record.submitted + && let Some(database) = journal.record.database + { + match observe_or_replay(client, journal, database, operation).await { + Ok(Some(status)) => { + journal.record.check_status(&request, &status)?; + journal.record.status = Some(status); + journal.save()?; + return wait_and_name(client, journal, wait, &cancel).await; + } + Ok(None) => (), + Err(error) => { + return Err(error).context("publication observation was not confirmed; resume the same directory") + } + } + } + let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis())?; + operation_expiry_ms(operation, now)?; + ensure!( + !cancel.is_cancelled(), + "publication cancelled; resume directory retained" + ); + if journal.record.database.is_none() { + let reservation = client + .reserve( + journal + .record + .reservation + .as_ref() + .context("creation reservation is missing")?, + ) + .await?; + ensure!( + reservation.operation_id == operation && reservation.staging_open, + "reservation was not confirmed for this operation" + ); + let returned = client.artifact_endpoint(&reservation.artifact_endpoint, approved_artifact)?; + ensure!( + returned.as_str() == artifact.as_str(), + "reservation changed the selected artifact endpoint" + ); + journal.record.database = Some(reservation.database_identity); + journal.save()?; + } + journal.verify_objects(cancel.clone()).await?; + for index in 0..journal.record.uploads.len() { + upload(client, &artifact, journal, index, &cancel).await?; + } + ensure!( + !cancel.is_cancelled(), + "publication cancelled; resume directory retained" + ); + // This marker must be durable before sending the admission request. A crash + // after the server commits but before the response can then observe/replay. + journal.record.submitted = true; + journal.save()?; + let status = client + .submit_bytes(journal.record.database.unwrap(), journal.record.request_json.as_bytes()) + .await + .context("publication outcome is not confirmed; resume the same directory")?; + journal.record.check_status(&request, &status)?; + journal.record.status = Some(status); + journal.save()?; + wait_and_name(client, journal, wait, &cancel).await +} + +/// Operation inspection has current database-read authorization. Exact PUT +/// replay instead authenticates the original admitted publisher and immutable +/// request. It must precede local artifact/expiry work after read revocation. +async fn observe_or_replay( + client: &PublisherClient, + journal: &Journal, + database: spacetimedb_lib::Identity, + operation: spacetimedb_lib::Uuid, +) -> Result> { + match client.status(database, operation).await { + Err(error) + if error + .downcast_ref::() + .is_some_and(|error| error.status == reqwest::StatusCode::FORBIDDEN) => + { + client + .submit_bytes(database, journal.record.request_json.as_bytes()) + .await + .map(Some) + .context("exact publication replay was not confirmed; keep the same resume directory") + } + result => result, + } +} + +async fn upload( + client: &PublisherClient, + artifact: &ArtifactEndpoint, + journal: &mut Journal, + index: usize, + cancel: &CancellationToken, +) -> Result<()> { + let database = journal.record.database.context("reservation missing")?; + let item = journal.record.uploads[index].clone(); + let mut status = if let Some(prior) = item.session { + match client.upload_status(artifact, database, prior.id, item.object).await { + Ok(status) => status, + Err(error) + if error.downcast_ref::().is_some_and(|error| { + matches!(error.status, reqwest::StatusCode::NOT_FOUND | reqwest::StatusCode::GONE) + }) => + { + client.begin_upload(artifact, database, item.kind, item.object).await? + } + Err(error) => return Err(error), + } + } else { + client.begin_upload(artifact, database, item.kind, item.object).await? + }; + store_upload(journal, index, &status)?; + if status.complete { + return Ok(()); + } + let mut file = tokio::fs::File::open(journal.object_path(&journal.record.uploads[index])).await?; + let mut ambiguous_attempts = 0; + while status.offset < status.object.size { + ensure!( + !cancel.is_cancelled(), + "artifact upload cancelled; resume directory retained" + ); + file.seek(std::io::SeekFrom::Start(status.offset)).await?; + let len = usize::try_from((status.object.size - status.offset).min(UPLOAD_CHUNK_BYTES as u64))?; + let mut bytes = vec![0; len]; + file.read_exact(&mut bytes) + .await + .context("retained artifact was truncated")?; + let expected = status.offset + len as u64; + match client.append_upload(artifact, database, &status, bytes).await { + Ok(next) => { + ensure!( + next.offset == expected, + "artifact append acknowledged an unexpected offset" + ); + status = next; + ambiguous_attempts = 0; + } + Err(error) => { + if !retryable(&error) { + return Err(error); + } + ambiguous_attempts += 1; + ensure!( + ambiguous_attempts <= 3, + "artifact append outcome remains unknown; resume the same directory" + ); + let next = client + .upload_status(artifact, database, status.id, status.object) + .await?; + ensure!( + next.offset == status.offset || next.offset == expected, + "artifact offset changed outside this append" + ); + status = next; + } + } + store_upload(journal, index, &status)?; + } + status = match client.complete_upload(artifact, database, &status).await { + Ok(status) => status, + Err(error) if retryable(&error) => { + let observed = client + .upload_status(artifact, database, status.id, status.object) + .await?; + if observed.complete { + observed + } else { + return Err(error).context("artifact completion is not confirmed; resume the same directory"); + } + } + Err(error) => return Err(error), + }; + store_upload(journal, index, &status) +} +fn store_upload(journal: &mut Journal, index: usize, status: &UploadStatus) -> Result<()> { + journal.record.uploads[index].session = Some(status.clone()); + journal.save() +} +fn retryable(error: &anyhow::Error) -> bool { + error + .downcast_ref::() + .is_some_and(|error| error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()) + || error.downcast_ref::().is_some_and(|error| { + error.status.is_server_error() + || matches!( + error.status, + reqwest::StatusCode::CONFLICT | reqwest::StatusCode::TOO_MANY_REQUESTS + ) + }) +} +async fn wait_and_name( + client: &PublisherClient, + journal: &mut Journal, + wait: Duration, + cancel: &CancellationToken, +) -> Result { + let deadline = tokio::time::Instant::now() + wait; + let request = journal.record.request()?; + loop { + let status = journal + .record + .status + .clone() + .context("confirmed publication status missing")?; + match status.phase { + PublicationPhase::AbortedBeforeCommit => return Ok(Outcome::Aborted(status)), + PublicationPhase::Complete => { + if let Some(name) = journal + .record + .requested_name + .clone() + .filter(|_| !journal.record.name_confirmed) + { + // Existing name replacement has no CAS token. Do not retry an + // ambiguous name update and overwrite aliases added later. + if journal.record.naming_attempted { + return Ok(Outcome::NamingUnconfirmed(status)); + } + journal.record.naming_attempted = true; + journal.save()?; + if client.set_name(status.database_identity, &name).await.is_err() { + return Ok(Outcome::NamingUnconfirmed(status)); + } + journal.record.name_confirmed = true; + journal.save()?; + } + return Ok(Outcome::Complete(status)); + } + _ => (), + } + if tokio::time::Instant::now() >= deadline { + return Ok(Outcome::Pending(status)); + } + tokio::select! { + _=cancel.cancelled()=>bail!("publication wait cancelled; its durable operation continues, resume the same directory"), + _=tokio::time::sleep_until((tokio::time::Instant::now()+Duration::from_secs(1)).min(deadline))=>(), + } + let next = tokio::time::timeout_at( + deadline, + observe_or_replay(client, journal, status.database_identity, status.operation_id), + ) + .await; + match next { + Ok(Ok(Some(next))) => { + journal.record.check_status(&request, &next)?; + journal.record.status = Some(next); + journal.save()?; + } + Ok(Ok(None)) => bail!("confirmed publication disappeared; keep the same operation and resume directory"), + Ok(Err(error)) => { + return Err(error).context("publication status is not confirmed; resume the same directory") + } + Err(_) => return Ok(Outcome::Pending(status)), + } + } +} diff --git a/crates/cli/src/container/publish/tests.rs b/crates/cli/src/container/publish/tests.rs new file mode 100644 index 00000000000..e47aa9b53bd --- /dev/null +++ b/crates/cli/src/container/publish/tests.rs @@ -0,0 +1,574 @@ +//! These fixtures own numeric-loopback sockets and in-memory credentials. They +//! never load CLI configuration, environment endpoints, or external builders. +use super::*; +use axum::{ + body::Bytes, + extract::State, + http::{HeaderMap, Method, StatusCode, Uri}, + response::{IntoResponse, Response}, + Json, Router, +}; +use client::{ObjectRef, UploadKind, UploadStatus}; +use journal::{Record, UploadRecord}; +use serde_json::json; +use spacetimedb_lib::{ + deployment::{api::*, manifest::*, *}, + Identity, Uuid, +}; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +const TOKEN: &str = "Bearer isolated-fixture-credential"; +pub(crate) fn publisher() -> Identity { + Identity::from_be_byte_array([42; 32]) +} +pub(crate) fn database() -> Identity { + Identity::from_be_byte_array([43; 32]) +} +#[derive(Default)] +pub(crate) struct Behavior { + pub permission: bool, + pub lose_append: bool, + pub lose_submit_after_commit: bool, + pub lose_submit_before_commit: bool, + pub lose_reservation: bool, + pub stale: bool, + pub bad_receipt: bool, + pub bad_status: bool, + pub wrong_reservation: bool, + pub deny_preflight: bool, + pub deny_upload: bool, + pub deny_status: bool, + pub fail_naming: bool, + pub redirect: Option, + pub prior: Option, + pub uploads: BTreeMap)>, + pub submits: Vec>, + pub reservations: Vec, + pub names: usize, + pub preflights: usize, + pub begin_count: usize, + pub status: Option, + pub authenticated: usize, +} +pub(crate) struct Fixture { + pub endpoint: String, + pub state: Arc>, + stop: CancellationToken, + task: tokio::task::JoinHandle<()>, +} +impl Fixture { + pub async fn new() -> Self { + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("http://{}/", listener.local_addr().unwrap()); + let state = Arc::new(Mutex::new(Behavior { + permission: true, + ..Default::default() + })); + let stop = CancellationToken::new(); + let cancellation = stop.clone(); + let app = Router::new() + .fallback(handler) + .layer(axum::extract::DefaultBodyLimit::max(32 * 1024 * 1024)) + .with_state((state.clone(), endpoint.clone())); + let task = tokio::spawn(async move { + axum::serve(listener, app) + .with_graceful_shutdown(cancellation.cancelled_owned()) + .await + .unwrap(); + }); + Self { + endpoint, + state, + stop, + task, + } + } + pub fn client(&self) -> PublisherClient { + PublisherClient::new(&self.endpoint, TOKEN.parse().unwrap()).unwrap() + } + pub fn record(&self, new: bool, name: bool) -> Record { + let bytes = SYSTEM_EMPTY_MODULE_V1_BYTES; + let envelope = PublishEnvelope { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), + expected_revision: None, + module_action: ModuleAction::Set(UserModule { + kind: UserModuleKind::Wasm, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }), + container_action: Default::default(), + }; + let module_artifact = SYSTEM_EMPTY_MODULE_V1_ARTIFACT; + let request = PublishRequest { + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + deployment: envelope.resolve(None, &Default::default()).unwrap(), + envelope, + module_artifact, + migration_policy: PreparedMigrationPolicy::Compatible, + }), + creation: new.then_some(CreationOptions { + parent: None, + organization: None, + num_replicas: None, + enforce_anti_affinity: true, + }), + image_source: None, + }; + let request_json = serde_json::to_string(&request).unwrap(); + Record { + version: 1, + server: self.endpoint.clone(), + artifact_endpoint: self.endpoint.clone(), + publisher: publisher(), + database: (!new).then_some(database()), + reservation: request.creation.clone().map(|options| ReserveDatabaseRequest { + version: PUBLISH_PROTOCOL_VERSION, + operation_id: request.manifest.current().envelope.operation_id, + options, + }), + requested_name: name.then_some("fixture-name".into()), + request_digest: spacetimedb_oci::sha256(request_json.as_bytes()), + request_json, + uploads: vec![UploadRecord { + kind: UploadKind::Module, + object: ObjectRef { + digest: module_artifact.digest, + size: module_artifact.size_bytes, + }, + session: None, + }], + submitted: false, + status: None, + name_confirmed: false, + naming_attempted: false, + } + } + pub async fn close(self) { + self.stop.cancel(); + self.task.await.unwrap(); + } +} +async fn handler( + State((shared, endpoint)): State<(Arc>, String)>, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let mut state = shared.lock().unwrap(); + let path = uri.path(); + if path == "/v1/containers/capabilities" { + assert!(headers.get("authorization").is_none()); + if let Some(location) = &state.redirect { + return (StatusCode::TEMPORARY_REDIRECT, [("location", location.clone())]).into_response(); + } + return Json(PublicationCapabilities { + version: 1, + enabled: true, + artifact_endpoint: Some(endpoint), + }) + .into_response(); + } + if headers.get("authorization").and_then(|v| v.to_str().ok()) != Some(TOKEN) { + return StatusCode::UNAUTHORIZED.into_response(); + } + state.authenticated += 1; + if path == "/v1/containers/publish-permission" { + return Json(PublishPermission { + identity: publisher(), + can_publish: state.permission, + source_revision: None, + }) + .into_response(); + } + if path == "/v1/containers/reservations" { + let request: ReserveDatabaseRequest = serde_json::from_slice(&body).unwrap(); + if let Some(prior) = state.reservations.first() { + assert_eq!(prior, &request); + } + state.reservations.push(request.clone()); + if std::mem::take(&mut state.lose_reservation) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(DatabaseReservation { + database_identity: database(), + operation_id: if state.wrong_reservation { + Uuid::from_u128(uuid::Uuid::now_v7().as_u128()) + } else { + request.operation_id + }, + expires_at: "fixture-only".into(), + staging_open: true, + artifact_endpoint: endpoint, + }) + .into_response(); + } + if path.ends_with("/pre_publish") { + state.preflights += 1; + if state.deny_preflight { + return StatusCode::FORBIDDEN.into_response(); + } + assert_eq!(body.as_ref(), SYSTEM_EMPTY_MODULE_V1_BYTES); + return Json(spacetimedb_client_api_messages::name::PrePublishResult::AutoMigrate( + spacetimedb_client_api_messages::name::PrePublishAutoMigrateResult { + migrate_plan: "fixture migration".into(), + break_clients: false, + token: spacetimedb_lib::Hash::ZERO, + major_version_upgrade: false, + }, + )) + .into_response(); + } + if path.ends_with("/names") { + state.names += 1; + if state.fail_naming { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(spacetimedb_client_api_messages::name::SetDomainsResult::Success).into_response(); + } + if path.contains("/deployment/operations/") { + if state.deny_status { + return StatusCode::FORBIDDEN.into_response(); + } + return state + .status + .clone() + .map(|status| Json(status).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + if path.ends_with("/deployment") { + if method == Method::GET { + return state + .prior + .clone() + .map(|p| Json(p).into_response()) + .unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + assert_eq!(method, Method::PUT); + state.submits.push(body.to_vec()); + if let Some(status) = &state.status { + assert_eq!(state.submits.first().unwrap().as_slice(), body.as_ref()); + return Json(status.clone()).into_response(); + } + if state.stale { + return (StatusCode::CONFLICT, "private-server-error-must-not-be-printed").into_response(); + } + if std::mem::take(&mut state.lose_submit_before_commit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let request: PublishRequest = serde_json::from_slice(&body).unwrap(); + request.manifest.validate(&Default::default()).unwrap(); + let status = PublicationStatus { + database_identity: database(), + operation_id: request.manifest.current().envelope.operation_id, + phase: PublicationPhase::Complete, + expected_revision: request.manifest.current().envelope.expected_revision, + proposed_revision: if state.bad_status { + spacetimedb_lib::Hash::ZERO + } else { + request.manifest.current().deployment.revision().unwrap() + }, + error: None, + }; + state.status = Some(status.clone()); + if std::mem::take(&mut state.lose_submit_after_commit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + return Json(status).into_response(); + } + if path.ends_with("/uploads") { + state.begin_count += 1; + if state.deny_upload { + return StatusCode::FORBIDDEN.into_response(); + } + let request: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + headers["x-spacetimedb-artifact-kind"], + request["kind"].as_str().unwrap() + ); + let object: ObjectRef = serde_json::from_value(request["object"].clone()).unwrap(); + let status = UploadStatus { + id: uuid::Uuid::new_v4(), + object, + offset: 0, + expires_at: u64::MAX, + complete: false, + }; + state.uploads.insert(status.id, (status.clone(), vec![])); + return Json(status).into_response(); + } + if let Some((_, tail)) = path.split_once("/uploads/") { + let id: uuid::Uuid = tail.split('/').next().unwrap().parse().unwrap(); + let Some((status, bytes)) = state.uploads.get_mut(&id) else { + return StatusCode::NOT_FOUND.into_response(); + }; + if method == Method::PATCH { + assert!(body.len() <= client::UPLOAD_CHUNK_BYTES); + let offset: u64 = uri.query().unwrap().strip_prefix("offset=").unwrap().parse().unwrap(); + assert_eq!(offset, status.offset); + bytes.extend_from_slice(&body); + status.offset += body.len() as u64; + } else if path.ends_with("/complete") { + assert_eq!(bytes.len() as u64, status.object.size); + assert_eq!(spacetimedb_oci::sha256(bytes), status.object.digest); + status.complete = true; + } + let mut response = status.clone(); + if method == Method::PATCH && std::mem::take(&mut state.lose_append) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + if state.bad_receipt { + response.object.size += 1; + } + return Json(response).into_response(); + } + panic!("unexpected fixture request: {method} {uri}"); +} +async fn run_now(client: &PublisherClient, journal: &mut Journal) -> Result { + run(client, journal, None, Duration::ZERO, CancellationToken::new()).await +} +fn journal(base: &Path, record: Record) -> Journal { + Journal::create(base, record, None, Some(SYSTEM_EMPTY_MODULE_V1_BYTES)).unwrap() +} +use std::path::Path; + +#[tokio::test] +async fn lost_append_and_committed_response_resume_without_reupload_or_current_permission() { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.lose_append = true; + s.lose_submit_after_commit = true; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.submitted); + let path = journal.directory().to_owned(); + drop(journal); + std::fs::remove_file(path.join("module.blob")).unwrap(); + fixture.state.lock().unwrap().permission = false; + let mut journal = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.submits, [exact.into_bytes()]); + assert_eq!(state.begin_count, 1); + } + drop(journal); + fixture.close().await; +} +#[tokio::test] +async fn lost_reservation_and_unadmitted_put_replay_exact_request_and_generated_identity() { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.lose_reservation = true; + s.lose_submit_before_commit = true; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(true, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.database.is_none()); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert_eq!(journal.record.database, Some(database())); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.reservations.len(), 2); + assert_eq!(state.submits, [exact.as_bytes(), exact.as_bytes()]); + assert_eq!(state.begin_count, 1); + } + fixture.close().await; +} +#[tokio::test] +async fn stale_revision_preserves_operation_and_redacts_response_without_fallback() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().stale = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); + assert!(!format!("{error:#}").contains("private-server-error")); + assert_eq!(journal.record.request_json, exact); + assert!(journal.record.submitted); + assert_eq!(fixture.state.lock().unwrap().submits.len(), 1); + fixture.close().await; +} +#[tokio::test] +async fn mismatched_reservation_or_upload_receipt_never_reaches_admission() { + for reservation in [true, false] { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.wrong_reservation = reservation; + s.bad_receipt = !reservation; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(reservation, false)); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(!journal.record.submitted); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + fixture.close().await; + } +} +#[tokio::test] +async fn wrong_publication_scope_and_denied_upload_are_not_accepted() { + for denied in [true, false] { + let fixture = Fixture::new().await; + { + let mut s = fixture.state.lock().unwrap(); + s.deny_upload = denied; + s.bad_status = !denied; + } + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert!(journal.record.status.is_none()); + assert_eq!(fixture.state.lock().unwrap().submits.len(), usize::from(!denied)); + fixture.close().await; + } +} +#[tokio::test] +async fn naming_failure_reports_active_identity_without_republishing_or_overwriting_later_names() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().fail_naming = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(true, true)); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::NamingUnconfirmed(_) + )); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::NamingUnconfirmed(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.names, 1); + assert_eq!(state.submits.len(), 1); + } + fixture.close().await; +} +#[tokio::test] +async fn redirect_and_cross_origin_never_forward_credentials_without_exact_approval() { + let first = Fixture::new().await; + let foreign = Fixture::new().await; + first.state.lock().unwrap().redirect = Some(format!("{}v1/containers/publish-permission", foreign.endpoint)); + assert!(first.client().capabilities().await.is_err()); + assert!(first.client().artifact_endpoint(&foreign.endpoint, None).is_err()); + assert!(first + .client() + .artifact_endpoint(&foreign.endpoint, Some(&first.endpoint)) + .is_err()); + assert!(first + .client() + .artifact_endpoint(&foreign.endpoint, Some(&foreign.endpoint)) + .is_ok()); + assert_eq!(foreign.state.lock().unwrap().authenticated, 0); + first.close().await; + foreign.close().await; +} +#[tokio::test] +async fn journal_locks_and_reverifies_local_bytes_without_persisting_credentials() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let path = journal.directory().to_owned(); + assert!(Journal::open(&path).is_err()); + let bytes = std::fs::read_to_string(path.join("publication.json")).unwrap(); + assert!(!bytes.contains("isolated-fixture-credential")); + std::fs::write(path.join("module.blob"), vec![0; SYSTEM_EMPTY_MODULE_V1_BYTES.len()]).unwrap(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + assert_eq!(fixture.state.lock().unwrap().begin_count, 0); + drop(journal); + let mut value: serde_json::Value = serde_json::from_str(&bytes).unwrap(); + value["request_json"] = json!("{}"); + std::fs::write(path.join("publication.json"), serde_json::to_vec(&value).unwrap()).unwrap(); + assert!(Journal::open(&path).is_err()); + fixture.close().await; +} +#[test] +fn endpoints_and_receipts_are_bounded_and_unambiguous() { + for endpoint in [ + "Maincloud", + "http://10.0.0.1", + "https://user:password@example.invalid", + "https://example.invalid/?token=secret", + ] { + assert!(client::endpoint(endpoint).is_err()); + } + for endpoint in ["http://localhost:3000", "http://[::1]:3000"] { + assert!(client::endpoint(endpoint).is_ok()); + } + let object = ObjectRef { + digest: SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size: 1, + }; + let status = UploadStatus { + id: uuid::Uuid::now_v7(), + object, + offset: 0, + expires_at: 0, + complete: false, + }; + assert!(status.validate(object, None).is_err()); +} + +#[tokio::test] +async fn incomplete_artifact_retention_never_publishes_a_resume_record_or_mutates_server() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let base = temporary.path().join("new").join("nested").join("state"); + let record = fixture.record(false, false); + let id = record.request().unwrap().manifest.current().envelope.operation_id; + // A missing promised module fails the artifact flush before publication.json + // can make this operation available for reservation or admission. + assert!(Journal::create(&base, record, None, None).is_err()); + assert!(!base.join(id.to_string()).exists()); + assert_eq!(fixture.state.lock().unwrap().authenticated, 0); + fixture.close().await; +} + +#[tokio::test] +async fn revoked_status_access_replays_original_put_without_missing_local_artifacts_or_new_uploads() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_after_commit = true; + let temporary = tempfile::tempdir().unwrap(); + let mut journal = journal(temporary.path(), fixture.record(false, false)); + let exact = journal.record.request_json.clone(); + assert!(run_now(&fixture.client(), &mut journal).await.is_err()); + let path = journal.directory().to_owned(); + drop(journal); + std::fs::remove_file(path.join("module.blob")).unwrap(); + { + let mut state = fixture.state.lock().unwrap(); + state.permission = false; + state.deny_status = true; + state.deny_upload = true; + } + let mut journal = Journal::open(&path).unwrap(); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.submits, [exact.as_bytes(), exact.as_bytes()]); + assert_eq!(state.begin_count, 1); + } + fixture.close().await; +} diff --git a/crates/cli/src/container/tests.rs b/crates/cli/src/container/tests.rs index 9d56133f360..110eaa4acac 100644 --- a/crates/cli/src/container/tests.rs +++ b/crates/cli/src/container/tests.rs @@ -11,7 +11,7 @@ fn platform() -> ImagePlatform { architecture: "amd64".into(), } } -fn declaration(image: serde_json::Value) -> ContainerConfig { +pub(crate) fn declaration(image: serde_json::Value) -> ContainerConfig { serde_json::from_value(json!({"image":image,"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}})).unwrap() } fn blob(layout: &Path, bytes: &[u8], media: &str) -> Descriptor { @@ -34,7 +34,7 @@ fn blob(layout: &Path, bytes: &[u8], media: &str) -> Descriptor { artifact_type: None, } } -fn fixture(layout: &Path) -> Descriptor { +pub(crate) fn fixture(layout: &Path) -> Descriptor { let mut archive = tar::Builder::new(Vec::new()); let mut header = tar::Header::new_gnu(); header.set_size(4); @@ -460,7 +460,7 @@ fn container_build_command_requires_explicit_platform_and_output() { } #[test] -fn legacy_publish_rejects_container_targets_without_affecting_plain_children() { +fn publish_preserves_container_target_metadata_without_affecting_plain_children() { use crate::subcommands::publish::{build_publish_schema, get_filtered_publish_configs}; let config: SpacetimeConfig = serde_json::from_value(json!({ "database":"container-db", "container":declaration(json!({"build":{}})), @@ -471,10 +471,9 @@ fn legacy_publish_rejects_container_targets_without_affecting_plain_children() { let schema = build_publish_schema(&command).unwrap(); for selected in ["container-db", "*"] { let args = command.clone().try_get_matches_from(["publish", selected]).unwrap(); - let error = get_filtered_publish_configs(&config, &command, &schema, &args) - .unwrap_err() - .to_string(); - assert!(error.contains("does not yet publish container")); + let targets = get_filtered_publish_configs(&config, &command, &schema, &args).unwrap(); + assert!(targets[0].container().is_some()); + assert!(targets.iter().skip(1).all(|target| target.container().is_none())); } let args = command .clone() diff --git a/crates/cli/src/spacetime_config.rs b/crates/cli/src/spacetime_config.rs index 51a31fd7ec8..dc493abba8f 100644 --- a/crates/cli/src/spacetime_config.rs +++ b/crates/cli/src/spacetime_config.rs @@ -258,6 +258,8 @@ pub struct CommandConfig<'a> { config_values: HashMap, /// CLI arguments matches: &'a ArgMatches, + /// A declaration belongs to this exact target and never inherits. + container: Option, } /// Schema that defines the contract between CLI arguments and config file keys. @@ -704,9 +706,23 @@ impl<'a> CommandConfig<'a> { schema, config_values: normalized_values, matches, + container: None, }) } + pub fn with_container(mut self, container: Option) -> Self { + self.container = container; + self + } + + pub fn container(&self) -> Option<&crate::container::config::ContainerConfig> { + self.container.as_ref() + } + + pub(crate) fn matches(&self) -> &ArgMatches { + self.matches + } + /// Get a single value from the config as a specific type. /// First checks clap args (via schema), then falls back to config values. /// diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs index 8badb72773a..a14aff0d16a 100644 --- a/crates/cli/src/subcommands/dev.rs +++ b/crates/cli/src/subcommands/dev.rs @@ -928,6 +928,10 @@ fn determine_publish_configs<'a>( } if !publish_configs.is_empty() { + anyhow::ensure!( + publish_configs.iter().all(|target| target.container().is_none()), + "spacetime dev does not yet run container targets; use spacetime publish for managed publication" + ); return Ok(publish_configs); } diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 6e66ac17ff9..68a90f6e178 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -19,6 +19,8 @@ use crate::util::{add_auth_header_opt, get_auth_header, strip_verbatim_prefix, A use crate::util::{decode_identity, y_or_n}; use crate::{build, common_args}; +mod managed; + /// Individual prompts that `--yes` can suppress. `All` is a shorthand for every category below. #[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] #[clap(rename_all = "kebab-case")] @@ -83,7 +85,7 @@ fn yes_flags_from_args(args: &ArgMatches) -> YesFlags { /// Build the CommandSchema for publish command pub fn build_publish_schema(command: &clap::Command) -> Result { - CommandSchemaBuilder::new() + managed::exclude_args(CommandSchemaBuilder::new()) .key(Key::new("database").from_clap("name|identity").required()) .key(Key::new("server")) .key(Key::new("module_path").module_specific()) @@ -171,9 +173,7 @@ pub fn get_filtered_publish_configs<'a>( let configs: Vec = filtered_targets .into_iter() .map(|target| { - anyhow::ensure!(target.container.is_none(), - "This CLI does not yet publish container declarations. Use `spacetime container build` to prepare the image; managed publication support is required before publishing this target."); - let config = CommandConfig::new(schema, target.fields, args)?; + let config = CommandConfig::new(schema, target.fields, args)?.with_container(target.container); config.validate()?; Ok(config) }) @@ -191,7 +191,7 @@ pub fn get_filtered_publish_configs<'a>( } pub fn cli() -> clap::Command { - clap::Command::new("publish") + managed::add_args(clap::Command::new("publish") .about("Create and update a SpacetimeDB database") .arg( common_args::clear_database() @@ -321,7 +321,7 @@ i.e. only lowercase ASCII letters and numbers, separated by dashes."), .action(SetTrue) .help("Use NativeAOT-LLVM compilation for C# modules (experimental, Windows only)") ) - .after_help("Run `spacetime help publish` for more detailed information.") + .after_help("Run `spacetime help publish` for more detailed information.")) } fn confirm_and_clear( @@ -383,6 +383,9 @@ pub async fn exec_with_options( quiet_config: bool, pre_loaded_config: Option<&LoadedConfig>, ) -> Result<(), anyhow::Error> { + if args.get_one::("resume_publication").is_some() { + return managed::resume(&mut config, args, yes_flags_from_args(args)).await; + } // Build schema let cmd = cli(); let schema = build_publish_schema(&cmd)?; @@ -498,20 +501,7 @@ async fn execute_publish_configs<'a>( }) }; - if using_config { - if let Some(path_to_project) = path_to_project.as_ref() { - println!( - "Publishing module {} to database '{}'", - strip_verbatim_prefix(path_to_project).display(), - name_or_identity.unwrap() - ); - } else { - println!( - "Publishing precompiled module to database '{}'", - name_or_identity.unwrap() - ); - } - } + managed::validate_target_options(&command_config)?; let database_host = config.get_host_url(server)?; let build_options = command_config .get_one::("build_options")? @@ -532,6 +522,36 @@ async fn execute_publish_configs<'a>( let (name_or_identity, parent) = validate_name_and_parent(name_or_identity, parent)?; + if managed::try_execute( + &command_config, + config_dir, + &database_host, + &auth_header, + name_or_identity, + parent, + clear_database, + yes, + ) + .await? + { + continue; + } + + if using_config { + if let Some(path_to_project) = path_to_project.as_ref() { + println!( + "Publishing module {} to database '{}'", + strip_verbatim_prefix(path_to_project).display(), + name_or_identity.unwrap() + ); + } else { + println!( + "Publishing precompiled module to database '{}'", + name_or_identity.unwrap() + ); + } + } + if let Some(path_to_project) = path_to_project.as_ref() && !path_to_project.exists() { diff --git a/crates/cli/src/subcommands/publish/managed.rs b/crates/cli/src/subcommands/publish/managed.rs new file mode 100644 index 00000000000..887497ed000 --- /dev/null +++ b/crates/cli/src/subcommands/publish/managed.rs @@ -0,0 +1,955 @@ +//! Managed publication frontend. The resume journal owns every byte needed to +//! retry; project configuration and mutable image tags are only read initially. +use super::{confirm_major_version_upgrade, YesFlags}; +use crate::{ + common_args::ClearMode, + config::Config, + container::{ + self, + oci::ArtifactKind, + process::LocalRunner, + publish::{ + self, + client::{ObjectRef, PublisherClient, UploadKind}, + journal::{Journal, Record, UploadRecord}, + Outcome, + }, + BuildSecret, BuildTools, + }, + spacetime_config::{CommandConfig, CommandSchemaBuilder}, + util::{get_auth_header, y_or_n, AuthHeader}, +}; +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use spacetimedb_client_api_messages::name::{is_identity, DomainName, PrePublishResult}; +use spacetimedb_lib::{ + container::{ContainerAction, ImagePlatform}, + deployment::{self, api::*, manifest::*, ModuleAction, PublishEnvelope, UserModule, UserModuleKind}, + Identity, Uuid, +}; +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; +use tokio_util::sync::CancellationToken; + +const ARGUMENTS: &[&str] = &[ + "managed", + "container_platform", + "artifact_endpoint", + "remove_container", + "remove_module", + "publication_state_dir", + "resume_publication", + "publication_wait", + "buildkit_host", + "buildctl", + "railpack", + "skopeo", + "registry_auth_file", + "build_secret", +]; +pub(super) fn exclude_args(mut schema: CommandSchemaBuilder) -> CommandSchemaBuilder { + for arg in ARGUMENTS { + schema = schema.exclude(*arg); + } + schema +} +pub(super) fn add_args(mut command: Command) -> Command { + command = command + .arg( + Arg::new("managed") + .long("managed") + .action(ArgAction::SetTrue) + .help("Use managed deployment publication, including for a module-only database"), + ) + .arg( + Arg::new("container_platform") + .long("container-platform") + .value_parser(["linux/amd64", "linux/arm64"]) + .help("Required platform when publishing a container declaration"), + ) + .arg( + Arg::new("artifact_endpoint") + .long("artifact-endpoint") + .help("Explicitly authorize this exact artifact URL to receive the publisher credential"), + ) + .arg( + Arg::new("remove_container") + .long("remove-container") + .action(ArgAction::SetTrue) + .help("Explicitly remove the container while preserving the module unless separately changed"), + ) + .arg( + Arg::new("remove_module") + .long("remove-module") + .action(ArgAction::SetTrue) + .conflicts_with_all(["module_path", "wasm_file", "js_file"]) + .help("Replace the module with the versioned empty module after migration preflight"), + ) + .arg( + Arg::new("publication_state_dir") + .long("publication-state-dir") + .value_parser(clap::value_parser!(PathBuf)) + .help("Private local directory retaining managed publication bytes and progress"), + ) + .arg( + Arg::new("resume_publication") + .long("resume-publication") + .value_parser(clap::value_parser!(PathBuf)) + .conflicts_with_all([ + "managed", + "remove_container", + "remove_module", + "module_path", + "wasm_file", + "js_file", + "container_platform", + "name|identity", + "parent", + "organization", + "clear-database", + ]) + .help("Resume this operation directory without rebuilding or reading spacetime.json"), + ) + .arg( + Arg::new("publication_wait") + .long("publication-wait") + .default_value("60") + .value_parser(clap::value_parser!(u64).range(0..=3600)) + .help("Seconds to wait for managed activation; pending operations retain their resume directory"), + ) + .arg( + Arg::new("buildkit_host") + .long("buildkit-host") + .help("Explicit local BuildKit Unix socket for container source builds"), + ) + .arg( + Arg::new("registry_auth_file") + .long("registry-auth-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Explicit registry auth JSON; otherwise image preparation is anonymous"), + ) + .arg( + Arg::new("build_secret") + .long("build-secret") + .action(ArgAction::Append) + .value_name("NAME=FILE") + .help("Build secret file, separate from runtime env_keys"), + ); + for tool in ["buildctl", "railpack", "skopeo"] { + command = command.arg( + Arg::new(tool) + .long(tool) + .default_value(tool) + .value_parser(clap::value_parser!(PathBuf)), + ); + } + command +} +fn approved(args: &ArgMatches) -> Option<&str> { + args.get_one::("artifact_endpoint").map(String::as_str) +} +fn wait(args: &ArgMatches) -> Duration { + Duration::from_secs(*args.get_one::("publication_wait").unwrap_or(&60)) +} +pub(super) fn validate_target_options(target: &CommandConfig<'_>) -> Result<()> { + let args = target.matches(); + if target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module") + { + ensure!( + !target.get_one::("anon_identity")?.unwrap_or(false), + "managed publication requires an authenticated publisher" + ); + } + Ok(()) +} +fn authenticated(server: &str, auth: &AuthHeader) -> Result { + PublisherClient::new( + server, + auth.to_header() + .context("managed publication requires an authenticated publisher; anonymous publication is unsupported")?, + ) +} +pub(super) async fn resume(config: &mut Config, args: &ArgMatches, yes: YesFlags) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let server = config.get_host_url(selection)?; + let mut journal = Journal::open(args.get_one::("resume_publication").unwrap())?; + ensure!( + container::publish::client::endpoint(&server)?.as_str() == journal.record.server, + "resume server differs from the original publication endpoint" + ); + let auth = get_auth_header(config, false, selection, !yes.skip_login).await?; + let client = authenticated(&server, &auth)?; + execute(&client, &mut journal, args).await +} + +/// False is only returned for an ordinary, un-managed module publication. +/// Once managed intent or a managed revision is known, errors cannot fall back. +#[allow(clippy::too_many_arguments)] +pub(super) async fn try_execute( + target: &CommandConfig<'_>, + config_dir: Option<&Path>, + server: &str, + auth: &AuthHeader, + name: Option<&str>, + parent: Option<&str>, + clear: ClearMode, + yes: YesFlags, +) -> Result { + let args = target.matches(); + let explicit = target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module"); + if !explicit && name.is_none() { + return Ok(false); + } + let client = match authenticated(server, auth) { + Ok(client) => client, + Err(error) if explicit => return Err(error), + Err(_) => return Ok(false), + }; + try_execute_with_client(target, config_dir, client, name, parent, clear, yes).await +} + +async fn try_execute_with_client( + target: &CommandConfig<'_>, + config_dir: Option<&Path>, + client: PublisherClient, + name: Option<&str>, + parent: Option<&str>, + clear: ClearMode, + yes: YesFlags, +) -> Result { + let args = target.matches(); + let explicit = target.container().is_some() + || args.get_flag("managed") + || args.get_flag("remove_container") + || args.get_flag("remove_module"); + let caps = client.capabilities().await?; + let Some(caps) = caps else { + ensure!( + !explicit, + "this server does not support managed publication; no module-only fallback was attempted" + ); + return Ok(false); + }; + let prior = if let Some(name) = name { + client.deployment(name).await? + } else { + None + }; + if !explicit && prior.as_ref().and_then(|p| p.revision).is_none() { + return Ok(false); + } + ensure!( + caps.enabled && caps.version == deployment::PUBLISH_PROTOCOL_VERSION, + "managed publication is disabled or incompatible on this server" + ); + ensure!( + clear == ClearMode::Never, + "managed publication does not support --delete-data; resolve migrations without replacing database storage" + ); + ensure!( + !(target.container().is_some() && args.get_flag("remove_container")), + "a container declaration and --remove-container cannot both select this target" + ); + if prior.is_none() { + ensure!( + !name.is_some_and(is_identity), + "a new database Identity must be generated by the reservation service; select a name or omit DATABASE" + ); + } + let artifact = client.artifact_endpoint( + caps.artifact_endpoint + .as_deref() + .context("server did not advertise an artifact endpoint")?, + approved(args), + )?; + let permission = client.permission().await?; + if target.container().is_some() || prior.is_none() { + ensure!( + permission.can_publish, + "the current publisher does not have container publication permission" + ); + } + if !container::publish::client::is_loopback(client.server()) { + ensure!( + y_or_n( + yes.publish_to_remote, + "Publish this managed deployment to the selected remote server?" + )?, + "publication cancelled" + ); + } + let cwd = std::env::current_dir()?; + let base = args + .get_one::("publication_state_dir") + .cloned() + .unwrap_or_else(|| config_dir.unwrap_or(&cwd).join(".spacetime/publications")); + std::fs::create_dir_all(&base)?; + let cancel = CancellationToken::new(); + let prepare = prepare_request( + &client, + target, + config_dir.unwrap_or(&cwd), + &base, + prior.as_ref(), + name, + parent, + permission.identity, + artifact.as_str(), + yes, + cancel.clone(), + ); + tokio::pin!(prepare); + let mut journal = tokio::select! { + result = &mut prepare => result?, + signal = tokio::signal::ctrl_c() => { + signal?; cancel.cancel(); let _ = prepare.await; + bail!("managed preparation cancelled before admission"); + } + }; + execute(&client, &mut journal, args).await?; + Ok(true) +} + +#[allow(clippy::too_many_arguments)] +async fn prepare_request( + client: &PublisherClient, + target: &CommandConfig<'_>, + config_dir: &Path, + base: &Path, + prior: Option<&DeploymentStatus>, + name: Option<&str>, + parent: Option<&str>, + publisher: Identity, + artifact_endpoint: &str, + yes: YesFlags, + cancel: CancellationToken, +) -> Result { + let args = target.matches(); + let image = if let Some(declaration) = target.container() { + let (os, architecture) = args + .get_one::("container_platform") + .context("publishing a container requires --container-platform linux/amd64 or linux/arm64")? + .split_once('/') + .unwrap(); + Some( + container::prepare_container( + declaration, + config_dir, + ImagePlatform { + os: os.into(), + architecture: architecture.into(), + }, + &tools(args)?, + base, + &LocalRunner, + cancel.clone(), + ) + .await?, + ) + } else { + None + }; + let has_module = ["module_path", "wasm_file", "js_file"] + .iter() + .any(|key| target.is_from_cli(key) || target.get_config_value(key).is_some()); + ensure!( + !(has_module && args.get_flag("remove_module")), + "module configuration and --remove-module cannot both select this target" + ); + let module = if !args.get_flag("remove_module") + && (has_module || (target.container().is_none() && !args.get_flag("remove_container"))) + { + Some(load_module(target, config_dir).await?) + } else { + None + }; + let module_action = if args.get_flag("remove_module") { + ModuleAction::Remove + } else if let Some((kind, bytes)) = &module { + ModuleAction::Set(UserModule { + kind: *kind, + program_hash: spacetimedb_lib::hash_bytes(bytes), + }) + } else { + ModuleAction::Keep + }; + let container_action = if let Some(image) = &image { + ContainerAction::Set(image.metadata.container.clone()) + } else if args.get_flag("remove_container") { + ContainerAction::Remove + } else { + ContainerAction::Keep + }; + let envelope = PublishEnvelope { + version: deployment::PUBLISH_PROTOCOL_VERSION, + operation_id: Uuid::from_u128(uuid::Uuid::now_v7().as_u128()), + expected_revision: prior.and_then(|p| p.revision), + module_action, + container_action, + }; + let deployment = envelope.resolve(prior.map(|p| &p.deployment), &Default::default())?; + let module_artifact = if let Some((_, bytes)) = &module { + ModuleArtifact { + digest: spacetimedb_oci::sha256(bytes), + size_bytes: bytes.len() as u64, + } + } else if let Some(prior) = prior.filter(|_| !matches!(envelope.module_action, ModuleAction::Remove)) { + let artifact = prior.module_artifact; + ModuleArtifact { + digest: artifact.digest, + size_bytes: artifact.size_bytes, + } + } else { + deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT + }; + let migration_policy = if let Some(prior) = prior { + if let Some((kind, bytes)) = &module { + migration(client, prior.database_identity, *kind, bytes, target, yes).await? + } else if matches!(envelope.module_action, ModuleAction::Remove) { + migration( + client, + prior.database_identity, + UserModuleKind::Wasm, + deployment::SYSTEM_EMPTY_MODULE_V1_BYTES, + target, + yes, + ) + .await? + } else { + PreparedMigrationPolicy::Compatible + } + } else { + PreparedMigrationPolicy::Compatible + }; + let creation = if prior.is_none() { + let parent = if let Some(parent) = parent { + Some( + client + .deployment(parent) + .await? + .context("parent database does not exist or is not accessible")? + .database_identity, + ) + } else { + None + }; + let organization = target + .get_one::("organization")? + .map(|value| { + value + .parse::() + .context("managed publication currently requires --organization IDENTITY, not an organization name") + }) + .transpose()?; + Some(CreationOptions { + parent, + organization, + num_replicas: target.get_one::("num_replicas")?.map(u32::from), + enforce_anti_affinity: true, + }) + } else { + None + }; + let request = PublishRequest { + manifest: PreparedDeploymentManifest::V1(PreparedDeploymentManifestV1 { + envelope, + deployment, + module_artifact, + migration_policy, + }), + creation, + image_source: image.as_ref().map(|image| ArtifactReference { + digest: image.metadata.manifest.digest, + size_bytes: image.metadata.manifest.size, + }), + }; + request.manifest.validate(&Default::default())?; + let request_json = serde_json::to_string(&request)?; + let mut uploads = image + .as_ref() + .map(|image| { + image + .metadata + .objects + .iter() + .map(|object| UploadRecord { + kind: match object.kind { + ArtifactKind::Manifest => UploadKind::Manifest, + ArtifactKind::Config => UploadKind::Config, + ArtifactKind::Layer => UploadKind::Layer, + }, + object: ObjectRef { + digest: object.descriptor.digest, + size: object.descriptor.size, + }, + session: None, + }) + .collect::>() + }) + .unwrap_or_default(); + if module.is_some() { + uploads.push(UploadRecord { + kind: UploadKind::Module, + object: ObjectRef { + digest: module_artifact.digest, + size: module_artifact.size_bytes, + }, + session: None, + }); + } + let requested_name = if prior.is_none() { + name.map(|name| name.parse::().map(|name| name.to_string())) + .transpose()? + } else { + None + }; + let record = Record { + version: 1, + server: client.server().to_string(), + artifact_endpoint: artifact_endpoint.into(), + publisher, + database: prior.map(|p| p.database_identity), + reservation: request.creation.clone().map(|options| ReserveDatabaseRequest { + version: deployment::PUBLISH_PROTOCOL_VERSION, + operation_id: request.manifest.current().envelope.operation_id, + options, + }), + requested_name, + request_digest: spacetimedb_oci::sha256(request_json.as_bytes()), + request_json, + uploads, + submitted: false, + status: None, + name_confirmed: false, + naming_attempted: false, + }; + Journal::create(base, record, image, module.as_ref().map(|(_, bytes)| bytes.as_slice())) +} +fn tools(args: &ArgMatches) -> Result { + let mut tools = BuildTools { + buildctl: args.get_one::("buildctl").unwrap().clone(), + railpack: args.get_one::("railpack").unwrap().clone(), + skopeo: args.get_one::("skopeo").unwrap().clone(), + buildkit_host: args.get_one::("buildkit_host").cloned(), + registry_auth_file: args.get_one::("registry_auth_file").cloned(), + secrets: vec![], + }; + for value in args.get_many::("build_secret").into_iter().flatten() { + let (name, file) = value.split_once('=').context("build secret must be NAME=FILE")?; + ensure!(!file.is_empty(), "build secret file is missing"); + tools.secrets.push(BuildSecret { + name: name.into(), + file: file.into(), + }); + } + Ok(tools) +} +async fn load_module(target: &CommandConfig<'_>, config_dir: &Path) -> Result<(UserModuleKind, Vec)> { + ensure!( + !target.get_one::("native_aot")?.unwrap_or(false), + "managed NativeAOT builds are not yet supported; build separately and pass --bin-path" + ); + let (path, kind) = if let Some(path) = target.get_resolved_path("wasm_file", Some(config_dir))? { + (path, "Wasm") + } else if let Some(path) = target.get_resolved_path("js_file", Some(config_dir))? { + (path, "Js") + } else { + let path = target + .get_resolved_path("module_path", Some(config_dir))? + .unwrap_or_else(|| super::default_publish_module_path(config_dir)); + crate::build::exec_with_argstring(&path, &target.get_one::("build_options")?.unwrap_or_default()) + .await? + }; + let kind = match kind { + "Wasm" => UserModuleKind::Wasm, + "Js" => UserModuleKind::Js, + _ => bail!("unsupported managed module kind"), + }; + let metadata = tokio::fs::metadata(&path).await?; + ensure!( + metadata.is_file() && metadata.len() > 0 && metadata.len() <= MAX_MODULE_ARTIFACT_BYTES, + "module must be a regular file of at most 32 MiB" + ); + use tokio::io::AsyncReadExt as _; + let mut bytes = Vec::new(); + tokio::fs::File::open(&path) + .await? + .take(MAX_MODULE_ARTIFACT_BYTES + 1) + .read_to_end(&mut bytes) + .await?; + ensure!( + !bytes.is_empty() && bytes.len() as u64 <= MAX_MODULE_ARTIFACT_BYTES, + "module exceeds 32 MiB" + ); + Ok((kind, bytes)) +} +async fn migration( + client: &PublisherClient, + database: Identity, + kind: UserModuleKind, + bytes: &[u8], + target: &CommandConfig<'_>, + yes: YesFlags, +) -> Result { + let pre = client + .preflight( + database, + match kind { + UserModuleKind::Wasm => "Wasm", + UserModuleKind::Js => "Js", + }, + bytes, + ) + .await?; + match pre { + PrePublishResult::ManualMigrate(_) => { + bail!("managed publication requires manual migration; no storage was cleared") + } + PrePublishResult::AutoMigrate(auto) => { + if auto.major_version_upgrade { + confirm_major_version_upgrade(yes.migrate_major_version)?; + } + println!("{}", auto.migrate_plan); + if auto.break_clients { + ensure!( + y_or_n( + yes.break_clients || target.get_one::("break_clients")?.unwrap_or(false), + "These changes will BREAK existing clients. Proceed?" + )?, + "publication cancelled" + ); + } + Ok(PreparedMigrationPolicy::BreakClients(auto.token)) + } + } +} +async fn execute(client: &PublisherClient, journal: &mut Journal, args: &ArgMatches) -> Result<()> { + println!( + "Publication {}. Resume directory: {}", + journal.operation()?, + journal.directory().display() + ); + let cancel = CancellationToken::new(); + let operation = publish::run(client, journal, approved(args), wait(args), cancel.clone()); + tokio::pin!(operation); + let outcome = tokio::select! { + result = &mut operation => result?, + signal = tokio::signal::ctrl_c() => { signal?; cancel.cancel(); bail!("publication interrupted; resume the same directory to determine its outcome"); } + }; + match outcome { + Outcome::Complete(status) => println!( + "Deployment {} is active on {}", + status.proposed_revision, status.database_identity + ), + Outcome::Pending(status) => println!( + "Publication {} is {:?} on {}; resume to observe activation", + status.operation_id, status.phase, status.database_identity + ), + Outcome::Aborted(status) => bail!( + "publication {} was aborted before commit on {}; resume state retained", + status.operation_id, + status.database_identity + ), + Outcome::NamingUnconfirmed(status) => bail!( + "deployment is active on {}, but naming was not confirmed; publication must not be repeated", + status.database_identity + ), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::container::publish::tests::{database, Fixture}; + use crate::spacetime_config::SpacetimeConfig; + use serde_json::json; + use spacetimedb_paths::FromPathUnchecked as _; + use std::collections::HashMap; + + #[tokio::test] + async fn container_only_frontend_uploads_verified_closure_with_server_reserved_identity() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let layout = temporary.path().join("layout"); + let selected = crate::container::tests::fixture(&layout); + let declaration = crate::container::tests::declaration(json!({"oci_ref":"oci:layout"})); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let state = temporary.path().join("state"); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--container-platform", + "linux/amd64", + "--publication-state-dir", + state.to_str().unwrap(), + "--publication-wait", + "0", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args) + .unwrap() + .with_container(Some(declaration)); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + { + let snapshot = fixture.state.lock().unwrap(); + assert_eq!(snapshot.reservations.len(), 1); + assert_eq!(snapshot.submits.len(), 1); + assert_eq!(snapshot.begin_count, 3); + let request: PublishRequest = serde_json::from_slice(&snapshot.submits[0]).unwrap(); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Keep + )); + assert!(matches!( + request.manifest.current().deployment.current().module, + deployment::ModuleComponent::SystemEmpty(1) + )); + assert_eq!( + request.manifest.current().module_artifact, + deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT + ); + assert_eq!(request.image_source.unwrap().digest, selected.digest); + let dir = state.join(request.manifest.current().envelope.operation_id.to_string()); + let journal = Journal::open(&dir).unwrap(); + assert_eq!(journal.record.database, Some(database())); + let bytes = std::fs::read_to_string(dir.join("publication.json")).unwrap(); + assert!(!bytes.contains("private-image-value")); + } + fixture.close().await; + } + #[tokio::test] + async fn existing_managed_module_update_keeps_container_and_preflights_without_pro() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let wasm = temporary.path().join("module.wasm"); + std::fs::write(&wasm, deployment::SYSTEM_EMPTY_MODULE_V1_BYTES).unwrap(); + let prior_request = fixture.record(false, false).request().unwrap(); + let prior = DeploymentStatus { + database_identity: database(), + revision: Some(prior_request.manifest.current().deployment.revision().unwrap()), + deployment: prior_request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size_bytes: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.size_bytes, + }, + }; + { + let mut state = fixture.state.lock().unwrap(); + state.permission = false; + state.prior = Some(prior.clone()); + } + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--bin-path", + wasm.to_str().unwrap(), + "--publication-state-dir", + temporary.path().join("state").to_str().unwrap(), + "--publication-wait", + "0", + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.preflights, 1); + assert_eq!(state.reservations.len(), 0); + let request: PublishRequest = serde_json::from_slice(&state.submits[0]).unwrap(); + assert_eq!(request.manifest.current().envelope.expected_revision, prior.revision); + assert!(matches!( + request.manifest.current().envelope.container_action, + ContainerAction::Keep + )); + assert!(matches!( + request.manifest.current().envelope.module_action, + ModuleAction::Set(_) + )); + } + fixture.close().await; + } + #[tokio::test] + async fn remove_module_requires_authorized_preflight_before_any_upload_or_submission() { + let fixture = Fixture::new().await; + let temporary = tempfile::tempdir().unwrap(); + let request = fixture.record(false, false).request().unwrap(); + { + let mut state = fixture.state.lock().unwrap(); + state.deny_preflight = true; + state.prior = Some(DeploymentStatus { + database_identity: database(), + revision: Some(request.manifest.current().deployment.revision().unwrap()), + deployment: request.manifest.current().deployment.clone(), + module_artifact: ArtifactReference { + digest: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.digest, + size_bytes: deployment::SYSTEM_EMPTY_MODULE_V1_ARTIFACT.size_bytes, + }, + }); + } + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from([ + "publish", + "fixture-name", + "--server", + fixture.endpoint.as_str(), + "--remove-module", + "--publication-state-dir", + temporary.path().to_str().unwrap(), + ]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(try_execute_with_client( + &target, + Some(temporary.path()), + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .is_err()); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.preflights, 1); + assert_eq!(state.begin_count, 0); + assert!(state.submits.is_empty()); + } + fixture.close().await; + } + #[tokio::test] + async fn ordinary_module_target_returns_to_legacy_only_when_no_managed_revision_exists() { + let fixture = Fixture::new().await; + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command + .try_get_matches_from(["publish", "fixture-name", "--server", fixture.endpoint.as_str()]) + .unwrap(); + let target = CommandConfig::new(&schema, HashMap::new(), &args).unwrap(); + assert!(!try_execute_with_client( + &target, + None, + fixture.client(), + Some("fixture-name"), + None, + ClearMode::Never, + YesFlags::all() + ) + .await + .unwrap()); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + fixture.close().await; + } + #[tokio::test] + async fn resume_entrypoint_ignores_changed_project_and_reuses_original_bytes() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_submit_before_commit = true; + let temporary = tempfile::tempdir().unwrap(); + let record = fixture.record(false, false); + let exact = record.request_json.clone(); + let mut journal = Journal::create( + temporary.path(), + record, + None, + Some(deployment::SYSTEM_EMPTY_MODULE_V1_BYTES), + ) + .unwrap(); + assert!(publish::run( + &fixture.client(), + &mut journal, + None, + Duration::ZERO, + CancellationToken::new() + ) + .await + .is_err()); + let resume = journal.directory().to_owned(); + drop(journal); + let cli_config = temporary.path().join("isolated-cli.toml"); + std::fs::write(&cli_config, "spacetimedb_token = 'isolated-fixture-credential'\n").unwrap(); + let config = Config::load(spacetimedb_paths::cli::CliTomlPath::from_path_unchecked(cli_config)).unwrap(); + let changed_project = crate::spacetime_config::LoadedConfig { + config: serde_json::from_value(json!({"database":"different-target", "module_path":"missing-build-source", "server":"must-never-resolve-this-alias"})).unwrap(), + config_dir: temporary.path().into(), loaded_files: vec![], has_dev_file: false, + }; + let args = super::super::cli() + .try_get_matches_from([ + "publish", + "--resume-publication", + resume.to_str().unwrap(), + "--server", + fixture.endpoint.as_str(), + "--publication-wait", + "0", + ]) + .unwrap(); + super::super::exec_with_options(config, &args, true, Some(&changed_project)) + .await + .unwrap(); + assert_eq!( + fixture.state.lock().unwrap().submits, + [exact.as_bytes(), exact.as_bytes()] + ); + fixture.close().await; + } + #[test] + fn nested_targets_keep_only_their_own_container_and_resume_rejects_new_inputs() { + let config: SpacetimeConfig = serde_json::from_value(json!({"database":"parent", "container":{"image":{"oci_ref":"oci:layout"},"resources":{"cpu_millicores":100,"memory_bytes":67108864,"scratch_bytes":1048576,"pids_max":32}}, "children":[{"database":"child"}]})).unwrap(); + let command = super::super::cli(); + let schema = super::super::build_publish_schema(&command).unwrap(); + let args = command.clone().try_get_matches_from(["publish"]).unwrap(); + let targets = super::super::get_filtered_publish_configs(&config, &command, &schema, &args).unwrap(); + assert_eq!(targets.len(), 2); + assert!(targets[0].container().is_some()); + assert!(targets[1].container().is_none()); + for flags in [ + ["--resume-publication", "operation", "--remove-module"], + ["--resume-publication", "operation", "--managed"], + ] { + assert!(command + .clone() + .try_get_matches_from(std::iter::once("publish").chain(flags)) + .is_err()); + } + } +} From dcdcb6a11886468b5a3d620feebd51b41409ec69 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 06:53:11 -0400 Subject: [PATCH 14/23] Drain physical module work before closing database writers --- crates/core/src/db/durability.rs | 20 +- crates/core/src/db/relational_db.rs | 91 ++-- .../src/db/relational_db/shutdown_tests.rs | 94 +++++ crates/core/src/error.rs | 2 + crates/core/src/host/module_host.rs | 177 ++++---- .../core/src/host/module_host/drain_tests.rs | 395 ++++++++++++++++++ .../core/src/host/module_host/operations.rs | 126 ++++++ crates/core/src/host/v8/mod.rs | 105 ++++- crates/core/src/sql/execute.rs | 9 +- .../subscription/module_subscription_actor.rs | 6 +- 10 files changed, 872 insertions(+), 153 deletions(-) create mode 100644 crates/core/src/db/relational_db/shutdown_tests.rs create mode 100644 crates/core/src/host/module_host/drain_tests.rs create mode 100644 crates/core/src/host/module_host/operations.rs diff --git a/crates/core/src/db/durability.rs b/crates/core/src/db/durability.rs index c17a10e9f63..ead4ca83e4e 100644 --- a/crates/core/src/db/durability.rs +++ b/crates/core/src/db/durability.rs @@ -1,15 +1,12 @@ -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; -use log::{error, info}; use spacetimedb_commitlog::payload::{ txdata::{Mutations, Ops}, Txdata, }; use spacetimedb_datastore::{execution_context::ReducerContext, traits::TxData}; use spacetimedb_durability::Transaction; -use spacetimedb_lib::Identity; use spacetimedb_sats::ProductValue; -use tokio::{runtime, time::timeout}; use crate::db::persistence::Durability; @@ -32,21 +29,6 @@ pub(super) fn request_durability( })); } -pub(super) fn spawn_close(durability: Arc, runtime: &runtime::Handle, database_identity: Identity) { - let rt = runtime.clone(); - rt.spawn(async move { - let label = format!("[{database_identity}]"); - match timeout(Duration::from_secs(10), durability.close()).await { - Err(_elapsed) => { - error!("{label} timeout waiting for durability shutdown"); - } - Ok(offset) => { - info!("{label} durability shut down at tx offset: {offset:?}"); - } - } - }); -} - fn prepare_tx_data_for_durability( tx_offset: u64, reducer_context: Option, diff --git a/crates/core/src/db/relational_db.rs b/crates/core/src/db/relational_db.rs index 26bb800ff9d..e192175883f 100644 --- a/crates/core/src/db/relational_db.rs +++ b/crates/core/src/db/relational_db.rs @@ -1,4 +1,4 @@ -use crate::db::durability::{request_durability, spawn_close as spawn_durability_close}; +use crate::db::durability::request_durability; use crate::db::MetricsRecorderQueue; use crate::error::{DBError, RestoreSnapshotError}; use crate::subscription::ExecutionCounters; @@ -6,6 +6,8 @@ use crate::util::asyncify; use crate::worker_metrics::WORKER_METRICS; use anyhow::{anyhow, Context}; use enum_map::EnumMap; +use futures::future::{BoxFuture, Shared}; +use futures::FutureExt; use spacetimedb_commitlog::repo::OnNewSegmentFn; use spacetimedb_commitlog::{self as commitlog, Commitlog, SizeOnDisk}; use spacetimedb_data_structures::map::HashSet; @@ -58,7 +60,9 @@ use spacetimedb_table::table_index::IndexKey; use std::borrow::Cow; use std::io; use std::ops::RangeBounds; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::sync::OnceLock; use tokio::sync::watch; pub use super::persistence::{DiskSizeFn, Durability, Persistence}; @@ -98,6 +102,8 @@ pub struct RelationalDB { hosted_admission: super::hosted_admission::HostedAdmission, inner: Locking, + commits_closed: Arc, + shutdown: OnceLock>>>, durability: Option>, durability_runtime: Option, snapshot_worker: Option, @@ -133,9 +139,12 @@ impl std::fmt::Debug for RelationalDB { impl Drop for RelationalDB { fn drop(&mut self) { - // Attempt to flush the outstanding transactions. - if let (Some(durability), Some(runtime)) = (self.durability.take(), self.durability_runtime.take()) { - spawn_durability_close(durability, &runtime, self.database_identity); + self.hosted_admission.seal(); + if let Some(runtime) = &self.durability_runtime { + // Join the same owned close when a cancelled shutdown waiter was the + // last database owner. Never start a second provider close early. + let close = self.start_shutdown(runtime); + runtime.spawn(close); } } } @@ -155,6 +164,8 @@ impl RelationalDB { Self { inner, + commits_closed: Default::default(), + shutdown: Default::default(), durability, durability_runtime, snapshot_worker, @@ -347,26 +358,41 @@ impl RelationalDB { /// Shut down the database, without dropping it. /// - /// Permanently closes hosted admission on this database object. - /// For a disk database, it also instructs the durability layer to shut down - /// and waits until all outstanding transactions are reported as durable. - /// - /// After calling this method, calling [Self::commit_tx_downgrade] or - /// [Self::commit_tx] will panic. - /// - /// Returns `None` if the database is in-memory only, - /// or nothing has been durably persisted yet. - /// - /// Returns the durable [TxOffset] in a `Some` otherwise. + /// Permanently closes hosted and commit admission, then joins the configured + /// durability writer. Transactions already holding the exclusive transaction + /// lock can finish before closure; later commits roll back with `DatabaseClosed`. + /// Caller cancellation does not cancel the physical close. pub async fn shutdown(&self) -> Option { - // Idle module handles may retain this database after its writer stops. - // They must not retain admission or complete an earlier startup sweep. self.hosted_admission.seal(); - if let Some(durability) = &self.durability { - return durability.close().await; - } - - None + self.start_shutdown(&tokio::runtime::Handle::current()).await + } + + fn start_shutdown(&self, runtime: &tokio::runtime::Handle) -> Shared>> { + self.shutdown + .get_or_init(|| { + let inner = self.inner.clone(); + let closed = self.commits_closed.clone(); + let durability = self.durability.clone(); + // A shutdown owns this task through both lock acquisition and actual + // writer completion. The blocking lock never occupies a runtime worker. + let task = runtime.spawn(async move { + tokio::task::spawn_blocking(move || { + let tx = inner.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + closed.store(true, Ordering::Relaxed); + let _ = inner.rollback_mut_tx(tx); + }) + .await + .expect("database close transaction worker panicked"); + match durability { + Some(durability) => durability.close().await, + None => None, + } + }); + async move { task.await.expect("database writer close task panicked") } + .boxed() + .shared() + }) + .clone() } /// Create any system tables that are missing from the datastore. @@ -827,6 +853,13 @@ impl RelationalDB { tx: MutTx, ) -> Result, TxMetrics, Option)>, DBError> { log::trace!("COMMIT MUT TX"); + // `tx` holds the same exclusive lock used by start_shutdown. The flag + // cannot change between this check and the durability append. + if self.commits_closed.load(Ordering::Relaxed) { + let (_, metrics, reducer) = self.rollback_mut_tx(tx); + self.report_tx_metrics(reducer, None, Some(metrics), None); + return Err(DBError::DatabaseClosed); + } let reducer_context = tx.ctx.reducer_context().cloned(); // TODO: Never returns `None` -- should it? @@ -843,8 +876,15 @@ impl RelationalDB { } #[tracing::instrument(level = "trace", skip_all)] - pub fn commit_tx_downgrade(&self, tx: MutTx, workload: Workload) -> (Arc, TxMetrics, Tx) { + pub fn commit_tx_downgrade(&self, tx: MutTx, workload: Workload) -> Result<(Arc, TxMetrics, Tx), DBError> { log::trace!("COMMIT MUT TX"); + // `tx` holds the same exclusive lock used by start_shutdown. The flag + // cannot change between this check and the durability append. + if self.commits_closed.load(Ordering::Relaxed) { + let (_, metrics, reducer) = self.rollback_mut_tx(tx); + self.report_tx_metrics(reducer, None, Some(metrics), None); + return Err(DBError::DatabaseClosed); + } let reducer_context = tx.ctx.reducer_context().cloned(); let (tx_data, tx_metrics, tx) = self.inner.commit_mut_tx_downgrade_and_then(tx, workload, |tx_data| { @@ -853,7 +893,7 @@ impl RelationalDB { self.maybe_do_snapshot(&tx_data); - (tx_data, tx_metrics, tx) + Ok((tx_data, tx_metrics, tx)) } /// Get the [`DurableOffset`] of this database, or `None` if this is an @@ -2298,6 +2338,9 @@ pub mod tests_utils { } } +#[cfg(test)] +mod shutdown_tests; + #[cfg(test)] mod tests { #![allow(clippy::disallowed_macros)] diff --git a/crates/core/src/db/relational_db/shutdown_tests.rs b/crates/core/src/db/relational_db/shutdown_tests.rs new file mode 100644 index 00000000000..67a3c0d4fd8 --- /dev/null +++ b/crates/core/src/db/relational_db/shutdown_tests.rs @@ -0,0 +1,94 @@ +use super::*; +use crate::db::environment; +use crate::host::module_host::{DatabaseUpdate, EventStatus, ModuleEvent, ModuleFunctionCall}; +use crate::subscription::module_subscription_actor::ModuleSubscriptions; +use tests_utils::TestDB; + +#[test] +fn operation_drain_shutdown_serializes_transactions_and_survives_cancelled_waiter() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let fixture = TestDB::durable().unwrap(); + let db = fixture.db.clone(); + runtime.block_on(async { + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Sql); + environment::set(&db, &mut tx, "BEFORE", "committed").unwrap(); + let closing = tokio::spawn({ + let db = db.clone(); + async move { db.shutdown().await } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while db.shutdown.get().is_none() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + closing.abort(); + assert!(closing.await.unwrap_err().is_cancelled()); + assert!(!db.commits_closed.load(Ordering::Relaxed)); + // This SQL/view transaction acquired the exclusive lock before shutdown. + let (_, _, read) = db.commit_tx_downgrade(tx, Workload::Sql).unwrap(); + let _ = db.release_tx(read); + assert!(db.shutdown().await.is_some()); + assert!(db.commits_closed.load(Ordering::Relaxed)); + for downgrade in [false, true] { + let mut late = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Subscribe); + environment::set(&db, &mut late, "LATE", "must-roll-back").unwrap(); + let error = if downgrade { + db.commit_tx_downgrade(late, Workload::Subscribe).err().unwrap() + } else { + db.commit_tx(late).err().unwrap() + }; + assert!(matches!(error, DBError::DatabaseClosed)); + } + let read = db.begin_tx(Workload::Internal); + assert_eq!(environment::get(&read, "BEFORE").unwrap().as_deref(), Some("committed")); + assert_eq!(environment::get(&read, "LATE").unwrap(), None); + let _ = db.release_tx(read); + }); +} + +#[test] +fn operation_drain_retained_sql_and_subscription_handles_reject_after_writer_close() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let fixture = TestDB::durable().unwrap(); + let db = fixture.db.clone(); + runtime.block_on(async { + let subscriptions = ModuleSubscriptions::for_test_enclosing_runtime(db.clone()); + db.shutdown().await; + let auth = spacetimedb_lib::identity::AuthCtx::new(db.owner_identity(), db.owner_identity()); + for statement in ["SELECT * FROM st_client", "SET env.LATE = 'denied'"] { + let error = crate::sql::execute::run( + db.clone(), + statement.into(), + auth.clone(), + Some(subscriptions.clone()), + None, + &mut vec![], + ) + .await + .err() + .unwrap(); + assert!(matches!(error, DBError::DatabaseClosed), "{error}"); + } + let mut tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Unsubscribe); + environment::set(&db, &mut tx, "LATE", "denied").unwrap(); + let event = ModuleEvent { + timestamp: spacetimedb_lib::Timestamp::now(), + caller_identity: db.owner_identity(), + caller_connection_id: None, + function_call: ModuleFunctionCall::update(), + status: EventStatus::Committed(DatabaseUpdate::default()), + reducer_return_value: None, + energy_quanta_used: crate::energy::EnergyQuanta::ZERO, + host_execution_duration: std::time::Duration::ZERO, + request_id: None, + timer: None, + }; + let error = subscriptions.commit_and_broadcast_event(None, event, tx).err().unwrap(); + assert!(matches!(error, DBError::DatabaseClosed)); + let tx = db.begin_tx(Workload::Internal); + assert_eq!(environment::get(&tx, "LATE").unwrap(), None); + let _ = db.release_tx(tx); + }); +} diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 2a6c328f276..53f7d1f852f 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -89,6 +89,8 @@ impl From for DatabaseError { #[derive(Error, Debug)] pub enum DBError { + #[error("database is closed")] + DatabaseClosed, #[error("LibError: {0}")] Lib(#[from] LibError), #[error("BufferError: {0}")] diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index b41d8ef8a34..58969a6da93 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -76,11 +76,16 @@ use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; use std::fmt; use std::num::NonZeroUsize; -use std::sync::atomic::AtomicBool; use std::sync::{Arc, Weak}; use std::time::{Duration, Instant}; use tokio::sync::{oneshot, OwnedSemaphorePermit, Semaphore}; +#[cfg(test)] +mod drain_tests; +mod operations; +use operations::ModuleOperations; +pub(in crate::host) use operations::OperationLease; + #[derive(Debug, Default, Clone, From)] pub struct DatabaseUpdate { pub tables: SmallVec<[DatabaseTableUpdate; 1]>, @@ -433,6 +438,7 @@ impl WasmtimeModuleHost { label: &str, on_panic: Arc, timer_guard: CallTimerGuard, + operation: OperationLease, arg: A, wasm: impl FnOnce(A, &mut ModuleInstance) + Send + 'static, ) where @@ -440,6 +446,7 @@ impl WasmtimeModuleHost { { let label = label.to_owned(); self.main_executor.enqueue_job(move |state| { + let _operation = operation; scopeguard::defer_on_unwind!({ log::warn!("wasm main operation {label} panicked"); on_panic(); @@ -457,6 +464,7 @@ impl WasmtimeModuleHost { label: &str, on_panic: Arc, timer_guard: CallTimerGuard, + operation: OperationLease, arg: A, wasm: impl AsyncFnOnce(A, &mut ModuleInstance) + Send + 'static, ) where @@ -464,11 +472,12 @@ impl WasmtimeModuleHost { { let instance_manager = self.procedure_instances.clone(); let ModuleInstanceLease { instance, slot } = instance_manager - .get_instance() + .get_instance(Some(operation.clone())) .await .unwrap_or_else(|never| match never {}); let label = label.to_owned(); self.procedure_executor.enqueue_job(async move || { + let _operation = operation; scopeguard::defer_on_unwind!({ log::warn!("wasm procedure {label} panicked"); on_panic(); @@ -492,7 +501,7 @@ struct V8ModuleHost { trait GenericModule { type Instance: GenericModuleInstance; type CreationError; - async fn create_instance(&self) -> Result; + async fn create_instance(&self, operation: Option) -> Result; fn host_type(&self) -> HostType; } @@ -523,7 +532,8 @@ impl GenericModuleInstance for Box { impl GenericModule for Arc { type Instance = Box; type CreationError = std::convert::Infallible; - async fn create_instance(&self) -> Result { + async fn create_instance(&self, operation: Option) -> Result { + let _operation = operation; Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { @@ -534,7 +544,8 @@ impl GenericModule for Arc { impl GenericModule for Arc { type Instance = Box; type CreationError = std::convert::Infallible; - async fn create_instance(&self) -> Result { + async fn create_instance(&self, operation: Option) -> Result { + let _operation = operation; Ok(Box::new((**self).create_instance())) } fn host_type(&self) -> HostType { @@ -545,8 +556,8 @@ impl GenericModule for Arc { impl GenericModule for super::v8::JsModule { type Instance = super::v8::JsProcedureInstance; type CreationError = anyhow::Error; - async fn create_instance(&self) -> Result { - self.create_instance().await + async fn create_instance(&self, operation: Option) -> Result { + self.create_instance_for_operation(operation).await } fn host_type(&self) -> HostType { HostType::Js @@ -1221,7 +1232,7 @@ struct ModuleInstanceManager { struct ModuleInstanceLease { instance: I, - slot: Option, + slot: Option>, } /// Holds the single shared instance used by the JS main execution path. @@ -1394,25 +1405,18 @@ impl ModuleInstanceManager { } } - async fn with_instance( + async fn get_instance( &self, - f: impl AsyncFnOnce(M::Instance) -> (R, M::Instance), - ) -> Result { - let ModuleInstanceLease { instance, slot } = self.get_instance().await?; - let (res, instance) = f(instance).await; - self.return_instance(ModuleInstanceLease { instance, slot }); - Ok(res) - } - - async fn get_instance(&self) -> Result, M::CreationError> { + operation: Option, + ) -> Result, M::CreationError> { let slot = if let Some(instance_slots) = &self.instance_slots { - Some( + Some(Arc::new( instance_slots .clone() .acquire_owned() .await .expect("module instance slot semaphore should not close"), - ) + )) } else { None }; @@ -1425,7 +1429,10 @@ impl ModuleInstanceManager { instance } else { let start_time = std::time::Instant::now(); - let res = self.module.create_instance().await?; + let res = self + .module + .create_instance(operation.map(|operation| operation.with_pool_slot(slot.clone()))) + .await?; let elapsed_time = start_time.elapsed(); self.metrics.observe_instance_created(elapsed_time); res @@ -1470,10 +1477,10 @@ pub struct ModuleHost { /// Called whenever a reducer call on this host panics. on_panic: Arc, - /// Marks whether this module has been closed by [`Self::exit`]. - /// - /// When this is true, most operations will fail with [`NoSuchModule`]. - closed: Arc, + /// Shared admission and physical-operation drainage for this module. + /// [`Self::exit`] closes admission, rejects new work with [`NoSuchModule`], + /// and waits for accepted executor jobs, including cancelled callers. + operations: Arc, } impl fmt::Debug for ModuleHost { @@ -1489,7 +1496,7 @@ pub struct WeakModuleHost { info: Arc, inner: Weak, on_panic: Weak, - closed: Weak, + operations: Weak, } #[derive(Debug)] @@ -1843,7 +1850,7 @@ impl ModuleHost { info, inner, on_panic, - closed: Arc::new(AtomicBool::new(false)), + operations: Arc::new(ModuleOperations::default()), } } @@ -1862,20 +1869,6 @@ impl ModuleHost { matches!(&*self.inner, ModuleHostInner::Js(_)) } - fn is_marked_closed(&self) -> bool { - // `self.closed` isn't used for any synchronization, it's just a shared flag, - // so `Ordering::Relaxed` is sufficient. - self.closed.load(std::sync::atomic::Ordering::Relaxed) - } - - fn guard_closed(&self) -> Result<(), NoSuchModule> { - if self.is_marked_closed() { - Err(NoSuchModule) - } else { - Ok(()) - } - } - fn start_call_timer(&self, label: &str) -> CallTimerGuard { // Record the time until our function starts running. let queue_timer = WORKER_METRICS @@ -1913,7 +1906,7 @@ impl ModuleHost { R: Send + 'static, A: Send + 'static, { - self.guard_closed()?; + let operation = self.operations.begin()?; let timer_guard = self.start_call_timer(label); scopeguard::defer_on_unwind!({ @@ -1926,6 +1919,7 @@ impl ModuleHost { let executor = host.main_executor.clone(); executor .run_job(move |state| { + let _operation = operation; state.with_instance(move |inst| { drop(timer_guard); wasm(arg, inst) @@ -1936,7 +1930,7 @@ impl ModuleHost { ModuleHostInner::Js(host) => { drop(timer_guard); host.main_instance - .with_instance(|inst| async move { js(arg, &inst).await }) + .with_instance(|inst| async move { js(arg, &inst.with_operation(operation)).await }) .await } }) @@ -1957,7 +1951,7 @@ impl ModuleHost { R: Send + 'static, A: Send + 'static, { - self.guard_closed()?; + let operation = self.operations.begin()?; let timer_guard = self.start_call_timer(label); scopeguard::defer_on_unwind!({ @@ -1968,29 +1962,36 @@ impl ModuleHost { Ok(match &*self.inner { ModuleHostInner::Wasm(host) => { let executor = host.procedure_executor.clone(); - let instance_manager = host.procedure_instances.clone(); - instance_manager - .with_instance(async move |mut inst| { - executor - .run_job(async move || { - drop(timer_guard); - let res = wasm(arg, &mut inst).await; - (res, inst) - }) - .await + let manager = host.procedure_instances.clone(); + let ModuleInstanceLease { mut instance, slot } = manager + .get_instance(Some(operation.clone())) + .await + .unwrap_or_else(|never| match never {}); + let operation = operation.with_pool_slot(slot); + executor + .run_job(async move || { + let _operation = operation; + drop(timer_guard); + let result = wasm(arg, &mut instance).await; + manager.return_instance(ModuleInstanceLease { instance, slot: None }); + result }) .await - .unwrap_or_else(|never| match never {}) } - ModuleHostInner::Js(host) => host - .procedure_instances - .with_instance(async |inst| { - drop(timer_guard); - let res = js(arg, &inst).await; - (res, inst) - }) - .await - .map_err(PooledCallError::Startup)?, + ModuleHostInner::Js(host) => { + let mut lease = host + .procedure_instances + .get_instance(Some(operation.clone())) + .await + .map_err(PooledCallError::Startup)?; + lease + .instance + .set_operation(Some(operation.with_pool_slot(lease.slot.take()))); + drop(timer_guard); + let result = js(arg, &lease.instance).await; + self.return_js_procedure_instance(lease); + result + } }) } @@ -2000,7 +2001,7 @@ impl ModuleHost { label: &str, arg: A, js: impl FnOnce(A, JsMainInstance, JsFatalHook) -> JsFut, - wasm: impl FnOnce(A, &WasmtimeModuleHost, JsFatalHook, CallTimerGuard) -> Result<(), NoSuchModule>, + wasm: impl FnOnce(A, &WasmtimeModuleHost, JsFatalHook, CallTimerGuard, OperationLease) -> Result<(), NoSuchModule>, ) -> Result<(), NoSuchModule> where A: Send + 'static, @@ -2012,21 +2013,20 @@ impl ModuleHost { (self.on_panic)(); }); + let operation = self.operations.begin()?; match &*self.inner { ModuleHostInner::Js(js_host) => { - self.guard_closed()?; let on_panic = self.on_panic.clone(); js_host .main_instance - .with_instance(|inst| js(arg, inst, on_panic)) + .with_instance(|inst| js(arg, inst.with_operation(operation), on_panic)) .await; Ok(()) } ModuleHostInner::Wasm(wasm_host) => { - self.guard_closed()?; let timer_guard = self.start_call_timer(label); let on_panic = self.on_panic.clone(); - wasm(arg, wasm_host, on_panic, timer_guard) + wasm(arg, wasm_host, on_panic, timer_guard, operation) } } } @@ -2043,12 +2043,13 @@ impl ModuleHost { label, (cmd, metric), |(cmd, metric), inst, on_panic| async move { inst.enqueue_call_view(cmd, metric, on_panic).await }, - move |(cmd, metric), wasm_host, on_panic, timer_guard| { + move |(cmd, metric), wasm_host, on_panic, timer_guard, operation| { let info = wasm_host.module.info(); wasm_host.enqueue_with_main_instance( label, on_panic, timer_guard, + operation, (cmd, metric), move |(cmd, metric), inst| { let result = inst.call_view(cmd); @@ -2494,11 +2495,12 @@ impl ModuleHost { reducer_name, call.params, |params, inst, on_panic| async move { inst.enqueue_reducer(params, on_panic).await }, - move |params, wasm_host, on_panic, timer_guard| { + move |params, wasm_host, on_panic, timer_guard, operation| { wasm_host.enqueue_with_main_instance( &reducer_label, on_panic, timer_guard, + operation, params, move |params, inst| { let _ = inst.call_reducer(params); @@ -2690,13 +2692,14 @@ impl ModuleHost { (self.on_panic)(); }); - if let Err(err) = self.guard_closed() { - return self.send_procedure_error(&procedure_name, timer, target, err.into()); - } + let operation = match self.operations.begin() { + Ok(operation) => operation, + Err(err) => return self.send_procedure_error(&procedure_name, timer, target, err.into()), + }; match &*self.inner { ModuleHostInner::Js(host) => { - let lease = match host.procedure_instances.get_instance().await { + let mut lease = match host.procedure_instances.get_instance(Some(operation.clone())).await { Ok(lease) => lease, Err(error) => { return self.send_procedure_error( @@ -2707,6 +2710,9 @@ impl ModuleHost { ); } }; + lease + .instance + .set_operation(Some(operation.with_pool_slot(lease.slot.take()))); let call = match lease.instance.enqueue_procedure(params).await { Ok(call) => call, Err(error) => { @@ -2758,6 +2764,7 @@ impl ModuleHost { &procedure_name, on_panic, timer_guard, + operation, params, async move |params, inst| { let ret = inst.call_procedure(params).await; @@ -2777,10 +2784,11 @@ impl ModuleHost { } } - fn return_js_procedure_instance(&self, lease: ModuleInstanceLease) { + fn return_js_procedure_instance(&self, mut lease: ModuleInstanceLease) { let ModuleHostInner::Js(host) = &*self.inner else { return; }; + lease.instance.set_operation(None); host.procedure_instances.return_instance(lease); } @@ -3303,14 +3311,17 @@ impl ModuleHost { } pub async fn exit(&self) { - // As in `Self::marked_closed`, `Relaxed` is sufficient because we're not synchronizing any external state. - self.closed.store(true, std::sync::atomic::Ordering::Relaxed); + // Admission and closure share one lock. Already admitted work retains + // its lease inside the physical executor after cancellation of a caller. + self.operations.close(); self.scheduler().close(); self.exited().await; } pub async fn exited(&self) { self.scheduler().closed().await; + self.operations.close(); + self.operations.drained().await; } pub fn inject_logs(&self, log_level: LogLevel, function_name: &str, message: &str) { @@ -3383,11 +3394,12 @@ impl ModuleHost { label, request, |request, inst, on_panic| async move { inst.enqueue_one_off_query(request, on_panic).await }, - move |request, wasm_host, on_panic, timer_guard| { + move |request, wasm_host, on_panic, timer_guard, operation| { let executor = wasm_host.main_executor.clone(); let info = wasm_host.module.info(); let label = label.to_owned(); executor.enqueue_job(move |_| { + let _operation = operation; scopeguard::defer_on_unwind!({ log::warn!("websocket one-off query operation {label} panicked"); on_panic(); @@ -3662,6 +3674,7 @@ impl ModuleHost { /// for tables without primary keys. It is only used in the benchmarks. /// Note: this doesn't drop the table, it just clears it! pub fn clear_table(&self, table_name: &str) -> Result<(), anyhow::Error> { + let _operation = self.operations.begin()?; let db = self.relational_db(); db.with_auto_commit(Workload::Internal, |tx| { @@ -3683,7 +3696,7 @@ impl ModuleHost { info: self.info.clone(), inner: Arc::downgrade(&self.inner), on_panic: Arc::downgrade(&self.on_panic), - closed: Arc::downgrade(&self.closed), + operations: Arc::downgrade(&self.operations), } } @@ -3722,12 +3735,12 @@ impl WeakModuleHost { pub fn upgrade(&self) -> Option { let inner = self.inner.upgrade()?; let on_panic = self.on_panic.upgrade()?; - let closed = self.closed.upgrade()?; + let operations = self.operations.upgrade()?; Some(ModuleHost { info: self.info.clone(), inner, on_panic, - closed, + operations, }) } } diff --git a/crates/core/src/host/module_host/drain_tests.rs b/crates/core/src/host/module_host/drain_tests.rs new file mode 100644 index 00000000000..50ed2fd2906 --- /dev/null +++ b/crates/core/src/host/module_host/drain_tests.rs @@ -0,0 +1,395 @@ +//! Actual local database writers, Wasm/JS worker queues and cancelled callers. +use super::*; +use crate::db::persistence::LocalPersistenceProvider; +use crate::host::host_controller::{HostController, HostRuntimeConfig}; +use crate::util::jobs::JobCores; +use spacetimedb_auth::identity::SpacetimeIdentityClaims; +use spacetimedb_datastore::system_tables::ModuleKind; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10Builder; +use spacetimedb_paths::{server::ServerDataDir, FromPathUnchecked}; +use tokio::time::timeout; + +fn javascript() -> Program { + let mut definition = RawModuleDefV10Builder::new(); + definition.add_lifecycle_reducer( + Lifecycle::OnDisconnect, + "disconnected", + spacetimedb_sats::ProductType::unit(), + ); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(definition.finish())).unwrap(); + Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{register_hooks}} from "spacetime:sys@1.0"; + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), + __call_reducer__: () => ({{tag:"ok"}}) }}); + "# + ) + .into_bytes(), + ) +} + +fn fixture(id: u64, program: Program) -> (tempfile::TempDir, HostController, Database) { + let directory = tempfile::tempdir().unwrap(); + let data = Arc::new(ServerDataDir::from_path_unchecked(directory.path().to_owned())); + let initial = program.clone(); + let controller = HostController::new( + data.clone(), + crate::db::Config { + storage: crate::db::Storage::Disk, + page_pool_max_size: None, + }, + HostRuntimeConfig::default(), + Arc::new(move |hash| { + let program = initial.clone(); + async move { Ok((program.hash == hash).then_some(program.bytes)) } + }), + Arc::new(crate::energy::NullEnergyMonitor), + Arc::new(LocalPersistenceProvider::new(data)), + JobCores::without_pinned_cores(), + ); + let database = Database { + id, + database_identity: Identity::from_u256(id.into()), + owner_identity: Identity::ONE, + host_type: if program.kind == ModuleKind::JS { + HostType::Js + } else { + HostType::Wasm + }, + initial_program: program.hash, + }; + (directory, controller, database) +} + +async fn until(mut condition: impl FnMut() -> bool) { + timeout(Duration::from_secs(5), async { + while !condition() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +fn client_auth() -> ConnectionAuthCtx { + SpacetimeIdentityClaims { + identity: Identity::ONE, + subject: "local-test".into(), + issuer: "local-test".into(), + audience: Box::new([]), + iat: std::time::SystemTime::now(), + exp: None, + extra: None, + } + .try_into() + .unwrap() +} + +async fn queued_disconnect(js: bool, cancel: bool, id: u64) { + let program = if js { + javascript() + } else { + crate::host::empty_module::program(1).unwrap() + }; + let (_directory, controller, database) = fixture(id, program); + let module = controller + .get_or_launch_module_host(database.clone(), id) + .await + .unwrap(); + let client = ClientActorId::for_test(Identity::ONE); + module + .call_identity_connected(client_auth(), client.connection_id) + .await + .unwrap(); + let db = module.relational_db().clone(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + assert!(tx.st_client_row(client.identity, client.connection_id).is_some()); + let call = tokio::spawn({ + let module = module.clone(); + async move { module.disconnect_client(client).await } + }); + until(|| module.operations.active() == 1).await; + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + if cancel { + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + } else { + drop(call); + } + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + let _ = db.rollback_mut_tx(tx); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let tx = db.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + assert!(tx.st_client_row(client.identity, client.connection_id).is_none()); + let _ = db.rollback_mut_tx(tx); + assert!(module.clear_all_clients().await.is_err()); + let result = crate::sql::execute::run( + db.clone(), + "SELECT * FROM st_client".into(), + AuthCtx::new(db.owner_identity(), db.owner_identity()), + None, + None, + &mut Vec::new(), + ) + .await; + assert!(matches!(result, Err(DBError::DatabaseClosed))); + assert!(controller.get_module_host(id).await.is_err()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_wasm_queued_disconnect_survives_cancellation() { + queued_disconnect(false, true, 0xd100).await; + queued_disconnect(false, false, 0xd101).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_js_queued_disconnect_survives_cancellation() { + queued_disconnect(true, true, 0xd102).await; + queued_disconnect(true, false, 0xd103).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_wasm_procedure_owns_external_wait_and_late_commit() { + let id = 0xd104; + let (_directory, controller, database) = fixture(id, crate::host::empty_module::program(1).unwrap()); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let db = module.relational_db().clone(); + let started = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + let call = tokio::spawn({ + let module = module.clone(); + let db = db.clone(); + let started = started.clone(); + let release = release.clone(); + async move { + module + .call_pooled( + "external-wait-test", + (), + async move |_, _| { + started.add_permits(1); + release.acquire().await.unwrap().forget(); + db.with_auto_commit(Workload::Internal, |tx| { + crate::db::environment::set(&db, tx, "AFTER_IO", "committed").map_err(anyhow::Error::from) + }) + .unwrap(); + }, + async |_, _| unreachable!(), + ) + .await + .unwrap(); + } + }); + timeout(Duration::from_secs(5), started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + release.add_permits(1); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let tx = db.begin_tx(Workload::Internal); + assert_eq!( + crate::db::environment::get(&tx, "AFTER_IO").unwrap().as_deref(), + Some("committed") + ); + let _ = db.release_tx(tx); +} + +/// Opt in only for the existing loopback test feature. Production IP filtering +/// stays enabled. Run with proxy variables cleared, as asserted below. +#[cfg(feature = "allow_loopback_http_for_tests")] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_js_procedure_owns_actual_http_until_late_commit() { + use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for variable in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + assert!( + std::env::var_os(variable).is_none(), + "clear {variable} for this disposable loopback test" + ); + } + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let address = listener.local_addr().unwrap(); + assert!(address.ip().is_loopback()); + let started = Arc::new(Semaphore::new(0)); + let release = Arc::new(Semaphore::new(0)); + let server = tokio::spawn({ + let started = started.clone(); + let release = release.clone(); + async move { + let (mut socket, peer) = listener.accept().await.unwrap(); + assert!(peer.ip().is_loopback()); + let mut request = [0u8; 4096]; + assert!(socket.read(&mut request).await.unwrap() > 0); + started.add_permits(1); + release.acquire().await.unwrap().forget(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + } + }); + let request = bsatn::to_vec(&spacetimedb_lib::http::Request { + method: spacetimedb_lib::http::Method::Get, + headers: std::iter::empty().collect(), + timeout: None, + uri: format!("http://{address}/owned-test"), + version: spacetimedb_lib::http::Version::Http11, + }) + .unwrap(); + let mut schema = RawModuleDefV10Builder::new(); + schema + .build_table_with_new_type("rows", [("value", AlgebraicType::U64)], true) + .finish(); + schema.add_procedure("task", spacetimedb_sats::ProductType::unit(), AlgebraicType::U64); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + let program = Program::from_bytes(ModuleKind::JS, format!(r#" + import {{register_hooks,table_id_from_name,datastore_insert_bsatn}} from "spacetime:sys@1.0"; + import {{register_hooks as procedures,procedure_http_request,procedure_start_mut_tx,procedure_commit_mut_tx}} from "spacetime:sys@1.2"; + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), __call_reducer__: () => ({{tag:"ok"}})}}); + procedures({{__call_procedure__: () => {{ + procedure_http_request(new Uint8Array({request:?}), ""); + procedure_start_mut_tx(); + datastore_insert_bsatn(table_id_from_name("rows"), new Uint8Array([42,0,0,0,0,0,0,0])); + procedure_commit_mut_tx(); + return new Uint8Array(8); + }} }}); + "#).into_bytes()); + let id = 0xd105; + let (_directory, controller, database) = fixture(id, program); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let db = module.relational_db().clone(); + let call = tokio::spawn({ + let module = module.clone(); + async move { + module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + } + }); + timeout(Duration::from_secs(10), started.acquire()) + .await + .unwrap() + .unwrap() + .forget(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(10)).await } + }); + until(|| module.operations.is_closed()).await; + assert_eq!(module.operations.active(), 1); + assert!(!closing.is_finished()); + release.add_permits(1); + timeout(Duration::from_secs(10), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + server.await.unwrap(); + let tx = db.begin_tx(Workload::Internal); + let table = db.table_id_from_name(&tx, "rows").unwrap().unwrap(); + assert_eq!(tx.table_row_count(table), Some(1)); + let _ = db.release_tx(tx); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn operation_drain_cancelled_js_procedure_startup_retains_physical_slot() { + use futures::StreamExt; + let mut schema = RawModuleDefV10Builder::new(); + schema.add_procedure("task", spacetimedb_sats::ProductType::unit(), AlgebraicType::U64); + let raw = bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + let program = Program::from_bytes( + ModuleKind::JS, + format!( + r#" + import {{register_hooks,console_log}} from "spacetime:sys@1.0"; + import {{register_hooks as procedures}} from "spacetime:sys@1.2"; + function startup() {{ console_log(2, "physical-startup-entered"); }} + startup(); + const until = Date.now() + 2000; + while (Date.now() < until) {{}} + register_hooks({{__describe_module__: () => new Uint8Array({raw:?}), __call_reducer__: () => ({{tag:"ok"}})}}); + procedures({{__call_procedure__: () => new Uint8Array(8)}}); + "# + ) + .into_bytes(), + ); + let id = 0xd106; + let (_directory, controller, database) = fixture(id, program); + let module = controller.get_or_launch_module_host(database, id).await.unwrap(); + let ModuleHostInner::Js(host) = &*module.inner else { + unreachable!() + }; + let slots = host.procedure_instances.instance_slots.as_ref().unwrap(); + let maximum = slots.available_permits(); + let mut logs = module.database_logger().tail(Some(0), true).await.unwrap(); + let call = tokio::spawn({ + let module = module.clone(); + async move { + module + .call_procedure(Identity::ONE, None, None, "task", FunctionArgs::Nullary) + .await + } + }); + timeout(Duration::from_secs(5), async { + loop { + let log = logs.next().await.unwrap().unwrap(); + if String::from_utf8_lossy(&log).contains("physical-startup-entered") { + break; + } + } + }) + .await + .unwrap(); + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + assert_eq!(module.operations.active(), 1); + assert_eq!(slots.available_permits(), maximum - 1); + let closing = tokio::spawn({ + let controller = controller.clone(); + async move { controller.exit_module_host(id, Duration::from_secs(5)).await } + }); + until(|| module.operations.is_closed()).await; + assert!(!closing.is_finished()); + timeout(Duration::from_secs(5), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(slots.available_permits(), maximum); +} diff --git a/crates/core/src/host/module_host/operations.rs b/crates/core/src/host/module_host/operations.rs new file mode 100644 index 00000000000..4906f304a5c --- /dev/null +++ b/crates/core/src/host/module_host/operations.rs @@ -0,0 +1,126 @@ +//! Physical operation ownership, independent of an HTTP/WebSocket waiter's lifetime. +use super::NoSuchModule; +use parking_lot::Mutex; +use std::sync::Arc; +use tokio::sync::{Notify, OwnedSemaphorePermit}; + +#[derive(Default)] +pub(super) struct ModuleOperations { + state: Mutex, + drained: Notify, +} + +#[derive(Default)] +struct State { + closed: bool, + active: usize, +} + +impl ModuleOperations { + pub(super) fn begin(self: &Arc) -> Result { + let mut state = self.state.lock(); + if state.closed { + return Err(NoSuchModule); + } + state.active = state.active.checked_add(1).ok_or(NoSuchModule)?; + Ok(OperationLease { + _token: Arc::new(ActiveOperation(self.clone())), + pool_slot: None, + }) + } + + #[cfg(test)] + pub(super) fn active(&self) -> usize { + self.state.lock().active + } + + #[cfg(test)] + pub(super) fn is_closed(&self) -> bool { + self.state.lock().closed + } + + pub(super) fn close(&self) { + self.state.lock().closed = true; + } + + pub(super) async fn drained(&self) { + loop { + let notified = self.drained.notified(); + tokio::pin!(notified); + // Register before reading the counter, including on a multi-threaded runtime. + notified.as_mut().enable(); + if self.state.lock().active == 0 { + return; + } + notified.await; + } + } +} + +/// Clones describe one admitted operation, not additional admissions. A clone +/// moves into the physical job/request before it is enqueued. Cancellation of +/// the caller therefore cannot release admission or a pooled instance's slot. +#[derive(Clone)] +pub(in crate::host) struct OperationLease { + _token: Arc, + pool_slot: Option>, +} + +impl OperationLease { + pub(super) fn with_pool_slot(mut self, slot: Option>) -> Self { + self.pool_slot = slot; + self + } +} + +struct ActiveOperation(Arc); + +impl Drop for ActiveOperation { + fn drop(&mut self) { + let mut state = self.0.state.lock(); + state.active -= 1; + if state.active == 0 { + self.0.drained.notify_waiters(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn close_counts_previously_admitted_work_even_before_enqueue() { + let operations = Arc::new(ModuleOperations::default()); + let operation = operations.begin().unwrap(); + operations.close(); + assert!(operations.begin().is_err()); + let waiter = tokio::spawn({ + let operations = operations.clone(); + async move { operations.drained().await } + }); + tokio::task::yield_now().await; + assert!(!waiter.is_finished()); + drop(operation); + waiter.await.unwrap(); + operations.drained().await; + } + + #[tokio::test] + async fn cancelled_waiter_does_not_release_physical_job_or_pool_slot() { + let operations = Arc::new(ModuleOperations::default()); + let slots = Arc::new(tokio::sync::Semaphore::new(1)); + let caller = operations + .begin() + .unwrap() + .with_pool_slot(Some(Arc::new(slots.clone().acquire_owned().await.unwrap()))); + let physical_job = caller.clone(); + drop(caller); + operations.close(); + assert_eq!(slots.available_permits(), 0); + assert_eq!(operations.state.lock().active, 1); + drop(physical_job); + operations.drained().await; + assert_eq!(slots.available_permits(), 1); + } +} diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index c21139238cd..7f8238c81f2 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -75,6 +75,7 @@ use crate::client::{ClientActorId, MeteredUnboundedReceiver, MeteredUnboundedSen use crate::config::{V8Config, V8HeapPolicyConfig}; use crate::host::host_controller::CallProcedureReturn; use crate::host::instance_env::{ChunkPool, InstanceEnv, TxSlot}; +use crate::host::module_host::OperationLease; use crate::host::module_host::{ call_identity_connected, init_database, ClientConnectedError, HttpHandlerCallError, OneOffQueryRequest, SqlCommand, SqlCommandResult, ViewCommand, ViewCommandMetric, ViewCommandResult, @@ -312,7 +313,10 @@ impl JsModule { self.procedure_instance_pool_size } - async fn create_procedure_instance(&self) -> anyhow::Result { + async fn create_procedure_instance( + &self, + operation: Option, + ) -> anyhow::Result { let program = self.program.clone(); let common = self.common.clone(); let load_balance_guard = self.load_balance_guard.clone(); @@ -329,13 +333,21 @@ impl JsModule { heap_policy, self.execution_timeout, metrics, + operation, ) .await?; Ok(instance) } + pub(in crate::host) async fn create_instance_for_operation( + &self, + operation: Option, + ) -> anyhow::Result { + self.create_procedure_instance(operation).await + } + pub async fn create_instance(&self) -> anyhow::Result { - self.create_procedure_instance().await + self.create_procedure_instance(None).await } } @@ -468,7 +480,8 @@ impl JsInstanceEnv { /// and friends. #[derive(Clone)] pub struct JsMainInstance { - tx: MeteredUnboundedSender, + tx: MeteredUnboundedSender>, + operation: Option, } /// A procedure instance for a [`JsModule`]. @@ -476,7 +489,8 @@ pub struct JsMainInstance { /// Procedure instances are checked out exclusively from the procedure pool and /// only execute procedure-style requests. pub struct JsProcedureInstance { - tx: mpsc::Sender, + tx: mpsc::Sender>, + operation: Option, startup_failure: ProcedureStartupStatus, } @@ -489,13 +503,34 @@ type ProcedureStartupStatus = Arc>; #[error("procedure isolate startup failed: {0}")] pub(in crate::host) struct JsProcedureStartupError(Arc); +struct PhysicalRequest { + request: R, + operation: Option, +} + impl JsMainInstance { + pub(in crate::host) fn with_operation(mut self, operation: OperationLease) -> Self { + self.operation = Some(operation); + self + } + async fn request(&self, request: R) -> R::Response { - send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| request.into_worker_request(reply_tx)).await + send_js_unbounded_request(R::CTX, &self.tx, |reply_tx| PhysicalRequest { + request: request.into_worker_request(reply_tx), + operation: self.operation.clone(), + }) + .await } async fn send_detached_request(&self, ctx: &'static str, request: JsMainWorkerRequest) { - if self.tx.send(request).is_err() { + if self + .tx + .send(PhysicalRequest { + request, + operation: self.operation.clone(), + }) + .is_err() + { panic!("JS worker exited before accepting `{ctx}`"); } } @@ -716,6 +751,10 @@ js_main_request! { } impl JsProcedureInstance { + pub(in crate::host) fn set_operation(&mut self, operation: Option) { + self.operation = operation; + } + pub(in crate::host) fn is_closed(&self) -> bool { self.startup_failure.get().is_some() || self.tx.is_closed() } @@ -725,7 +764,11 @@ impl JsProcedureInstance { ctx: &'static str, request: impl FnOnce(JsReplyTx) -> JsProcedureWorkerRequest, ) -> Result { - send_js_request(ctx, &self.tx, &self.startup_failure, request).await + send_js_request(ctx, &self.tx, &self.startup_failure, |reply_tx| PhysicalRequest { + request: request(reply_tx), + operation: self.operation.clone(), + }) + .await } pub async fn call_procedure(&self, params: CallProcedureParams) -> CallProcedureReturn { @@ -758,7 +801,10 @@ impl JsProcedureInstance { let (reply_tx, reply_rx) = oneshot::channel(); if self .tx - .send(JsProcedureWorkerRequest::CallProcedure { reply_tx, params }) + .send(PhysicalRequest { + request: JsProcedureWorkerRequest::CallProcedure { reply_tx, params }, + operation: self.operation.clone(), + }) .await .is_err() { @@ -810,10 +856,10 @@ where } } -async fn send_js_unbounded_request( +async fn send_js_unbounded_request( ctx: &'static str, - tx: &MeteredUnboundedSender, - request: impl FnOnce(JsReplyTx) -> JsMainWorkerRequest, + tx: &MeteredUnboundedSender, + request: impl FnOnce(JsReplyTx) -> Req, ) -> T { let (reply_tx, reply_rx) = oneshot::channel(); if tx.send(request(reply_tx)).is_err() { @@ -1336,10 +1382,12 @@ async fn spawn_main_instance_worker( heap_policy, execution_timeout, metrics, + None, ) .await } +#[allow(clippy::too_many_arguments)] async fn spawn_procedure_instance_worker( program: Arc, module_or_mcc: Either, @@ -1348,6 +1396,7 @@ async fn spawn_procedure_instance_worker( heap_policy: V8HeapPolicyConfig, execution_timeout: Duration, metrics: InstanceManagerMetrics, + operation: Option, ) -> anyhow::Result<(ModuleCommon, JsProcedureInstance)> { spawn_instance_worker::( program, @@ -1357,6 +1406,7 @@ async fn spawn_procedure_instance_worker( heap_policy, execution_timeout, metrics, + operation, ) .await } @@ -1377,7 +1427,7 @@ trait JsWorkerSpec { fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance; - fn blocking_recv(rx: &mut Self::Receiver) -> Option; + fn blocking_recv(rx: &mut Self::Receiver) -> Option>; fn handle_request( request: Self::Request, @@ -1391,8 +1441,8 @@ trait JsWorkerSpec { impl JsWorkerSpec for MainJsWorker { type Request = JsMainWorkerRequest; type Instance = JsMainInstance; - type Sender = MeteredUnboundedSender; - type Receiver = MeteredUnboundedReceiver; + type Sender = MeteredUnboundedSender>; + type Receiver = MeteredUnboundedReceiver>; const KIND: JsWorkerKind = JsWorkerKind::Main; @@ -1408,10 +1458,10 @@ impl JsWorkerSpec for MainJsWorker { } fn make_instance(tx: Self::Sender, _startup_failure: ProcedureStartupStatus) -> Self::Instance { - JsMainInstance { tx } + JsMainInstance { tx, operation: None } } - fn blocking_recv(rx: &mut Self::Receiver) -> Option { + fn blocking_recv(rx: &mut Self::Receiver) -> Option> { rx.blocking_recv() } @@ -1429,8 +1479,8 @@ impl JsWorkerSpec for MainJsWorker { impl JsWorkerSpec for ProcedureJsWorker { type Request = JsProcedureWorkerRequest; type Instance = JsProcedureInstance; - type Sender = mpsc::Sender; - type Receiver = mpsc::Receiver; + type Sender = mpsc::Sender>; + type Receiver = mpsc::Receiver>; const KIND: JsWorkerKind = JsWorkerKind::Procedure; @@ -1439,10 +1489,14 @@ impl JsWorkerSpec for ProcedureJsWorker { } fn make_instance(tx: Self::Sender, startup_failure: ProcedureStartupStatus) -> Self::Instance { - JsProcedureInstance { tx, startup_failure } + JsProcedureInstance { + tx, + startup_failure, + operation: None, + } } - fn blocking_recv(rx: &mut Self::Receiver) -> Option { + fn blocking_recv(rx: &mut Self::Receiver) -> Option> { rx.blocking_recv() } @@ -1651,6 +1705,7 @@ fn spawn_v8_worker_thread(worker_kind: JsWorkerKind, database_identity: Identity /// /// `load_balance_guard` and `core_pinner` should both be from the same /// [`AllocatedJobCore`], and are used to manage the core pinning of this thread. +#[allow(clippy::too_many_arguments)] async fn spawn_instance_worker( program: Arc, module_or_mcc: Either, @@ -1659,6 +1714,7 @@ async fn spawn_instance_worker( heap_policy: V8HeapPolicyConfig, execution_timeout: Duration, instance_metrics: InstanceManagerMetrics, + operation: Option, ) -> anyhow::Result<(ModuleCommon, W::Instance)> where W: JsWorkerSpec + 'static, @@ -1685,6 +1741,7 @@ where let mut startup_result_tx = Some(result_tx); let mut module_common_for_recreate = None::; + let mut physical_operation = operation; 'worker: loop { let replacing_instance = module_common_for_recreate.is_some(); let generation_start_time = replacing_instance.then(Instant::now); @@ -1793,9 +1850,11 @@ where // // The loop is terminated when the last worker instance handle is dropped. // This will cause channels, scopes, and the isolate to be cleaned up. + physical_operation.take(); let mut requests_since_heap_check = 0u64; let mut last_heap_check_at = Instant::now(); - while let Some(request) = W::blocking_recv(&mut request_rx) { + while let Some(PhysicalRequest { request, operation }) = W::blocking_recv(&mut request_rx) { + physical_operation = operation; core_pinner.pin_if_changed(); let mut outcome = @@ -1826,7 +1885,9 @@ where } match outcome { - WorkerRequestOutcome::Continue => {} + WorkerRequestOutcome::Continue => { + physical_operation.take(); + } WorkerRequestOutcome::RecreateInstance => { instance_metrics.track_instance_removed(); continue 'worker; diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 5c3c8fea4b2..d7b97a479cd 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -12,7 +12,7 @@ use crate::host::module_host::{ ViewOutcome, WasmInstance, }; use crate::host::{ArgsTuple, ModuleHost}; -use crate::subscription::module_subscription_actor::{commit_and_broadcast_event, ModuleSubscriptions}; +use crate::subscription::module_subscription_actor::ModuleSubscriptions; use crate::subscription::module_subscription_manager::TransactionOffset; use crate::subscription::tx::DeltaTx; use anyhow::anyhow; @@ -131,7 +131,7 @@ fn run_inner( None => (tx, false), }; - let (tx_data, tx_metrics_mut, tx) = db.commit_tx_downgrade(tx, Workload::Sql); + let (tx_data, tx_metrics_mut, tx) = db.commit_tx_downgrade(tx, Workload::Sql)?; let (tx_offset_send, tx_offset) = oneshot::channel(); // Release the tx on drop, so that we record metrics @@ -246,7 +246,10 @@ fn run_inner( request_id: None, timer: None, }; - let res = commit_and_broadcast_event(&subs.unwrap(), None, event, tx); + let res = subs + .unwrap() + .commit_and_broadcast_event(None, event, tx)? + .map_err(|_| anyhow!("SQL transaction write conflict"))?; Ok(( SqlResult { tx_offset: res.tx_offset, diff --git a/crates/core/src/subscription/module_subscription_actor.rs b/crates/core/src/subscription/module_subscription_actor.rs index 642c32531fd..7bfb60f960a 100644 --- a/crates/core/src/subscription/module_subscription_actor.rs +++ b/crates/core/src/subscription/module_subscription_actor.rs @@ -1709,7 +1709,7 @@ impl ModuleSubscriptions { // We'll later ensure tx is released/cleaned up once out of scope. let (read_tx, tx_data, tx_metrics_mut) = match &mut event.status { EventStatus::Committed(db_update) => { - let (tx_data, tx_metrics, read_tx) = stdb.commit_tx_downgrade(tx, Workload::Update); + let (tx_data, tx_metrics, read_tx) = stdb.commit_tx_downgrade(tx, Workload::Update)?; *db_update = DatabaseUpdate::from_writes(&tx_data); (read_tx, tx_data, tx_metrics) } @@ -1817,7 +1817,7 @@ impl ModuleSubscriptions { sender: Identity, ) -> Result<(TxGuard, TransactionOffset), DBError> { Self::_unsubscribe_views(&mut tx, view_collector, sender)?; - let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Unsubscribe); + let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Unsubscribe)?; let opts = GuardTxOptions::from_mut(tx_data, tx_metrics_mut); Ok(self.guard_tx(tx, opts)) } @@ -1852,7 +1852,7 @@ impl ModuleSubscriptions { (tx, trapped) = ModuleHost::materialize_views(tx, instance, view_collector, sender, Workload::Subscribe)?; }; - let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Subscribe); + let (tx_data, tx_metrics_mut, tx) = self.relational_db.commit_tx_downgrade(tx, Workload::Subscribe)?; let opts = GuardTxOptions::from_mut(tx_data, tx_metrics_mut); let (a, b) = self.guard_tx(tx, opts); From 15aff92bac7fa2e0e61068d1f49ae1278a3302cd Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 07:45:30 -0400 Subject: [PATCH 15/23] Bound hosted credentials by confirmed elapsed lifetime --- crates/auth/src/hosted.rs | 68 +++++++++- crates/auth/src/hosted/expiration_tests.rs | 136 ++++++++++++++++++++ crates/core/src/client/client_connection.rs | 48 ++++++- 3 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 crates/auth/src/hosted/expiration_tests.rs diff --git a/crates/auth/src/hosted.rs b/crates/auth/src/hosted.rs index 3043ec47952..2f860510801 100644 --- a/crates/auth/src/hosted.rs +++ b/crates/auth/src/hosted.rs @@ -11,7 +11,7 @@ use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, EncodingKey, H use serde::{Deserialize, Serialize}; use spacetimedb_lib::Identity; use std::fmt; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; pub const HOSTED_TOKEN_KIND: &str = "spacetimedb_hosted_v1"; pub const HOSTED_TOKEN_TYPE: &str = "spacetimedb-hosted+jwt"; @@ -58,6 +58,8 @@ pub struct HostedTokenBinding { #[derive(Clone)] pub struct VerifiedHostedAuth { claims: HostedTokenClaims, + // Local lifetime state only. Never serialized into signed claims. + monotonic_deadline: Instant, } impl fmt::Debug for VerifiedHostedAuth { @@ -103,11 +105,57 @@ impl VerifiedHostedAuth { /// Expiry has no positive leeway. This does not replace generation/grant fencing. pub fn check_at(&self, now: SystemTime) -> anyhow::Result<()> { + self.check_at_clocks(now, Instant::now()) + } + + fn check_at_clocks(&self, now: SystemTime, monotonic_now: Instant) -> anyhow::Result<()> { ensure!(now >= self.issued_at(), "hosted credential is not yet valid"); - ensure!(now < self.expires_at(), "hosted credential expired"); + ensure!( + now < self.expires_at() && monotonic_now < self.monotonic_deadline, + "hosted credential expired" + ); Ok(()) } + /// Cap the local lifetime using a fresh authority confirmation. The caller + /// records `confirmation_started` before sending the request and obtains + /// `confirmed_time` from the successful authoritative response. Charging the + /// entire round trip against the signed remaining lifetime is conservative. + /// Neither a receiving clock behind authority nor a later confirmation can + /// extend this proof. The original signed claims are preserved exactly. + pub fn constrain_expiration( + mut self, + confirmed_time: SystemTime, + confirmation_started: Instant, + ) -> anyhow::Result { + let now = Instant::now(); + ensure!(confirmation_started <= now, "invalid hosted confirmation clock"); + self.check_at_clocks(confirmed_time, now)?; + let remaining = self.expires_at().duration_since(confirmed_time)?; + let confirmed_deadline = confirmation_started + .checked_add(remaining) + .context("hosted confirmation deadline overflow")?; + self.monotonic_deadline = self.monotonic_deadline.min(confirmed_deadline); + self.check_at_clocks(confirmed_time, now)?; + Ok(self) + } + + /// Remaining connection lifetime, bounded by both signed wall-clock expiry + /// and the already captured local deadline. Repeated calls never reset it. + pub fn remaining_lifetime(&self, now: SystemTime) -> Duration { + self.remaining_lifetime_at_clocks(now, Instant::now()) + } + + fn remaining_lifetime_at_clocks(&self, now: SystemTime, monotonic_now: Instant) -> Duration { + if self.check_at_clocks(now, monotonic_now).is_err() { + return Duration::ZERO; + } + self.expires_at() + .duration_since(now) + .unwrap_or_default() + .min(self.monotonic_deadline.saturating_duration_since(monotonic_now)) + } + pub fn into_connection_auth(self) -> anyhow::Result { // Keep the actual claims, including the source/target/generation restrictions. // Normalizing JSON whitespace does not alter any signed claim values. @@ -206,6 +254,7 @@ pub fn verify_hosted_token( binding: &HostedTokenBinding, now: SystemTime, ) -> anyhow::Result { + let verification_started = Instant::now(); ensure!(token.len() <= MAX_HOSTED_TOKEN_BYTES, "hosted credential too large"); let header = decode_header(token)?; ensure!(header.alg == Algorithm::ES256, "hosted credential requires ES256"); @@ -222,7 +271,16 @@ pub fn verify_hosted_token( validation.validate_exp = false; let claims = decode::(token, public_key, &validation)?.claims; validate_claims(&claims, trusted_issuer, binding, now)?; - Ok(VerifiedHostedAuth { claims }) + let remaining = (UNIX_EPOCH + Duration::from_secs(claims.expires_at)).duration_since(now)?; + let monotonic_deadline = verification_started + .checked_add(remaining) + .context("hosted credential deadline overflow")?; + let proof = VerifiedHostedAuth { + claims, + monotonic_deadline, + }; + proof.check_at(now)?; + Ok(proof) } /// Mint from the broker's authoritative binding, with no guest-selected sender or generation. @@ -339,3 +397,7 @@ mod identity_hex { Identity::from_hex(value).map_err(serde::de::Error::custom) } } + +#[cfg(test)] +#[path = "hosted/expiration_tests.rs"] +mod expiration_tests; diff --git a/crates/auth/src/hosted/expiration_tests.rs b/crates/auth/src/hosted/expiration_tests.rs new file mode 100644 index 00000000000..1624bfc474f --- /dev/null +++ b/crates/auth/src/hosted/expiration_tests.rs @@ -0,0 +1,136 @@ +use super::*; + +fn proof() -> (VerifiedHostedAuth, SystemTime, Instant) { + let wall = UNIX_EPOCH + Duration::from_secs(1_700_000_000); + let monotonic = Instant::now(); + ( + VerifiedHostedAuth { + claims: HostedTokenClaims { + kind: HOSTED_TOKEN_KIND.into(), + issuer: "platform.test".into(), + subject: Identity::ZERO.to_hex().to_string().into(), + source_database: Identity::ZERO, + target_database: Identity::ZERO, + generation: 1, + grant_revision: 2, + issued_at: 1_700_000_000, + expires_at: 1_700_000_030, + token_id: "expiration-test".into(), + }, + monotonic_deadline: monotonic + Duration::from_secs(30), + }, + wall, + monotonic, + ) +} + +#[test] +fn receiving_clock_behind_control_cannot_extend_confirmed_lifetime() { + let (proof, wall, monotonic) = proof(); + let request_started = monotonic - Duration::from_secs(2); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), request_started) + .unwrap(); + let deadline = request_started + Duration::from_secs(5); + assert_eq!(proof.monotonic_deadline, deadline); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(3), deadline - Duration::from_nanos(1)) + .is_ok()); + assert!(proof.check_at_clocks(wall + Duration::from_secs(3), deadline).is_err()); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(3), deadline), + Duration::ZERO + ); +} + +#[test] +fn confirmation_round_trip_consumes_remaining_signed_lifetime() { + let (proof, wall, monotonic) = proof(); + assert!(proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic - Duration::from_secs(6)) + .is_err()); +} + +#[test] +fn later_confirmation_cannot_extend_a_previous_deadline() { + let (proof, wall, monotonic) = proof(); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic) + .unwrap(); + let first_deadline = proof.monotonic_deadline; + // Even a second response with an earlier authority timestamp cannot extend + // the stricter deadline already held by this authentication proof. + let proof = proof + .constrain_expiration(wall + Duration::from_secs(20), Instant::now()) + .unwrap(); + assert_eq!(proof.monotonic_deadline, first_deadline); +} + +#[test] +fn backward_wall_clock_does_not_restart_the_local_lifetime() { + let (proof, wall, monotonic) = proof(); + let later = monotonic + Duration::from_secs(29); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(1), later), + Duration::from_secs(1) + ); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(1), monotonic + Duration::from_secs(30)) + .is_err()); +} + +#[test] +fn wall_clock_expiration_remains_an_independent_limit() { + let (proof, wall, monotonic) = proof(); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall + Duration::from_secs(29), monotonic), + Duration::from_secs(1) + ); + assert!(proof + .check_at_clocks(wall + Duration::from_secs(30), monotonic) + .is_err()); + assert_eq!( + proof.remaining_lifetime_at_clocks(wall - Duration::from_secs(1), monotonic), + Duration::ZERO + ); +} + +#[test] +fn invalid_confirmation_clocks_are_rejected() { + let (proof, wall, monotonic) = proof(); + assert!(proof + .clone() + .constrain_expiration(wall - Duration::from_secs(1), monotonic) + .is_err()); + assert!(proof + .clone() + .constrain_expiration(wall + Duration::from_secs(30), monotonic) + .is_err()); + assert!(proof + .constrain_expiration(wall, Instant::now() + Duration::from_secs(60)) + .is_err()); +} + +#[test] +fn cloning_and_connection_conversion_preserve_the_cap_and_signed_claims() { + let (proof, wall, monotonic) = proof(); + let original_claims = serde_json::to_value(&proof.claims).unwrap(); + let proof = proof + .constrain_expiration(wall + Duration::from_secs(25), monotonic) + .unwrap(); + let deadline = proof.monotonic_deadline; + let cloned = proof.clone(); + let connection = proof.into_connection_auth().unwrap(); + let connection = connection.clone(); + let retained = connection.hosted.as_ref().unwrap(); + assert_eq!(cloned.monotonic_deadline, deadline); + assert_eq!(retained.monotonic_deadline, deadline); + assert_eq!(connection.claims.exp, Some(wall + Duration::from_secs(30))); + assert_eq!( + serde_json::from_str::(&connection.jwt_payload).unwrap(), + original_claims + ); + assert!(retained + .check_at_clocks(wall + Duration::from_secs(1), deadline) + .is_err()); +} diff --git a/crates/core/src/client/client_connection.rs b/crates/core/src/client/client_connection.rs index 85edfece555..378842f3b29 100644 --- a/crates/core/src/client/client_connection.rs +++ b/crates/core/src/client/client_connection.rs @@ -568,8 +568,7 @@ fn spawn_hosted_connection_watchdog( let Some(proof) = connection.auth.hosted.clone() else { return; }; - let deadline = - tokio::time::Instant::now() + proof.expires_at().duration_since(SystemTime::now()).unwrap_or_default(); + let deadline = tokio::time::Instant::now() + proof.remaining_lifetime(SystemTime::now()); drop(connection); let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -1657,6 +1656,51 @@ mod tests { assert!(actor.await.unwrap_err().is_cancelled()); } + #[tokio::test] + async fn hosted_connection_uses_confirmed_expiry_while_authority_read_is_blocked() { + struct BlockedAuthority; + impl DurableOffsetSupply for BlockedAuthority { + fn durable_offset(&mut self) -> Result, NoSuchModule> { + Ok(None) + } + + fn check_hosted_auth( + &mut self, + _: &VerifiedHostedAuth, + ) -> futures::future::BoxFuture<'static, anyhow::Result<()>> { + Box::pin(std::future::pending()) + } + } + + let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); + set_fence(&db, 1, true); + let (mut sender, _receiver) = ClientConnectionSender::dummy_with_channel( + ClientActorId::for_test(db.database_identity()), + ClientConfig::for_test(), + db.db.clone(), + ); + let proof = hosted_auth(&db, std::time::Duration::from_secs(20)).hosted.unwrap(); + let confirmed_time = proof.expires_at() - std::time::Duration::from_secs(5); + // Authority is ahead of the receiving wall clock. Almost all of the + // five remaining seconds elapsed while its confirmation was in flight. + let started = std::time::Instant::now() - std::time::Duration::from_millis(4_900); + sender.auth = proof + .constrain_expiration(confirmed_time, started) + .unwrap() + .into_connection_auth() + .unwrap(); + let actor = tokio::spawn(std::future::pending::<()>()); + sender.abort_handle = actor.abort_handle(); + let sender = Arc::new(sender); + let watchdog = spawn_hosted_connection_watchdog(Arc::downgrade(&sender), BlockedAuthority, None); + tokio::time::timeout(std::time::Duration::from_secs(2), watchdog) + .await + .unwrap() + .unwrap(); + assert!(sender.is_cancelled()); + assert!(actor.await.unwrap_err().is_cancelled()); + } + #[tokio::test] async fn hosted_idle_connection_rechecks_durable_revocation() { let db = crate::db::relational_db::tests_utils::TestDB::in_memory().unwrap(); From a00f0880d9f21a3a290e9c011959c2a69d48970e Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 08:01:12 -0400 Subject: [PATCH 16/23] Match managed CLI uploads to the real artifact protocol --- crates/cli/src/container/publish/client.rs | 10 +++- crates/cli/src/container/publish/tests.rs | 54 +++++++++++++++++++ crates/cli/src/main.rs | 11 ++++ crates/cli/src/subcommands/publish/managed.rs | 16 +++++- 4 files changed, 88 insertions(+), 3 deletions(-) diff --git a/crates/cli/src/container/publish/client.rs b/crates/cli/src/container/publish/client.rs index 819085c8613..ebea3d20545 100644 --- a/crates/cli/src/container/publish/client.rs +++ b/crates/cli/src/container/publish/client.rs @@ -336,7 +336,7 @@ impl PublisherClient { database: Identity, status: &UploadStatus, ) -> Result { - let next: UploadStatus = json( + let completed: ObjectRef = json( self.request( Method::POST, route( @@ -356,8 +356,14 @@ impl PublisherClient { "artifact upload completion", ) .await?; + // The completion endpoint confirms the immutable object, rather than + // returning session status. The route already binds the original UUID; + // retain that session only after checking the exact digest and size. + ensure!(completed == status.object, "artifact completion descriptor changed"); + let mut next = status.clone(); + next.offset = completed.size; + next.complete = true; next.validate(status.object, Some(status.id))?; - ensure!(next.complete, "artifact completion not confirmed"); Ok(next) } } diff --git a/crates/cli/src/container/publish/tests.rs b/crates/cli/src/container/publish/tests.rs index e47aa9b53bd..6c9d04ce73a 100644 --- a/crates/cli/src/container/publish/tests.rs +++ b/crates/cli/src/container/publish/tests.rs @@ -31,11 +31,13 @@ pub(crate) fn database() -> Identity { pub(crate) struct Behavior { pub permission: bool, pub lose_append: bool, + pub lose_completion: bool, pub lose_submit_after_commit: bool, pub lose_submit_before_commit: bool, pub lose_reservation: bool, pub stale: bool, pub bad_receipt: bool, + pub bad_completion: bool, pub bad_status: bool, pub wrong_reservation: bool, pub deny_preflight: bool, @@ -319,6 +321,16 @@ async fn handler( status.complete = true; } let mut response = status.clone(); + if path.ends_with("/complete") { + if std::mem::take(&mut state.lose_completion) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + let mut object = response.object; + if state.bad_completion { + object.size += 1; + } + return Json(object).into_response(); + } if method == Method::PATCH && std::mem::take(&mut state.lose_append) { return StatusCode::SERVICE_UNAVAILABLE.into_response(); } @@ -425,6 +437,48 @@ async fn mismatched_reservation_or_upload_receipt_never_reaches_admission() { fixture.close().await; } } +#[tokio::test] +async fn completion_descriptor_must_match_before_admission() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().bad_completion = true; + let directory = tempfile::tempdir().unwrap(); + let mut journal = journal(directory.path(), fixture.record(true, false)); + let error = run_now(&fixture.client(), &mut journal).await.unwrap_err(); + assert!(error.to_string().contains("completion descriptor changed")); + assert!(fixture.state.lock().unwrap().submits.is_empty()); + assert!(!journal.record.submitted); + fixture.state.lock().unwrap().bad_completion = false; + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + let status = journal.record.uploads[0].session.as_ref().unwrap(); + assert!(status.complete); + assert_eq!(status.offset, status.object.size); + fixture.close().await; +} + +#[tokio::test] +async fn lost_completion_response_observes_the_same_complete_session() { + let fixture = Fixture::new().await; + fixture.state.lock().unwrap().lose_completion = true; + let directory = tempfile::tempdir().unwrap(); + let mut journal = journal(directory.path(), fixture.record(true, false)); + assert!(matches!( + run_now(&fixture.client(), &mut journal).await.unwrap(), + Outcome::Complete(_) + )); + let session = journal.record.uploads[0].session.as_ref().unwrap(); + assert!(session.complete); + { + let state = fixture.state.lock().unwrap(); + assert_eq!(state.begin_count, 1); + assert_eq!(state.uploads.len(), 1); + assert!(state.uploads[&session.id].0.complete); + } + fixture.close().await; +} + #[tokio::test] async fn wrong_publication_scope_and_denied_upload_are_not_accepted() { for denied in [true, false] { diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 35c1a2bfed8..f1092aa8b62 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -139,3 +139,14 @@ Commands: "#, ) } + +#[cfg(test)] +mod tests { + #[test] + fn managed_publish_help_passes_entrypoint_validation() { + let help = super::get_command() + .try_get_matches_from(["spacetime", "publish", "--help"]) + .unwrap_err(); + assert_eq!(help.kind(), clap::error::ErrorKind::DisplayHelp); + } +} diff --git a/crates/cli/src/subcommands/publish/managed.rs b/crates/cli/src/subcommands/publish/managed.rs index 887497ed000..f1e4bd055f0 100644 --- a/crates/cli/src/subcommands/publish/managed.rs +++ b/crates/cli/src/subcommands/publish/managed.rs @@ -137,9 +137,23 @@ pub(super) fn add_args(mut command: Command) -> Command { .value_name("NAME=FILE") .help("Build secret file, separate from runtime env_keys"), ); - for tool in ["buildctl", "railpack", "skopeo"] { + for (tool, help) in [ + ( + "buildctl", + "Path to the buildctl executable for Dockerfile or Railpack builds", + ), + ( + "railpack", + "Path to the Railpack executable for explicitly selected Railpack builds", + ), + ( + "skopeo", + "Path to the skopeo executable for copying prebuilt OCI images", + ), + ] { command = command.arg( Arg::new(tool) + .help(help) .long(tool) .default_value(tool) .value_parser(clap::value_parser!(PathBuf)), From c9596cc0193b47b2500c3aff4f581da52c55b360 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 08:08:55 -0400 Subject: [PATCH 17/23] Handle shallow JavaScript logging stack traces safely --- .../execution_deadline_tests.rs | 61 +++++++++++++++++++ crates/core/src/host/v8/syscall/common.rs | 20 +++--- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/crates/core/src/host/host_controller/execution_deadline_tests.rs b/crates/core/src/host/host_controller/execution_deadline_tests.rs index 700843c8fdc..edc7dfa9ac2 100644 --- a/crates/core/src/host/host_controller/execution_deadline_tests.rs +++ b/crates/core/src/host/host_controller/execution_deadline_tests.rs @@ -348,3 +348,64 @@ async fn execution_deadline_bounds_startup_description_and_failed_update() { .await .unwrap(); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn javascript_logging_handles_direct_startup_and_preserves_wrapped_call_locations() { + use futures::TryStreamExt as _; + + let mut schema = RawModuleDefV10Builder::new(); + schema.add_reducer("log", ProductType::unit()); + let schema = spacetimedb_lib::bsatn::to_vec(&spacetimedb_lib::RawModuleDef::V10(schema.finish())).unwrap(); + // Direct top-level logging has exactly one JS frame. It previously passed + // index1 to V8's unchecked GetFrame and could crash the entire host process. + let source = format!( + r#"import {{ register_hooks, console_log }} from "spacetime:sys@1.0"; +console_log(2, "direct-startup"); +function wrapped(message) {{ console_log(2, message); }} +wrapped("wrapped-startup"); +register_hooks({{ + __describe_module__: () => new Uint8Array({schema:?}), + __call_reducer__: () => {{ + console_log(2, "direct-reducer"); + wrapped("wrapped-reducer"); + return {{ tag: "ok" }}; + }}, +}}); +"# + ); + let program = Program::from_bytes(ModuleKind::JS, source.into_bytes()); + let (_directory, controller, database) = controller_fixture(0xed08, &program, config()); + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); + assert!(call(&module, "log").await.is_ok()); + let chunks: Vec<_> = module + .database_logger() + .tail(None, false) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + controller + .exit_module_host(database.id, Duration::from_secs(5)) + .await + .unwrap(); + let bytes = chunks.into_iter().flatten().collect::>(); + let records = String::from_utf8(bytes) + .unwrap() + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect::>(); + for (message, line) in [ + ("direct-startup", 2), + ("wrapped-startup", 4), + ("direct-reducer", 8), + ("wrapped-reducer", 9), + ] { + let record = records.iter().find(|record| record["message"] == message).unwrap(); + assert_eq!(record["line_number"], line, "wrong call location for {message}"); + assert!(record["filename"].as_str().is_some_and(|filename| !filename.is_empty())); + } +} diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index c495b7b0c64..38c130fa84d 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -451,13 +451,19 @@ pub fn console_log<'scope>( let mut buf = scratch_buf::<128>(); let msg = msg.to_rust_cow_lossy(scope, &mut buf); - let frame: Local<'_, v8::StackFrame> = v8::StackTrace::current_stack_trace(scope, 2) - .ok_or_else(exception_already_thrown)? - .get_frame(scope, 1) - .ok_or_else(exception_already_thrown)?; + let trace = v8::StackTrace::current_stack_trace(scope, 2).ok_or_else(exception_already_thrown)?; + // The normal bindings add a logging wrapper, but modules may call this + // syscall directly, including from top-level code. V8's GetFrame does not + // check the index in release builds, so never request an absent frame. + let frame = match trace.get_frame_count() { + 0 => None, + 1 => trace.get_frame(scope, 0), + _ => trace.get_frame(scope, 1), + }; + let line_number = frame.map(|frame| frame.get_line_number() as u32); let mut buf = scratch_buf::<32>(); let filename = frame - .get_script_name(scope) + .and_then(|frame| frame.get_script_name(scope)) .map(|s| s.to_rust_cow_lossy(scope, &mut buf)); let level = (level as u8).into(); @@ -471,7 +477,7 @@ pub fn console_log<'scope>( tracing::warn!( "{}:{} {msg}", filename.as_deref().unwrap_or("unknown"), - frame.get_line_number() + line_number.unwrap_or_default() ); })?; @@ -481,7 +487,7 @@ pub fn console_log<'scope>( ts: InstanceEnv::now_for_logging(), target: None, filename: filename.as_deref(), - line_number: Some(frame.get_line_number() as u32), + line_number, function, message: &msg, }; From b8b7539448163e5451f77f4bf58ddb8a2ffd9350 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 08:40:27 -0400 Subject: [PATCH 18/23] Qualify actual local Dockerfile and Railpack builds --- crates/cli/src/container/process.rs | 40 ++ crates/cli/src/container/tests.rs | 12 +- .../container_build_acceptance/README.md | 77 +++ .../container_build_acceptance/acceptance.py | 469 ++++++++++++++++++ .../test_fixture.py | 64 +++ .../container_build_acceptance/tool-lock.json | 31 ++ crates/cli/tests/real_container_builder.rs | 80 +++ 7 files changed, 771 insertions(+), 2 deletions(-) create mode 100644 crates/cli/tests/container_build_acceptance/README.md create mode 100644 crates/cli/tests/container_build_acceptance/acceptance.py create mode 100644 crates/cli/tests/container_build_acceptance/test_fixture.py create mode 100644 crates/cli/tests/container_build_acceptance/tool-lock.json create mode 100644 crates/cli/tests/real_container_builder.rs diff --git a/crates/cli/src/container/process.rs b/crates/cli/src/container/process.rs index ea0a03ea5f6..3d548833178 100644 --- a/crates/cli/src/container/process.rs +++ b/crates/cli/src/container/process.rs @@ -74,6 +74,8 @@ mod local { if let Some(pid) = self.0.take() { match rustix::process::kill_process_group(pid, rustix::process::Signal::KILL) { Ok(()) | Err(rustix::io::Errno::SRCH) => (), + #[cfg(target_os = "macos")] + Err(rustix::io::Errno::PERM) if exited_leader_is_sole_member(pid) => (), Err(error) => return Err(error.into()), } } @@ -101,6 +103,44 @@ mod local { } } } + + #[cfg(target_os = "macos")] + fn exited_leader_is_sole_member(pid: Pid) -> bool { + // XNU excludes zombies from killpg's eligible processes, and reports + // EPERM when only an unreaped leader remains. Do not generalize EPERM: + // prove this exact child has exited and its group contains only it. + // WNOWAIT keeps the PID/PGID pinned across the bounded inventory. + if !matches!( + rustix::process::waitid( + rustix::process::WaitId::Pid(pid), + WaitIdOptions::EXITED | WaitIdOptions::NOWAIT | WaitIdOptions::NOHANG, + ), + Ok(Some(_)) + ) { + return false; + } + #[link(name = "proc")] + unsafe extern "C" { + fn proc_listpgrppids( + pgrpid: std::ffi::c_int, + buffer: *mut std::ffi::c_void, + buffersize: std::ffi::c_int, + ) -> std::ffi::c_int; + } + // libproc returns the number of PIDs copied, including zombies. Two + // slots distinguish the sole leader from any extra/truncated members. + let mut members = [0i32; 2]; + // SAFETY: writable aligned storage and its exact byte size are passed; + // the group ID belongs to the still-unreaped child observed above. + let count = unsafe { + proc_listpgrppids( + pid.as_raw_pid(), + members.as_mut_ptr().cast(), + std::mem::size_of_val(&members) as std::ffi::c_int, + ) + }; + count == 1 && members[0] == pid.as_raw_pid() + } impl Drop for ProcessGroup { fn drop(&mut self) { let _ = self.kill(); diff --git a/crates/cli/src/container/tests.rs b/crates/cli/src/container/tests.rs index 110eaa4acac..6282db120f9 100644 --- a/crates/cli/src/container/tests.rs +++ b/crates/cli/src/container/tests.rs @@ -554,7 +554,14 @@ async fn subprocess_exit_cancellation_caller_drop_and_output_overflow_reap_befor let root = tempfile::tempdir().unwrap(); // These scripts exercise only owned local subprocesses. No Docker, server, // network, saved configuration, or user credentials are involved. - for mode in ["exit", "cancel", "drop", "overflow", "timeout"] { + for mode in [ + "exit", + "exit_without_descendants", + "cancel", + "drop", + "overflow", + "timeout", + ] { let workspace = Arc::new( tempfile::Builder::new() .prefix("fake-tool-") @@ -565,6 +572,7 @@ async fn subprocess_exit_cancellation_caller_drop_and_output_overflow_reap_befor let pid_file = root.path().join(format!("{mode}.pid")); let script = match mode { "exit" => "echo $$ > \"$1\"; sleep 60 & exit 0", + "exit_without_descendants" => "echo $$ > \"$1\"; exit 0", "overflow" => "echo $$ > \"$1\"; yes x", _ => "echo $$ > \"$1\"; sleep 60 & wait", }; @@ -608,7 +616,7 @@ async fn subprocess_exit_cancellation_caller_drop_and_output_overflow_reap_befor .await .unwrap() .unwrap(); - assert_eq!(result.is_ok(), mode == "exit"); + assert_eq!(result.is_ok(), matches!(mode, "exit" | "exit_without_descendants")); } tokio::time::timeout(Duration::from_secs(3), async { while path.exists() { diff --git a/crates/cli/tests/container_build_acceptance/README.md b/crates/cli/tests/container_build_acceptance/README.md new file mode 100644 index 00000000000..fd8954c0f07 --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/README.md @@ -0,0 +1,77 @@ +# Real local container builder acceptance + +This opt-in fixture exercises the CLI with actual BuildKit and explicitly +selected Railpack. It does not connect to a SpacetimeDB server. The resulting +verified OCI layouts can be consumed by the separate managed publication +acceptance test. + +The checked-in tool lock currently supports macOS on arm64. It records official +release download URLs, SHA-256 checksums, and image digests. The fixture downloads +the two tools into its own workspace and extracts only the expected regular +binary. Nothing is installed globally. Release checksum provenance is recorded +in `tool-lock.json`; image pins were verified against the primary registries. + +Build the CLI and deadline harness from the public workspace, without selecting +or connecting to a server: + +```sh +cargo build --locked --offline -p spacetimedb-cli --bin spacetimedb-cli +cargo test --locked --offline -p spacetimedb-cli --test real_container_builder --no-run +``` + +Use the absolute test executable path printed by the second command. Invoke the +script with absolute paths and a new workspace directory: + +```sh +python3 crates/cli/tests/container_build_acceptance/acceptance.py \ + --docker-socket /absolute/path/to/verified/docker-desktop.sock \ + --cli /absolute/path/to/public/target/debug/spacetimedb-cli \ + --deadline-test-binary /absolute/path/to/public/target/debug/deps/real_container_builder-HASH \ + --workspace /private/tmp/new-owned-builder-workspace +``` + +The socket must already be verified as an owned disposable Docker Desktop +endpoint. The script checks its reported host identity again. Every Docker +command specifies that socket and an empty fixture configuration; saved Docker +contexts and registry credentials are not used. The fixture pulls pinned public +images, starts a uniquely named BuildKit container with a dedicated cache volume, +and binds its port to numeric loopback. A private Unix socket forwards only to +that port. The container is limited to two CPUs, 2 GiB of memory and 256 processes. +Container logs rotate at 4 MiB with at most two files. BuildKit requires +privileged execution in this local fixture; the Docker socket +is not mounted into it. This is a trusted-tool test, not a sandbox for arbitrary +build programs. + +The checks cover: + +- A Dockerfile using a build-secret mount and an explicitly selected Railpack + shell-script build. Neither successful nor failed build logs may reveal the + secret, and retained image objects must not contain it. +- Exact digest, size, platform and executable manifest/config/layer closure of + both prepared outputs. A modified retained object is rejected on import. +- Failed builds and failed Railpack detection, with no automatic builder fallback. +- A destination created while a build is running, which must remain untouched. +- SIGINT cancellation of real `buildctl`, followed by positive PID absence and + workspace release. +- The same cleanup on a two-second deadline through the production local process + runner. Only the test runner shortens the normal build deadline. + +Commands have finite deadlines. Teardown stops the proxy, validates the exact +owned container's name and label, waits for its successful synchronous removal, +then removes its cache volume. If the run reply is lost, cleanup looks up only +the original generated name and requires the same ownership label. An ambiguous +daemon answer fails cleanup; an arbitrary inspection error is not treated as +proof of absence. A teardown failure fails the fixture. Downloaded binaries, +diagnostic inputs and the two OCI layouts remain in the private workspace; public image cache entries +are not pruned. `acceptance.json` records the completed checks and the output +paths without credentials or build-secret values. + +The retained layouts establish build and artifact preparation behavior. They do +not establish container execution, readiness, or isolation in the production +runtime. Those are separate Linux supervisor and Kata acceptance boundaries. + +The cleanup failure tests need no Docker daemon or network: + +```sh +python3 -B -m unittest discover -s crates/cli/tests/container_build_acceptance -p test_fixture.py -v +``` diff --git a/crates/cli/tests/container_build_acceptance/acceptance.py b/crates/cli/tests/container_build_acceptance/acceptance.py new file mode 100644 index 00000000000..ebbc575dd6c --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/acceptance.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Explicitly invoked real local-builder acceptance, never part of cargo test. + +Only an explicitly verified Docker Desktop Unix socket is accepted. The CLI +uses a fixture-owned BuildKit Unix proxy and never opens Spacetime credentials. +Downloaded tools and retained OCI output live under a new private workspace. +""" +import argparse +import concurrent.futures +import gzip +import hashlib +import json +import os +from pathlib import Path +import platform +import selectors +import shutil +import shlex +import signal +import socket +import subprocess +import tarfile +import threading +import time +import urllib.request +import uuid + + +LOCK = json.loads(Path(__file__).with_name("tool-lock.json").read_text()) +LIMIT = 32 * 1024 * 1024 + + +def require(condition, message): + if not condition: + raise RuntimeError(message) + + +def environment(root): + return { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(root / "home"), + "TMPDIR": str(root / "tmp"), + } + + +def run(args, env, timeout=120, check=True): + # No shell expansion, inherited proxy/daemon/server variables or credentials. + child = subprocess.Popen(args, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + out, err = child.communicate(timeout=timeout) + except BaseException: + child.kill() + child.communicate(timeout=10) + raise + require(len(out) <= LIMIT and len(err) <= LIMIT, "fixture command output exceeded bound") + if check: + require(child.returncode == 0, + f"fixture command failed: {Path(args[0]).name} (status {child.returncode})") + return child.returncode, out, err + + +def tools(root): + require(platform.system() == "Darwin" and platform.machine() == "arm64", + "the checked-in tool lock supports Darwin arm64") + directory = root / "tools" + directory.mkdir() + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + binaries = {} + for name, pin in LOCK["tools"].items(): + print(f"Downloading and verifying pinned {name} {pin['version']}", flush=True) + with opener.open(pin["url"], timeout=60) as response: + data = response.read(LIMIT + 1) + require(len(data) <= LIMIT, "tool archive exceeded bound") + require(hashlib.sha256(data).hexdigest() == pin["sha256"], "tool archive checksum mismatch") + archive = directory / (name + ".tar.gz") + archive.write_bytes(data) + with tarfile.open(archive) as contents: + matches = [item for item in contents if item.name.removeprefix("./") == pin["member"]] + require(len(matches) == 1 and matches[0].isreg(), "unexpected tool archive member") + require(matches[0].size <= 128 * 1024 * 1024, "tool binary exceeded bound") + binary = directory / name + with contents.extractfile(matches[0]) as source: + binary.write_bytes(source.read()) + binary.chmod(0o700) + binaries[name] = binary + return binaries + + +class UnixProxy: + """Forward only to this fixture's Docker-published numeric-loopback port.""" + def __init__(self, path, port): + self.path, self.port = path, port + self.stop = threading.Event() + self.slots = threading.BoundedSemaphore(32) + self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=32) + self.listener = socket.socket(socket.AF_UNIX) + self.listener.bind(str(path)) + path.chmod(0o600) + self.listener.listen(32) + self.listener.settimeout(0.2) + self.thread = threading.Thread(target=self.accept) + self.thread.start() + + def accept(self): + while not self.stop.is_set(): + try: + client, _ = self.listener.accept() + except TimeoutError: + continue + except OSError: + break + if self.slots.acquire(blocking=False): + self.pool.submit(self.forward, client) + else: + client.close() + + def forward(self, client): + try: + with client, socket.create_connection(("127.0.0.1", self.port), timeout=5) as upstream: + # Bounded socket writes and periodic stop checks; no filesystem + # path or request header can select another destination. + client.settimeout(1) + upstream.settimeout(1) + with selectors.DefaultSelector() as select: + select.register(client, selectors.EVENT_READ, upstream) + select.register(upstream, selectors.EVENT_READ, client) + while not self.stop.is_set(): + for key, _ in select.select(0.2): + data = key.fileobj.recv(64 * 1024) + if not data: + return + key.data.sendall(data) + except (OSError, TimeoutError): + pass + finally: + self.slots.release() + + def close(self): + self.stop.set() + self.listener.close() + self.thread.join(timeout=3) + require(not self.thread.is_alive(), "BuildKit proxy did not stop accepting") + self.pool.shutdown(wait=True) + self.path.unlink() + + +class Builder: + def __init__(self, root, docker_socket, binaries): + self.root, self.binaries = root, binaries + self.env = environment(root) + docker = shutil.which("docker") + require(docker is not None, "Docker executable is required") + self.docker = [docker, "--host", "unix://" + str(docker_socket), + "--config", str(root / "docker-config")] + self.name = "stdb-cli-build-" + uuid.uuid4().hex + self.volume = self.name + "-cache" + self.container = None + self.run_attempted = False + self.proxy = None + self.volume_created = False + + def command(self, *args, timeout=120, check=True): + return run(self.docker + list(args), self.env, timeout, check) + + def start(self): + _, info, _ = self.command("info", "--format", "{{.Name}}|{{.OperatingSystem}}|{{.OSType}}") + require(info.decode().strip() == "docker-desktop|Docker Desktop|linux", + "explicit socket is not the expected local Docker Desktop fixture host") + print("Verified explicit Docker Desktop socket; starting isolated BuildKit", flush=True) + self.command("pull", "--platform", "linux/arm64", LOCK["images"]["buildkit"], timeout=300) + _, raw, _ = self.command("image", "inspect", LOCK["images"]["buildkit"]) + inspected = json.loads(raw)[0] + require(LOCK["images"]["buildkit"] in inspected["RepoDigests"], "pulled BuildKit digest mismatch") + require(inspected["Architecture"] == "arm64", "BuildKit architecture mismatch") + self.command("volume", "create", "--label", "spacetimedb.fixture=" + self.name, self.volume) + self.volume_created = True + # A timed-out CLI may have created the container before losing its + # response. Retain the exact name/label cleanup key before dispatch. + self.run_attempted = True + _, identifier, _ = self.command( + "run", "--detach", "--pull", "never", "--platform", "linux/arm64", + "--name", self.name, "--label", "spacetimedb.fixture=" + self.name, + "--privileged", "--cpus", "2", "--memory", "2g", "--memory-swap", "2g", + "--log-driver", "json-file", "--log-opt", "max-size=4m", "--log-opt", "max-file=2", + "--pids-limit", "256", "--mount", f"type=volume,src={self.volume},dst=/var/lib/buildkit", + "--publish", "127.0.0.1::1234", LOCK["images"]["buildkit"], + "--addr", "tcp://0.0.0.0:1234", "--oci-worker-snapshotter=native") + self.container = identifier.decode().strip() + _, raw, _ = self.command("inspect", self.container) + owned = json.loads(raw)[0] + require(owned["Config"]["Labels"]["spacetimedb.fixture"] == self.name, "container ownership mismatch") + ports = owned["NetworkSettings"]["Ports"]["1234/tcp"] + require(len(ports) == 1 and ports[0]["HostIp"] == "127.0.0.1", "BuildKit is not loopback bound") + self.proxy = UnixProxy(self.root / "buildkit.sock", int(ports[0]["HostPort"])) + deadline = time.monotonic() + 30 + while True: + status, _, _ = run([str(self.binaries["buildctl"]), "--addr", self.endpoint, + "debug", "workers"], self.env, timeout=5, check=False) + if status == 0: + break + require(time.monotonic() < deadline, "BuildKit fixture did not become ready") + time.sleep(0.2) + + @property + def endpoint(self): + return "unix://" + str(self.root / "buildkit.sock") + + def close(self): + errors = [] + if self.proxy: + try: + self.proxy.close() + except Exception as error: + errors.append(str(error)) + if self.run_attempted: + try: + # Inspect only the exact returned ID or generated name. A + # failed/ambiguous inspect is not proof of absence. The label + # must match before deletion, including after a lost run reply. + _, raw, _ = self.command("container", "inspect", self.container or self.name) + objects = json.loads(raw) + require(len(objects) == 1, "ambiguous owned container lookup") + owned = objects[0] + require(owned["Name"] == "/" + self.name and + owned["Config"]["Labels"]["spacetimedb.fixture"] == self.name, + "container cleanup ownership mismatch") + if self.container: + require(owned["Id"] == self.container, "container cleanup ID mismatch") + # A successful synchronous removal is positive completion. + self.command("rm", "--force", "--volumes", owned["Id"]) + except Exception as error: + errors.append(str(error)) + if self.volume_created: + try: + self.command("volume", "rm", self.volume) + except Exception as error: + errors.append(str(error)) + require(not errors, "positive builder teardown failed: " + "; ".join(errors)) + + +def config(project, image): + project.mkdir() + document = { + "database": "local-builder-fixture", + "container": { + "image": image, + "env_keys": ["RUNTIME_ONLY"], + "resources": {"cpu_millicores": 1000, "memory_bytes": 536870912, + "scratch_bytes": 1073741824, "pids_max": 128}, + }, + } + (project / "spacetime.json").write_text(json.dumps(document)) + + +def verify_layout(layout, secret): + metadata = json.loads((layout / "prepared.json").read_text()) + require(secret not in (layout / "prepared.json").read_bytes(), "build secret entered prepared metadata") + objects = {} + for item in metadata["objects"]: + descriptor = item["descriptor"] + path = Path(item["path"]) + require(not path.is_absolute() and ".." not in path.parts, "artifact path escaped layout") + data = (layout / path).read_bytes() + require(len(data) == descriptor["size"], "artifact size mismatch") + require("sha256:" + hashlib.sha256(data).hexdigest() == descriptor["digest"], "artifact hash mismatch") + require(secret not in data, "build secret entered retained image object") + if item["kind"] == "layer" and data[:2] == b"\x1f\x8b": + require(secret not in gzip.decompress(data), "build secret entered retained image layer") + objects[descriptor["digest"]] = (item["kind"], data) + manifest = json.loads(objects[metadata["manifest"]["digest"]][1]) + require(set(objects) == {metadata["manifest"]["digest"], manifest["config"]["digest"], + *(entry["digest"] for entry in manifest["layers"])}, + "prepared closure is not the exact executable manifest/config/layers") + image = json.loads(objects[manifest["config"]["digest"]][1]) + require(image["os"] == "linux" and image["architecture"] == "arm64", "image platform mismatch") + return metadata, image + + +def cases(root, cli, binaries, builder, deadline_test): + secret = ("build-secret-" + uuid.uuid4().hex).encode() + secret_file = root / "build-secret" + secret_file.write_bytes(secret) + secret_file.chmod(0o600) + base = [str(cli), "--root-dir", str(root / "cli-root"), "--config-path", str(root / "unused-config"), + "container", "build", "--platform", "linux/arm64", "--buildctl", str(binaries["buildctl"]), + "--railpack", str(binaries["railpack"]), "--buildkit-host", builder.endpoint] + + def build(project, output, success=True, secrets=False): + args = base + ["--project-path", str(project), "--out-dir", str(output)] + if secrets: + args += ["--build-secret", "BUILD_SENTINEL=" + str(secret_file)] + status, out, err = run(args, environment(root), timeout=1200, check=False) + require(secret not in out + err, "CLI leaked builder secret diagnostics") + if (status == 0) != success: + # Error output has been checked for the generated sentinel and the + # fixture never supplies ordinary or registry credentials. + raise RuntimeError("CLI build outcome mismatch: " + (out + err).decode(errors="replace")[:2048]) + return out + err + + project = root / "dockerfile-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\n" + "RUN --mount=type=secret,id=BUILD_SENTINEL test -s /run/secrets/BUILD_SENTINEL && cat /run/secrets/BUILD_SENTINEL >&2\n" + "WORKDIR /app\nUSER 1001:1001\nENV IMAGE_DEFAULT=retained\n" + 'ENTRYPOINT ["/bin/sh"]\nCMD ["-c", "echo dockerfile-ready"]\n') + output = root / "dockerfile-output" + print("Actual Dockerfile build with explicit secret mount", flush=True) + build(project, output, secrets=True) + metadata, image = verify_layout(output, secret) + require(metadata["container"]["argv"] == ["/bin/sh", "-c", "echo dockerfile-ready"], "argv was not normalized") + require(metadata["container"]["user"] == "1001:1001", "image user was lost") + require(metadata["container"]["working_directory"] == "/app", "image working directory was lost") + require("IMAGE_DEFAULT=retained" in image["config"]["Env"], "image environment defaults were lost") + + project = root / "railpack-project" + config(project, {"build": {"builder": "railpack", "context": "."}}) + (project / "start.sh").write_text("#!/bin/sh\nprintf 'railpack-ready\\n'\n") + (project / "start.sh").chmod(0o700) + output = root / "railpack-output" + print("Actual explicitly selected Railpack build", flush=True) + build(project, output) + verify_layout(output, secret) + + print("Failed secret-using build must not expose logs or publish output", flush=True) + project = root / "failed-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\n" + "RUN --mount=type=secret,id=BUILD_SENTINEL cat /run/secrets/BUILD_SENTINEL >&2; exit 23\n") + output = root / "failed-output" + build(project, output, success=False, secrets=True) + require(not output.exists(), "failed build published output") + + print("Explicit unsupported Railpack detection cannot fall back to Dockerfile", flush=True) + project = root / "unsupported-project" + config(project, {"build": {"builder": "railpack", "context": "."}}) + (project / "Dockerfile").write_text('FROM scratch\nCMD ["would-be-wrong-builder"]\n') + output = root / "unsupported-output" + build(project, output, success=False) + require(not output.exists(), "unsupported Railpack silently used another builder") + + print("Tampering with a retained OCI object must fail before output publication", flush=True) + tampered = root / "tampered-layout" + shutil.copytree(root / "dockerfile-output", tampered) + metadata = json.loads((tampered / "prepared.json").read_text()) + target = next(item for item in metadata["objects"] if item["kind"] == "config") + with (tampered / target["path"]).open("ab") as changed: + changed.write(b" ") + project = root / "tampered-project" + config(project, {"oci_ref": "oci:" + str(tampered)}) + output = root / "tampered-output" + build(project, output, success=False) + require(not output.exists(), "tampered OCI image was accepted") + + print("A path created during the actual build must never be replaced", flush=True) + project = root / "replacement-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\nRUN sleep 3\nCMD [\"/bin/true\"]\n") + output = root / "replacement-output" + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task = pool.submit(build, project, output, False) + deadline = time.monotonic() + 30 + while not list(root.glob(".spacetime-image-*")): + require(not task.done(), "build finished before concurrent replacement could be tested") + require(time.monotonic() < deadline, "build never created its private workspace") + time.sleep(0.01) + output.mkdir() + (output / "sentinel").write_text("must survive") + task.result(timeout=60) + require(list(output.iterdir()) == [output / "sentinel"], "existing output was replaced") + require((output / "sentinel").read_text() == "must survive", "existing output content changed") + + print("SIGINT must cancel the real buildctl and positively reap it", flush=True) + project = root / "cancel-project" + config(project, {"build": {"builder": "dockerfile", "context": "."}}) + (project / "Dockerfile").write_text( + f"FROM {LOCK['images']['alpine']}\nRUN sleep 120\nCMD [\"/bin/true\"]\n") + output = root / "cancel-output" + pid_file = root / "actual-buildctl.pid" + wrapper = root / "record-buildctl" + wrapper.write_text("#!/bin/sh\nprintf '%s\\n' \"$$\" > " + shlex.quote(str(pid_file)) + + "\nexec " + shlex.quote(str(binaries["buildctl"])) + ' "$@"\n') + wrapper.chmod(0o700) + args = list(base) + args[args.index("--buildctl") + 1] = str(wrapper) + args += ["--project-path", str(project), "--out-dir", str(output)] + child = subprocess.Popen(args, env=environment(root), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 30 + while not pid_file.exists(): + require(child.poll() is None, "CLI exited before invoking real buildctl") + require(time.monotonic() < deadline, "real buildctl did not start") + time.sleep(0.01) + tool_pid = int(pid_file.read_text()) + child.send_signal(signal.SIGINT) + out, err = child.communicate(timeout=15) + require(child.returncode != 0 and b"cancelled" in out + err, "CLI did not report cancellation") + require(secret not in out + err, "cancelled CLI leaked a secret") + try: + os.kill(tool_pid, 0) + except ProcessLookupError: + pass + else: + raise RuntimeError("real buildctl PID still exists after acknowledged CLI cancellation") + require(not output.exists(), "cancelled build published output") + finally: + if child.poll() is None: + child.kill() + child.communicate(timeout=10) + require(not list(root.glob(".spacetime-image-*")), "completed CLI left builder workspaces behind") + + print("A short harness deadline must reap real buildctl before releasing its workspace", flush=True) + pid_file.unlink() + deadline_env = environment(root) + deadline_env.update({ + "STDB_BUILDER_CONTEXT": str(project), + "STDB_BUILDER_BUILDCTL": str(wrapper), + "STDB_BUILDER_PID_FILE": str(pid_file), + "STDB_BUILDER_WORKSPACE": str(root), + "STDB_BUILDER_SOCKET": builder.endpoint, + }) + run([str(deadline_test), "--ignored", "--exact", + "actual_buildctl_deadline_reaps_before_workspace_release", "--test-threads=1"], + deadline_env, timeout=30) + return { + "version": 1, + "dockerfile": str(root / "dockerfile-output"), + "railpack": str(root / "railpack-output"), + "tool_lock": LOCK, + "checks": ["actual_dockerfile", "actual_railpack", "secret_logs", "failed_build", + "no_fallback", "tampered_closure", "concurrent_output", "cancel_and_reap", + "deadline_and_reap"], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--docker-socket", type=Path, required=True) + parser.add_argument("--cli", type=Path, required=True) + parser.add_argument("--deadline-test-binary", type=Path, required=True) + parser.add_argument("--workspace", type=Path, required=True) + args = parser.parse_args() + require(args.docker_socket.is_absolute() and args.docker_socket.is_socket(), "explicit Docker Unix socket required") + require(args.cli.is_absolute() and args.cli.is_file(), "absolute prebuilt CLI executable required") + require(args.deadline_test_binary.is_absolute() and args.deadline_test_binary.is_file(), + "absolute prebuilt deadline test executable required") + require(args.workspace.is_absolute() and not args.workspace.exists(), "workspace must be a new absolute directory") + args.workspace.mkdir(mode=0o700) + for child in ["home", "tmp", "docker-config"]: + (args.workspace / child).mkdir(mode=0o700) + # A missing named config proves local build dispatch does not read global + # credentials or open the supplied ordinary configuration path. + binaries = tools(args.workspace) + builder = Builder(args.workspace, args.docker_socket, binaries) + try: + builder.start() + receipt = cases(args.workspace, args.cli, binaries, builder, args.deadline_test_binary) + finally: + builder.close() + (args.workspace / "acceptance.json").write_text(json.dumps(receipt, indent=2)) + print("Builder acceptance passed; retained OCI layouts are ready for managed publication", flush=True) + + +if __name__ == "__main__": + main() diff --git a/crates/cli/tests/container_build_acceptance/test_fixture.py b/crates/cli/tests/container_build_acceptance/test_fixture.py new file mode 100644 index 00000000000..40ed63ea94d --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/test_fixture.py @@ -0,0 +1,64 @@ +"""Owned-resource failure paths, without contacting Docker or any server.""" +import json +import unittest + +from acceptance import Builder + + +class CleanupTests(unittest.TestCase): + def builder(self, reply): + builder = Builder.__new__(Builder) + builder.name = "fixture-owned-name" + builder.container = None # docker run reply was lost + builder.run_attempted = True + builder.proxy = None + builder.volume_created = False + calls = [] + + def command(*args): + calls.append(args) + if args[:2] == ("container", "inspect"): + if isinstance(reply, Exception): + raise reply + return 0, json.dumps(reply).encode(), b"" + return 0, b"", b"" + + builder.command = command + return builder, calls + + def owned(self): + return [{"Name": "/fixture-owned-name", "Id": "owned-id", "Config": { + "Labels": {"spacetimedb.fixture": "fixture-owned-name"}}}] + + def test_lost_run_reply_removes_only_exact_owned_container(self): + builder, calls = self.builder(self.owned()) + builder.close() + self.assertEqual(calls, [ + ("container", "inspect", "fixture-owned-name"), + ("rm", "--force", "--volumes", "owned-id"), + ]) + + def test_wrong_label_never_authorizes_removal(self): + reply = self.owned() + reply[0]["Config"]["Labels"]["spacetimedb.fixture"] = "another-fixture" + builder, calls = self.builder(reply) + with self.assertRaisesRegex(RuntimeError, "ownership mismatch"): + builder.close() + self.assertEqual(len(calls), 1) + + def test_inspect_error_is_incomplete_cleanup(self): + builder, calls = self.builder(RuntimeError("daemon unavailable")) + with self.assertRaisesRegex(RuntimeError, "positive builder teardown failed"): + builder.close() + self.assertEqual(len(calls), 1) + + def test_returned_identifier_must_match_inspected_object(self): + builder, calls = self.builder(self.owned()) + builder.container = "different-id" + with self.assertRaisesRegex(RuntimeError, "ID mismatch"): + builder.close() + self.assertEqual(len(calls), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/crates/cli/tests/container_build_acceptance/tool-lock.json b/crates/cli/tests/container_build_acceptance/tool-lock.json new file mode 100644 index 00000000000..e62b8e64ec2 --- /dev/null +++ b/crates/cli/tests/container_build_acceptance/tool-lock.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "platform": "darwin-arm64", + "tools": { + "buildctl": { + "version": "0.33.0", + "url": "https://github.com/moby/buildkit/releases/download/v0.33.0/buildkit-v0.33.0.darwin-arm64.tar.gz", + "sha256": "730ac4ffd6f4a88dc404fc675aeaf4cfee414915036042847f0a60861cc8790c", + "member": "bin/buildctl", + "source": "https://api.github.com/repos/moby/buildkit/releases/tags/v0.33.0" + }, + "railpack": { + "version": "0.35.0", + "url": "https://github.com/railwayapp/railpack/releases/download/v0.35.0/railpack-v0.35.0-arm64-apple-darwin.tar.gz", + "sha256": "fb4c16d57458eb7868d48ed8a454014ef40716a6c939929d7b7d5986563a0c65", + "member": "railpack", + "source": "https://api.github.com/repos/railwayapp/railpack/releases/tags/v0.35.0" + } + }, + "images": { + "buildkit": "moby/buildkit@sha256:6c2fa84a6b61ccd72899dde4239f8d5717f05f9a8ca6f3cad185fb1a95a94de3", + "buildkit_arm64": "sha256:e8efce994e456acb94944bcc0530b3188478572914f413d4476568d8b63515c6", + "alpine": "alpine@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1", + "railpack_frontend": "ghcr.io/railwayapp/railpack-frontend@sha256:bc73534934e7929ab3dc41765fb7e25c8c69d9be98c43ef8792fea51f65317bd" + }, + "image_sources": [ + "https://registry-1.docker.io/v2/moby/buildkit/manifests/v0.33.0", + "https://registry-1.docker.io/v2/library/alpine/manifests/3.22.1", + "https://ghcr.io/v2/railwayapp/railpack-frontend/manifests/v0.35.0" + ] +} diff --git a/crates/cli/tests/real_container_builder.rs b/crates/cli/tests/real_container_builder.rs new file mode 100644 index 00000000000..cad32c9c63b --- /dev/null +++ b/crates/cli/tests/real_container_builder.rs @@ -0,0 +1,80 @@ +//! Run only from container_build_acceptance/acceptance.py, which owns and +//! verifies the Docker/BuildKit endpoint and all paths below. No saved defaults. +#![cfg(any(target_os = "macos", target_os = "linux"))] + +use anyhow::{ensure, Context, Result}; +use spacetimedb_cli::container::{ + config::ContainerConfig, + prepare_container, + process::{Invocation, LocalRunner, Output, Runner}, + BuildTools, +}; +use spacetimedb_lib::container::ImagePlatform; +use std::{path::PathBuf, time::Duration}; +use tokio_util::sync::CancellationToken; + +struct ShortDeadline; +impl Runner for ShortDeadline { + async fn run(&self, mut invocation: Invocation) -> Result { + // Exercise the production local owner/kill/reap path without making + // acceptance wait for the ordinary thirty-minute build deadline. + invocation.timeout = Duration::from_secs(2); + LocalRunner.run(invocation).await + } +} + +fn input(name: &str) -> Result { + let path = PathBuf::from(std::env::var_os(name).with_context(|| format!("explicit {name} is required"))?); + ensure!(path.is_absolute(), "fixture input must be absolute"); + Ok(path) +} + +#[tokio::test] +#[ignore = "requires the explicitly owned local BuildKit acceptance fixture"] +async fn actual_buildctl_deadline_reaps_before_workspace_release() -> Result<()> { + let context = input("STDB_BUILDER_CONTEXT")?.canonicalize()?; + let tool = input("STDB_BUILDER_BUILDCTL")?.canonicalize()?; + let pid_file = input("STDB_BUILDER_PID_FILE")?; + let workspace = input("STDB_BUILDER_WORKSPACE")?.canonicalize()?; + let endpoint = std::env::var("STDB_BUILDER_SOCKET")?; + ensure!( + endpoint.starts_with("unix:///"), + "explicit local BuildKit Unix socket is required" + ); + let document: serde_json::Value = serde_json::from_slice(&std::fs::read(context.join("spacetime.json"))?)?; + let configuration: ContainerConfig = serde_json::from_value(document["container"].clone())?; + let result = prepare_container( + &configuration, + &context, + ImagePlatform { + os: "linux".into(), + architecture: "arm64".into(), + }, + &BuildTools { + buildctl: tool, + buildkit_host: Some(endpoint), + ..Default::default() + }, + &workspace, + &ShortDeadline, + CancellationToken::new(), + ) + .await; + let error = result.err().context("long-running real build unexpectedly succeeded")?; + ensure!( + error.to_string().contains("build deadline"), + "unexpected build failure: {error:#}" + ); + let pid = std::fs::read_to_string(pid_file)?.trim().parse()?; + let pid = rustix::process::Pid::from_raw(pid).context("invalid recorded builder PID")?; + ensure!( + rustix::process::test_kill_process(pid) == Err(rustix::io::Errno::SRCH), + "real buildctl PID still exists after the deadline returned" + ); + ensure!( + std::fs::read_dir(workspace)?.all(|entry| entry + .is_ok_and(|entry| !entry.file_name().to_string_lossy().starts_with(".spacetime-image-"))), + "deadline returned before its owned workspace was released" + ); + Ok(()) +} From fd35a64db3f0003dced9730cf0993af93e8a22f8 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 09:19:04 -0400 Subject: [PATCH 19/23] Join accepted snapshot work during terminal shutdown --- crates/core/src/db/snapshot.rs | 81 +++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/crates/core/src/db/snapshot.rs b/crates/core/src/db/snapshot.rs index 26e3d8373cf..93e17b79abd 100644 --- a/crates/core/src/db/snapshot.rs +++ b/crates/core/src/db/snapshot.rs @@ -7,7 +7,11 @@ use std::{ }; use anyhow::Context as _; -use futures::{channel::mpsc, StreamExt as _}; +use futures::{ + channel::mpsc, + future::{BoxFuture, Shared}, + FutureExt as _, StreamExt as _, +}; use log::{info, warn}; use parking_lot::RwLock; use prometheus::{Histogram, IntGauge}; @@ -61,6 +65,7 @@ pub struct SnapshotWorker { snapshot_created: watch::Sender, request_snapshot: mpsc::UnboundedSender, snapshot_repository: Arc, + completion: Shared>>>, } impl SnapshotWorker { @@ -86,15 +91,27 @@ impl SnapshotWorker { stats: <_>::default(), }), }; - tokio::spawn(actor.run()); + let task = tokio::spawn(actor.run()); + let completion = async move { task.await.map_err(|error| Arc::::from(error.to_string())) } + .boxed() + .shared(); Self { snapshot_created, request_snapshot: request_tx, snapshot_repository, + completion, } } + /// Permanently close snapshot admission and join all previously accepted + /// snapshot and compression I/O. All clones observe the same completion. + /// Cancelling a waiter does not cancel the worker or its blocking I/O. + pub async fn shutdown(&self) -> Result<(), Arc> { + self.request_snapshot.close_channel(); + self.completion.clone().await + } + /// Finish the initialization of [Self] by passing a [SnapshotDatabaseState], /// or replace the current [SnapshotDatabaseState] with a new one. /// @@ -370,3 +387,63 @@ impl Compressor { } } } + +#[cfg(test)] +mod shutdown_tests { + use super::*; + use spacetimedb_datastore::{ + execution_context::Workload, + system_tables::{StEnvRow, ST_ENV_ID}, + traits::{IsolationLevel, MutTx as _}, + }; + use spacetimedb_paths::{server::SnapshotsPath, FromPathUnchecked}; + use spacetimedb_snapshot::SnapshotRepository; + use spacetimedb_table::page_pool::PagePool; + + #[tokio::test(flavor = "multi_thread")] + async fn terminal_shutdown_drains_accepted_snapshot_after_waiter_cancellation() { + let dir = tempfile::tempdir().unwrap(); + let repository = Arc::new( + SnapshotRepository::open(SnapshotsPath::from_path_unchecked(dir.path()), Identity::ONE, 1).unwrap(), + ); + let datastore = Locking::bootstrap(Identity::ONE, PagePool::new_for_test()).unwrap(); + let mut tx = datastore.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); + tx.insert_via_serialize_bsatn( + ST_ENV_ID, + &StEnvRow { + key: "PERSISTED".into(), + value: "snapshot drain".into(), + }, + ) + .unwrap(); + datastore.commit_mut_tx(tx).unwrap(); + let worker = SnapshotWorker::new(repository.clone(), Compression::Enabled); + worker.set_state(datastore.committed_state.clone()); + let state = datastore.committed_state.write_arc(); + worker.request_snapshot(); + let mut first = tokio::spawn({ + let worker = worker.clone(); + async move { worker.shutdown().await } + }); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut first) + .await + .is_err()); + first.abort(); + assert!(first.await.unwrap_err().is_cancelled()); + let mut second = tokio::spawn({ + let worker = worker.clone(); + async move { worker.shutdown().await } + }); + assert!(tokio::time::timeout(Duration::from_millis(50), &mut second) + .await + .is_err()); + assert!(worker.request_snapshot.unbounded_send(Request::TakeSnapshot).is_err()); + assert_eq!(repository.latest_snapshot().unwrap(), None); + drop(state); + second.await.unwrap().unwrap(); + worker.shutdown().await.unwrap(); + assert_eq!(repository.latest_snapshot().unwrap(), Some(0)); + let snapshot = repository.read_snapshot(0, &PagePool::new_for_test()).unwrap(); + assert_eq!(snapshot.tx_offset, 0); + } +} From 69e474aef4ec589f92af423af83132ab3f10ca73 Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 10:31:14 -0400 Subject: [PATCH 20/23] Remove unrelated C++ header reordering from container hosting changes --- .../internal/autogen/RawMiscModuleExportV9.g.h | 2 +- .../internal/autogen/RawModuleDefV10Section.g.h | 16 ++++++++-------- .../internal/autogen/RawModuleDefV8.g.h | 4 ++-- .../internal/autogen/RawModuleDefV9.g.h | 4 ++-- .../internal/autogen/RawProcedureDefV10.g.h | 2 +- .../internal/autogen/RawProcedureDefV9.g.h | 2 +- .../internal/autogen/RawReducerDefV9.g.h | 2 +- .../internal/autogen/RawTableDefV10.g.h | 6 +++--- .../internal/autogen/RawTableDefV8.g.h | 4 ++-- .../internal/autogen/RawTableDefV9.g.h | 4 ++-- .../include/spacetimedb/internal/v10_builder.h | 1 + .../include/spacetimedb/procedure_context.h | 2 +- 12 files changed, 25 insertions(+), 24 deletions(-) diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h index 654260b3532..494243dd470 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawMiscModuleExportV9.g.h @@ -12,9 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "RawColumnDefaultValueV9.g.h" #include "RawProcedureDefV9.g.h" #include "RawViewDefV9.g.h" -#include "RawColumnDefaultValueV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index d5a27b62571..b435119ba5c 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -12,19 +12,19 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "ExplicitNames.g.h" -#include "RawHttpRouteDefV10.g.h" -#include "RawTypeDefV10.g.h" -#include "Typespace.g.h" -#include "RawReducerDefV10.g.h" #include "RawProcedureDefV10.g.h" -#include "RawViewDefV10.g.h" +#include "CaseConversionPolicy.g.h" #include "RawLifeCycleReducerDefV10.g.h" +#include "RawReducerDefV10.g.h" #include "RawHttpHandlerDefV10.g.h" -#include "RawTableDefV10.g.h" +#include "RawTypeDefV10.g.h" +#include "ExplicitNames.g.h" +#include "RawViewDefV10.g.h" #include "RawScheduleDefV10.g.h" +#include "Typespace.g.h" +#include "RawTableDefV10.g.h" #include "RawRowLevelSecurityDefV9.g.h" -#include "CaseConversionPolicy.g.h" +#include "RawHttpRouteDefV10.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h index f9c5ec18db2..6936f2f32c5 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV8.g.h @@ -12,10 +12,10 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "Typespace.g.h" +#include "MiscModuleExport.g.h" #include "ReducerDef.g.h" #include "TableDesc.g.h" -#include "MiscModuleExport.g.h" +#include "Typespace.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h index a64a4bf380c..cf6881a9bb2 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV9.g.h @@ -13,11 +13,11 @@ #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" #include "RawTableDefV9.g.h" +#include "RawRowLevelSecurityDefV9.g.h" #include "Typespace.g.h" #include "RawReducerDefV9.g.h" -#include "RawRowLevelSecurityDefV9.g.h" -#include "RawMiscModuleExportV9.g.h" #include "RawTypeDefV9.g.h" +#include "RawMiscModuleExportV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h index f316264fc5c..dc84b35e602 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV10.g.h @@ -12,9 +12,9 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" +#include "AlgebraicType.g.h" #include "FunctionVisibility.g.h" #include "ProductType.g.h" -#include "AlgebraicType.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h index a49d9d78970..667d9864a2a 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawProcedureDefV9.g.h @@ -12,8 +12,8 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "AlgebraicType.g.h" #include "ProductType.g.h" +#include "AlgebraicType.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV9.g.h index 964ed98df12..8121773a40d 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawReducerDefV9.g.h @@ -12,8 +12,8 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "ProductType.g.h" #include "Lifecycle.g.h" +#include "ProductType.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h index a2db62eb7d1..46fc7ca6ed1 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV10.g.h @@ -13,11 +13,11 @@ #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" #include "TableAccess.g.h" -#include "RawColumnDefaultValueV10.g.h" -#include "RawIndexDefV10.g.h" +#include "RawSequenceDefV10.g.h" #include "RawConstraintDefV10.g.h" +#include "RawIndexDefV10.g.h" #include "TableType.g.h" -#include "RawSequenceDefV10.g.h" +#include "RawColumnDefaultValueV10.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV8.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV8.g.h index 6b050914df0..4a85aabd2f3 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV8.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV8.g.h @@ -12,10 +12,10 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawColumnDefV8.g.h" +#include "RawIndexDefV8.g.h" #include "RawSequenceDefV8.g.h" #include "RawConstraintDefV8.g.h" -#include "RawIndexDefV8.g.h" +#include "RawColumnDefV8.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h index 619da3f8d24..e817785f690 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawTableDefV9.g.h @@ -12,11 +12,11 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawConstraintDefV9.g.h" #include "TableType.g.h" -#include "RawIndexDefV9.g.h" #include "RawSequenceDefV9.g.h" #include "TableAccess.g.h" +#include "RawIndexDefV9.g.h" +#include "RawConstraintDefV9.g.h" #include "RawScheduleDefV9.g.h" namespace SpacetimeDB::Internal { diff --git a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h index 5b893d9bc6f..e585d7f787a 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/v10_builder.h @@ -688,3 +688,4 @@ V10Builder& getV10Builder(); } // namespace SpacetimeDB #endif // SPACETIMEDB_V10_BUILDER_H + diff --git a/crates/bindings-cpp/include/spacetimedb/procedure_context.h b/crates/bindings-cpp/include/spacetimedb/procedure_context.h index 75f6847c580..b6270deb5af 100644 --- a/crates/bindings-cpp/include/spacetimedb/procedure_context.h +++ b/crates/bindings-cpp/include/spacetimedb/procedure_context.h @@ -108,7 +108,7 @@ struct ProcedureContext { * @code * auto module_id = ctx.database_identity(); * std::string url = "http://localhost:3000/v1/database/" + - * module_id.to_hex_string() + "/schema?version=11"; + * module_id.to_hex_string() + "/schema?version=10"; * @endcode */ Identity database_identity() const { From 5d9915b02e16b746a7751ceceff8d6c08866d19e Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 11:47:27 -0400 Subject: [PATCH 21/23] Remove cold storage cleanup while preserving joined database shutdown --- crates/core/src/host/host_controller.rs | 27 +- .../{retained.rs => lifecycle.rs} | 114 +----- .../{retained_tests.rs => lifecycle_tests.rs} | 338 ++---------------- .../core/src/host/host_controller/registry.rs | 12 +- 4 files changed, 42 insertions(+), 449 deletions(-) rename crates/core/src/host/host_controller/{retained.rs => lifecycle.rs} (50%) rename crates/core/src/host/host_controller/{retained_tests.rs => lifecycle_tests.rs} (50%) diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index b4e8486317d..40d670a8888 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -68,12 +68,12 @@ const IN_MEMORY_DATABASE_LOGGER_MAX_SIZE: u64 = 0x1_000_000; /// A shared mutable cell containing a module host and associated database. type HostCell = Arc>>; +mod lifecycle; mod registry; -mod retained; use registry::{Hosts, Registration}; #[cfg(test)] -mod retained_tests; +mod lifecycle_tests; #[cfg(test)] mod deployment_tests; @@ -123,8 +123,6 @@ pub struct HostController { /// Map of all hosts managed by this controller, /// keyed by replica id. hosts: Hosts, - /// Held by physical cold operations through their joined writer shutdown. - retained_capacity: Arc, /// The root directory for database data. pub data_dir: Arc, /// The default configuration to use for databases created by this @@ -258,7 +256,6 @@ impl HostController { ) -> Self { Self { hosts: <_>::default(), - retained_capacity: retained::capacity(), default_config, program_storage, energy_monitor, @@ -500,9 +497,9 @@ impl HostController { let update_result = match update_result { Ok(result) => result, Err(panic) => { - if let Err(error) = retained::close_host(host).await { - if matches!(error, retained::CloseFailure::WriterUnconfirmed) { - guard.quarantine(None); + if let Err(error) = lifecycle::close_host(host).await { + if matches!(error, lifecycle::CloseFailure::WriterUnconfirmed) { + guard.quarantine(); } return Err(error.into()); } @@ -515,9 +512,9 @@ impl HostController { // executable cannot be retained after activation failure. // Close clients/scheduler and reconstruct from stored program // on the next leader lookup or reconciliation attempt. - if let Err(error) = retained::close_host(host).await { - if matches!(error, retained::CloseFailure::WriterUnconfirmed) { - guard.quarantine(None); + if let Err(error) = lifecycle::close_host(host).await { + if matches!(error, lifecycle::CloseFailure::WriterUnconfirmed) { + guard.quarantine(); } return Err(error.into()); } @@ -672,8 +669,8 @@ impl HostController { ) -> anyhow::Result { let database_identity = database.database_identity; let result = Host::try_init(self, database, replica_id, guard.registration()).await; - if result.as_ref().is_err_and(retained::writer_unconfirmed) { - guard.quarantine(None); + if result.as_ref().is_err_and(lifecycle::writer_unconfirmed) { + guard.quarantine(); } result.with_context(|| format!("failed to init replica {} for {}", replica_id, database_identity)) } @@ -929,7 +926,7 @@ impl Host { let metrics_cleanup = scopeguard::guard(tx_metrics_recorder_task.clone(), |task| task.abort()); let (db, connected_clients, joined) = - retained::open_database(host_controller, &database, replica_id, false, Some(tx_metrics_queue)).await?; + lifecycle::open_database(host_controller, &database, replica_id, Some(tx_metrics_queue)).await?; let initialized = std::panic::AssertUnwindSafe(async { let (mut program, program_needs_init, initial_deployment) = match db.program()? { // Launch module with program from existing database. @@ -1095,7 +1092,7 @@ impl Host { registration.activate(); scheduler_starter.start(&module_host)?; #[cfg(test)] - if retained::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + if lifecycle::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START .lock() .remove(&replica_ctx.database_identity) { diff --git a/crates/core/src/host/host_controller/retained.rs b/crates/core/src/host/host_controller/lifecycle.rs similarity index 50% rename from crates/core/src/host/host_controller/retained.rs rename to crates/core/src/host/host_controller/lifecycle.rs index e661012a716..3c1237035b4 100644 --- a/crates/core/src/host/host_controller/retained.rs +++ b/crates/core/src/host/host_controller/lifecycle.rs @@ -1,6 +1,5 @@ -//! Operation-scoped access to initialized retained storage, without starting a -//! module. Only the configured durability writer is joined on close; shared -//! provider snapshot and archival services retain their existing ownership. +//! Shared physical writer completion for ordinary module initialization and +//! shutdown. Provider snapshot and archival services keep their own ownership. use super::*; use crate::db::persistence::Persistence; @@ -9,7 +8,6 @@ use futures::FutureExt; use spacetimedb_durability::{Close, DurableOffset, PreparedTx}; use std::panic::{resume_unwind, AssertUnwindSafe}; use std::sync::OnceLock; -use tokio::sync::Semaphore; #[derive(Debug, thiserror::Error)] pub(super) enum CloseFailure { @@ -74,13 +72,12 @@ impl Durability for JoinedDurability { } } -/// Shared with ordinary initialization, so a failed normal replay cannot leave -/// its writer shutting down behind a later retained-storage operation. +/// A failed replay joins the same physical writer close before another normal +/// initialization can acquire the replica's canonical registry cell. pub(super) async fn open_database( controller: &HostController, database: &Database, replica_id: u64, - retained_only: bool, tx_metrics_queue: Option, ) -> anyhow::Result<( Arc, @@ -88,7 +85,6 @@ pub(super) async fn open_database( Option>, )> { if matches!(controller.default_config.storage, db::Storage::Memory) { - anyhow::ensure!(!retained_only, "retained storage requires disk persistence"); let (db, clients) = RelationalDB::open( database.database_identity, database.owner_identity, @@ -101,20 +97,6 @@ pub(super) async fn open_database( } let replica_dir = controller.data_dir.replica(replica_id); - if retained_only { - let commit_log = replica_dir.commit_log(); - // Fs::new can create a missing directory. Establish existing history - // before calling either that helper or the configured provider. - asyncify(move || { - anyhow::ensure!(commit_log.is_dir(), "retained database history is absent"); - anyhow::ensure!( - spacetimedb_commitlog::committed_meta(commit_log)?.is_some(), - "retained database history is empty" - ); - Ok::<_, anyhow::Error>(()) - }) - .await?; - } let history = relational_db::local_history(&replica_dir).await?; let mut persistence = controller.persistence.persistence(database, replica_id).await?; let joined = JoinedDurability::wrap(&mut persistence); @@ -138,93 +120,9 @@ pub(super) async fn open_database( } }; let db = Arc::new(db); - if retained_only { - let validation = db - .metadata() - .and_then(|metadata| metadata.ok_or_else(|| anyhow!("retained database is not initialized").into())); - if let Err(error) = validation { - joined.join().await?; - drop(db); - return Err(error.into()); - } - } Ok((db, clients, Some(joined))) } -impl HostController { - /// Access retained initialized database storage without launching user code. - /// - /// This is a trusted host API, not an external authorization endpoint. The - /// caller must confirm current leadership and operation authority. It must - /// not retain or return the supplied database/module handles, spawn work - /// that outlives the returned future, or reenter this replica's controller. - /// A cold open keeps hosted admission closed and never runs initialization, - /// lifecycle reducers, scheduled functions, or module compilation. - /// - /// Caller cancellation does not stop the owned operation. Its finite - /// capacity permit and registry pin remain until its writer has closed. - pub async fn with_retained_database( - &self, - database: Database, - replica_id: u64, - operation: F, - ) -> anyhow::Result - where - T: Send + 'static, - F: FnOnce(Arc, Option) -> Fut + Send + 'static, - Fut: Future> + Send + 'static, - { - let permit = self - .retained_capacity - .clone() - .try_acquire_owned() - .map_err(|_| anyhow!("retained database operation capacity exhausted"))?; - let controller = self.clone(); - tokio::spawn(async move { - let _permit = permit; - let guard = controller - .acquire_write_lock(replica_id) - .await - .map_err(|_| anyhow!("unable to lock retained database"))?; - if let Some(host) = guard.as_ref() { - anyhow::ensure!( - host.replica_ctx.database.database_identity == database.database_identity - && host.replica_ctx.database.owner_identity == database.owner_identity, - "retained database identity mismatch" - ); - let module = host.module.borrow().clone(); - let db = host.replica_ctx.relational_db().clone(); - return operation(db, Some(module)).await; - } - let (db, _clients, joined) = match open_database(&controller, &database, replica_id, true, None).await { - Ok(opened) => opened, - Err(error) => { - if writer_unconfirmed(&error) { - guard.quarantine(Some(_permit)); - } - return Err(error); - } - }; - let result = AssertUnwindSafe(async { operation(db.clone(), None).await }) - .catch_unwind() - .await; - // Keep the guard and permit while waiting for the actual first close - // even if an inner helper or RelationalDB::Drop also requests close. - if let Err(error) = joined.expect("retained open always uses disk persistence").join().await { - guard.quarantine(Some(_permit)); - return Err(error.into()); - } - drop(db); - drop(guard); - match result { - Ok(result) => result, - Err(panic) => resume_unwind(panic), - } - }) - .await? - } -} - pub(super) async fn close_host(host: Host) -> Result<(), CloseFailure> { let module = host.module.borrow().clone(); let info = module.info(); @@ -245,7 +143,3 @@ pub(super) async fn close_host(host: Host) -> Result<(), CloseFailure> { writer.map_err(|_| CloseFailure::WriterUnconfirmed)?; exited.map_err(|_| CloseFailure::ModuleExit) } - -pub(super) fn capacity() -> Arc { - Arc::new(Semaphore::new(2)) -} diff --git a/crates/core/src/host/host_controller/retained_tests.rs b/crates/core/src/host/host_controller/lifecycle_tests.rs similarity index 50% rename from crates/core/src/host/host_controller/retained_tests.rs rename to crates/core/src/host/host_controller/lifecycle_tests.rs index ef7fda18bb7..6cacab25172 100644 --- a/crates/core/src/host/host_controller/retained_tests.rs +++ b/crates/core/src/host/host_controller/lifecycle_tests.rs @@ -161,31 +161,8 @@ async fn wait_for_close(probe: &Probe) { .forget(); } -async fn wait_for_capacity(controller: &HostController) { - timeout(Duration::from_secs(5), async { - while controller.retained_capacity.available_permits() != 2 { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_missing_history_does_not_create_a_replica_or_lookup_program() { - let (_directory, controller, database, probe, lookups) = fixture(0xc001); - assert!(controller - .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) - .await - .is_err()); - assert!(!controller.data_dir.replica(database.id).0.exists()); - assert_eq!(probe.opened.load(Ordering::SeqCst), 0); - assert_eq!(lookups.load(Ordering::SeqCst), 0); - assert!(controller.managed_replicas().is_empty()); -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_idle_module_and_queued_relookup_do_not_break_positive_close() { +async fn lifecycle_idle_module_and_queued_relookup_do_not_break_positive_close() { let (_directory, controller, database, probe, _) = fixture(0xc002); let idle = controller .get_or_launch_module_host(database.clone(), database.id) @@ -223,13 +200,8 @@ async fn retained_idle_module_and_queued_relookup_do_not_break_positive_close() assert_eq!(probe.opened.load(Ordering::SeqCst), 2); assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); assert!(!Arc::ptr_eq(idle.relational_db(), current.relational_db())); - let seen_live = controller - .with_retained_database(database.clone(), database.id, |db, module| async move { - Ok(module.is_some() && db.metadata()?.is_some()) - }) - .await - .unwrap(); - assert!(seen_live); + let seen_live = controller.get_module_host(database.id).await.unwrap(); + assert!(Arc::ptr_eq(seen_live.relational_db(), current.relational_db())); assert_eq!(probe.opened.load(Ordering::SeqCst), 2); controller .exit_module_host(database.id, Duration::from_secs(5)) @@ -241,119 +213,7 @@ async fn retained_idle_module_and_queued_relookup_do_not_break_positive_close() } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_cancelled_call_keeps_capacity_and_writer_until_positive_close() { - let (_directory, controller, database, probe, _) = fixture(0xc003); - closed_seed(&controller, &database, &probe).await; - probe.block_close.store(true, Ordering::SeqCst); - let first = { - let controller = controller.clone(); - let database = database.clone(); - tokio::spawn(async move { - controller - .with_retained_database(database.clone(), database.id, |_, module| async move { - assert!(module.is_none()); - Ok(()) - }) - .await - }) - }; - wait_for_close(&probe).await; - first.abort(); - assert!(first.await.unwrap_err().is_cancelled()); - assert_eq!(controller.retained_capacity.available_permits(), 1); - let second = { - let controller = controller.clone(); - let database = database.clone(); - tokio::spawn(async move { - controller - .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) - .await - }) - }; - timeout(Duration::from_secs(5), async { - while controller.retained_capacity.available_permits() != 0 { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - assert!(controller - .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) - .await - .is_err()); - assert_eq!(probe.opened.load(Ordering::SeqCst), 2); - probe.release_close.add_permits(1); - wait_for_close(&probe).await; - assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); - probe.release_close.add_permits(1); - second.await.unwrap().unwrap(); - wait_for_capacity(&controller).await; - assert_eq!(probe.active.load(Ordering::SeqCst), 0); - assert!(controller.managed_replicas().is_empty()); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_failed_replay_joins_the_first_close_before_retry() { - let (_directory, controller, database, probe, _) = fixture(0xc004); - closed_seed(&controller, &database, &probe).await; - let mut wrong_owner = database.clone(); - wrong_owner.owner_identity = Identity::from_u256(123456_u64.into()); - probe.block_close.store(true, Ordering::SeqCst); - let failed = { - let controller = controller.clone(); - tokio::spawn(async move { - controller - .with_retained_database(wrong_owner.clone(), wrong_owner.id, |_, _| async { Ok(()) }) - .await - }) - }; - wait_for_close(&probe).await; - assert!(!failed.is_finished()); - let next = { - let controller = controller.clone(); - let database = database.clone(); - tokio::spawn(async move { - controller - .with_retained_database(database.clone(), database.id, |db, _| async move { - Ok(db.metadata()?.is_some()) - }) - .await - }) - }; - tokio::task::yield_now().await; - assert_eq!(probe.opened.load(Ordering::SeqCst), 2); - probe.release_close.add_permits(1); - assert!(failed.await.unwrap().is_err()); - wait_for_close(&probe).await; - assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); - probe.release_close.add_permits(1); - assert!(next.await.unwrap().unwrap()); - assert_eq!(probe.active.load(Ordering::SeqCst), 0); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_operation_panic_closes_writer_and_releases_its_pin() { - let (_directory, controller, database, probe, _) = fixture(0xc005); - closed_seed(&controller, &database, &probe).await; - assert!(controller - .with_retained_database(database.clone(), database.id, |_, _| async { - panic!("injected trusted operation panic"); - #[allow(unreachable_code)] - Ok::<_, anyhow::Error>(()) - }) - .await - .is_err()); - assert_eq!(probe.active.load(Ordering::SeqCst), 0); - controller - .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) - .await - .unwrap(); - assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); - assert!(controller.managed_replicas().is_empty()); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_failed_normal_initialization_closes_writer_before_cold_retry() { +async fn lifecycle_failed_normal_initialization_joins_writer_before_retry() { let (_directory, controller, database, probe, _) = fixture(0xc008); closed_seed(&controller, &database, &probe).await; let mut wrong_owner = database.clone(); @@ -373,58 +233,32 @@ async fn retained_failed_normal_initialization_closes_writer_before_cold_retry() let database = database.clone(); tokio::spawn(async move { controller - .with_retained_database(database.clone(), database.id, |_, module| async move { - assert!(module.is_none()); - Ok(()) - }) + .get_or_launch_module_host(database.clone(), database.id) .await }) }; assert!(!failed.is_finished()); probe.release_close.add_permits(1); assert!(failed.await.unwrap().is_err()); - wait_for_close(&probe).await; + let reopened = next.await.unwrap().unwrap(); assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); - probe.release_close.add_permits(1); - next.await.unwrap().unwrap(); - assert_eq!(probe.active.load(Ordering::SeqCst), 0); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_nonempty_history_without_initialized_module_is_rejected() { - let (_directory, controller, database, probe, lookups) = fixture(0xc009); - let (db, _, joined) = retained::open_database(&controller, &database, database.id, false, None) + probe.block_close.store(false, Ordering::SeqCst); + controller + .exit_module_host(database.id, Duration::from_secs(5)) .await .unwrap(); - db.with_auto_commit(Workload::ForTests, |tx| { - crate::db::environment::set(&db, tx, "PENDING", "value") - }) - .unwrap(); - assert!(db.metadata().unwrap().is_none()); - joined.unwrap().close().await; - drop(db); - let result = controller - .with_retained_database(database.clone(), database.id, |_, _| async { - panic!("uninitialized storage must never reach the operation"); - #[allow(unreachable_code)] - Ok::<_, anyhow::Error>(()) - }) - .await; - assert!(result.unwrap_err().to_string().contains("not initialized")); assert_eq!(probe.active.load(Ordering::SeqCst), 0); - assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); - assert_eq!(lookups.load(Ordering::SeqCst), 0); - assert!(controller.managed_replicas().is_empty()); + drop(reopened); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_module_exit_panic_reports_error_after_positive_writer_close_and_allows_reopen() { +async fn lifecycle_module_exit_panic_reports_error_after_positive_writer_close_and_allows_reopen() { let (_directory, controller, database, probe, _) = fixture(0xc00a); let idle = controller .get_or_launch_module_host(database.clone(), database.id) .await .unwrap(); - retained::FAIL_NEXT_MODULE_EXIT + lifecycle::FAIL_NEXT_MODULE_EXIT .lock() .insert(database.database_identity); let error = controller @@ -447,16 +281,18 @@ async fn retained_module_exit_panic_reports_error_after_positive_writer_close_an } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_unconfirmed_writer_close_reports_error_and_quarantines_capacity_and_replica() { +async fn lifecycle_unconfirmed_writer_close_reports_error_and_quarantines_replica() { let (_directory, controller, database, probe, _) = fixture(0xc00b); - closed_seed(&controller, &database, &probe).await; + let module = controller + .get_or_launch_module_host(database.clone(), database.id) + .await + .unwrap(); probe.panic_close.store(true, Ordering::SeqCst); let error = controller - .with_retained_database(database.clone(), database.id, |_, _| async { Ok(()) }) + .exit_module_host(database.id, Duration::from_secs(5)) .await .unwrap_err(); assert!(error.to_string().contains("writer close panicked")); - assert_eq!(controller.retained_capacity.available_permits(), 1); let error = timeout( Duration::from_secs(1), controller.get_or_launch_module_host(database.clone(), database.id), @@ -465,7 +301,7 @@ async fn retained_unconfirmed_writer_close_reports_error_and_quarantines_capacit .unwrap() .unwrap_err(); assert!(error.to_string().contains("unable to lock")); - assert_eq!(probe.opened.load(Ordering::SeqCst), 2); + assert_eq!(probe.opened.load(Ordering::SeqCst), 1); assert!(controller .exit_module_host(database.id, Duration::from_secs(1)) .await @@ -473,12 +309,13 @@ async fn retained_unconfirmed_writer_close_reports_error_and_quarantines_capacit .to_string() .contains("quarantined")); assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); + drop(module); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_panic_callback_before_initial_host_install_still_owns_cleanup() { +async fn lifecycle_panic_callback_before_initial_host_install_still_owns_cleanup() { let (_directory, controller, database, probe, _) = fixture(0xc00c); - retained::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START + lifecycle::PANIC_CALLBACK_AT_INITIAL_SCHEDULER_START .lock() .insert(database.database_identity); let idle = controller @@ -507,7 +344,7 @@ async fn retained_panic_callback_before_initial_host_install_still_owns_cleanup( } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_stale_panic_callback_does_not_unregister_updated_or_reopened_host() { +async fn lifecycle_stale_panic_callback_does_not_unregister_updated_or_reopened_host() { let (_directory, controller, database, probe, _) = fixture(0xc006); controller .get_or_launch_module_host(database.clone(), database.id) @@ -558,132 +395,3 @@ async fn retained_stale_panic_callback_does_not_unregister_updated_or_reopened_h .unwrap(); assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); } - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn retained_cold_snapshot_cleanup_never_executes_stored_javascript_or_opens_admission() { - use crate::db::deployment::{install_container_fence, install_publication_fence, record_deployment_commit}; - use crate::host::container_environment; - use spacetimedb_datastore::system_tables::StContainerFenceRow; - use spacetimedb_lib::container::*; - use spacetimedb_lib::container_environment::EnvironmentSnapshotScope; - use spacetimedb_lib::deployment::{DeploymentSpec, DeploymentSpecV1, ModuleComponent, UserModule, UserModuleKind}; - use spacetimedb_lib::{hash_bytes, Uuid}; - - let (_directory, controller, database, probe, lookups) = fixture(0xc007); - let idle = controller - .get_or_launch_module_host(database.clone(), database.id) - .await - .unwrap(); - let hostile = Program::from_bytes( - ModuleKind::JS, - b"throw new Error('cold storage must never execute this module');".to_vec(), - ); - let uuid = || Uuid::from_u128(uuid::Uuid::now_v7().as_u128()); - let spec = ContainerSpec { - image_manifest: OciDigest::sha256([7; 32]), - image_platform: ImagePlatform { - os: "linux".into(), - architecture: "arm64".into(), - }, - argv: vec!["/app/agent".into()], - user: "1000:1000".into(), - working_directory: "/app".into(), - mode: ContainerMode::Job, - restart: RestartPolicy::Never, - env_keys: vec!["SECRET".into()], - resources: ContainerResources { - cpu_millicores: 1000, - memory_bytes: 64 * 1024 * 1024, - scratch_bytes: 64 * 1024 * 1024, - pids_max: 64, - }, - ports: vec![], - mounts: vec![], - stop_grace_ms: DEFAULT_STOP_GRACE_MS, - }; - let publication = DeploymentCommit { - operation_id: uuid(), - publication_epoch: 1, - publisher: database.owner_identity, - expected_revision: None, - prepared_manifest_hash: hash_bytes(b"cold cleanup fixture"), - deployment: DeploymentSpec::V1(DeploymentSpecV1 { - module: ModuleComponent::User(UserModule { - kind: UserModuleKind::Js, - program_hash: hostile.hash, - }), - container: Some(spec.clone()), - }), - }; - let fence = StContainerFenceRow { - source_identity: database.database_identity.into(), - generation: 1, - target_grant_revision: 1, - target_set_hash: hash_bytes(b"targets"), - allowed: true, - }; - idle.relational_db() - .with_auto_commit(Workload::ForTests, |tx| -> anyhow::Result<()> { - idle.relational_db().update_program(tx, hostile)?; - install_publication_fence(tx, publication.publication_epoch, publication.operation_id)?; - record_deployment_commit(tx, &publication, Timestamp::now(), &Default::default())?; - install_container_fence(idle.relational_db(), tx, &fence)?; - crate::db::environment::set(idle.relational_db(), tx, "SECRET", "retained-fixture-secret")?; - Ok(()) - }) - .unwrap(); - let scope = EnvironmentSnapshotScope { - cluster: "local-test".into(), - database_id: database.id, - database_identity: database.database_identity, - node_id: 2, - node_incarnation: uuid(), - generation: 1, - deployment_revision: publication.deployment.revision().unwrap(), - start_request: publication.operation_id, - env_generation: uuid(), - env_keys: spec.env_keys, - }; - let receipt = container_environment::capture(idle.relational_db().clone(), scope.clone()) - .await - .unwrap(); - idle.relational_db() - .hosted_admission() - .begin() - .unwrap() - .complete() - .unwrap(); - controller - .exit_module_host(database.id, Duration::from_secs(5)) - .await - .unwrap(); - let expected_scope = scope.clone(); - controller - .with_retained_database(database.clone(), database.id, move |db, module| async move { - assert!(module.is_none()); - assert!(!db.hosted_admission().is_open()); - let values = container_environment::read(db.clone(), receipt.receipt).await?; - assert_eq!(values.receipt.selected_values["SECRET"], "retained-fixture-secret"); - db.with_auto_commit(Workload::ForTests, |tx| { - install_container_fence( - &db, - tx, - &StContainerFenceRow { - generation: 2, - allowed: false, - ..fence - }, - ) - })?; - container_environment::close(db, expected_scope, 2).await?; - Ok(()) - }) - .await - .unwrap(); - assert!(controller.get_module_host(database.id).await.is_err()); - assert_eq!(lookups.load(Ordering::SeqCst), 1); - assert_eq!(probe.maximum.load(Ordering::SeqCst), 1); - assert_eq!(probe.active.load(Ordering::SeqCst), 0); - assert_eq!(controller.retained_capacity.available_permits(), 2); - drop(idle); -} diff --git a/crates/core/src/host/host_controller/registry.rs b/crates/core/src/host/host_controller/registry.rs index 3b4ababd3f6..030a1d7c3fe 100644 --- a/crates/core/src/host/host_controller/registry.rs +++ b/crates/core/src/host/host_controller/registry.rs @@ -26,10 +26,6 @@ pub(super) struct Entry { /// caller's wait expired. Provider-owned snapshot/archival services are separate. pub(super) struct Closing { result: watch::Sender>>>, - /// An unconfirmed cold writer remains charged to the finite capacity. - /// Recovery requires positive external repair or process termination; - /// recreating a controller is not proof that an old writer has stopped. - _retained_capacity: Option, } impl Closing { @@ -183,7 +179,7 @@ impl WriteGuard { self.pin.as_ref().unwrap().publish(self.as_ref()); } - pub fn quarantine(&self, capacity: Option) { + pub fn quarantine(&self) { let pin = self.pin.as_ref().unwrap(); let Some(hosts) = pin.hosts.upgrade() else { return }; let mut entries = hosts.lock(); @@ -197,7 +193,6 @@ impl WriteGuard { result: watch::Sender::new(Some(Err( "storage writer close is unconfirmed; replica is quarantined".into() ))), - _retained_capacity: capacity, })); } } @@ -288,7 +283,6 @@ fn request_close(hosts: &Hosts, replica: u64, entry: &mut Entry) -> CloseRequest } let completion = Arc::new(Closing { result: watch::Sender::new(None), - _retained_capacity: None, }); entry.closing = Some(completion.clone()); entry.pins += 1; @@ -314,10 +308,10 @@ impl CloseOwner { return; } let result = match guard.take() { - Some(host) => super::retained::close_host(host).await, + Some(host) => super::lifecycle::close_host(host).await, None => Ok(()), }; - let writer_closed = !matches!(result, Err(super::retained::CloseFailure::WriterUnconfirmed)); + let writer_closed = !matches!(result, Err(super::lifecycle::CloseFailure::WriterUnconfirmed)); // Publish under the cell lock, then release it before final registry // unpin/removal. Closing stays set during both steps, so reopen waits. self.pin.publish(None); From b1c6d69b7a3d812bfe5311557f924282d96d9a3a Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 11:47:41 -0400 Subject: [PATCH 22/23] Expose validated container URL discovery in the CLI --- crates/cli/src/lib.rs | 12 +- crates/cli/src/subcommands/container.rs | 19 +- crates/cli/src/subcommands/container/url.rs | 172 +++++++++++++++ .../src/subcommands/container/url/tests.rs | 198 ++++++++++++++++++ crates/cli/src/subcommands/subscribe.rs | 39 ++-- crates/cli/src/util.rs | 1 - crates/lib/src/container.rs | 3 + crates/lib/src/container/endpoints.rs | 57 +++++ 8 files changed, 477 insertions(+), 24 deletions(-) create mode 100644 crates/cli/src/subcommands/container/url.rs create mode 100644 crates/cli/src/subcommands/container/url/tests.rs create mode 100644 crates/lib/src/container/endpoints.rs diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 9340d6cc66e..aec1536f28e 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -48,8 +48,14 @@ pub fn get_subcommands() -> Vec { /// server settings or credentials. Future container network commands use the /// ordinary authenticated dispatcher below. pub async fn exec_local_subcommand(cmd: &str, args: &ArgMatches) -> Option> { - if cmd == "container" && args.subcommand_name() == Some("build") { - Some(subcommands::container::exec(args).await.map(|()| ExitCode::SUCCESS)) + if cmd == "container" + && let Some(("build", args)) = args.subcommand() + { + Some( + subcommands::container::exec_build(args) + .await + .map(|()| ExitCode::SUCCESS), + ) } else { None } @@ -75,7 +81,7 @@ pub async fn exec_subcommand( "list" => list::exec(config, args).await, "init" => init::exec(config, args).await.map(|_| ()), "build" => build::exec(config, args).await.map(drop), - "container" => subcommands::container::exec(args).await, + "container" => subcommands::container::exec(config, args).await, "server" => server::exec(config, paths, args).await, "subscribe" => subscribe::exec(config, args).await, "start" => return start::exec(config, paths, args).await, diff --git a/crates/cli/src/subcommands/container.rs b/crates/cli/src/subcommands/container.rs index 4248996798b..3dec5973fbf 100644 --- a/crates/cli/src/subcommands/container.rs +++ b/crates/cli/src/subcommands/container.rs @@ -1,4 +1,7 @@ -//! Local container tooling. Database selection here never contacts a server. +//! Container build and operation commands. Local builds do not read saved +//! server credentials; network commands use the explicitly selected server. +mod url; + use crate::{ container::{config::ContainerConfig, prepare_container, process::LocalRunner, BuildSecret, BuildTools}, spacetime_config::{find_and_load_with_env_from, SpacetimeConfig}, @@ -12,6 +15,7 @@ pub fn cli() -> Command { Command::new("container") .about("Build and manage a database's container") .subcommand_required(true) + .subcommand(url::cli()) .subcommand( Command::new("build") .about("Prepare verified OCI artifacts locally without publishing") @@ -111,10 +115,15 @@ pub(crate) fn select(config: &SpacetimeConfig, database: Option<&str>) -> Result .context("selected database has no container declaration; containers are not inherited") } -pub async fn exec(args: &ArgMatches) -> Result<()> { - let ("build", args) = args.subcommand().context("missing container command")? else { - anyhow::bail!("unsupported container command"); - }; +pub async fn exec(config: crate::Config, args: &ArgMatches) -> Result<()> { + match args.subcommand().context("missing container command")? { + ("build", args) => exec_build(args).await, + ("url", args) => url::exec(&config, args).await, + _ => anyhow::bail!("unsupported container command"), + } +} + +pub async fn exec_build(args: &ArgMatches) -> Result<()> { let project = args.get_one::("project_path").unwrap().canonicalize()?; let loaded = find_and_load_with_env_from(args.get_one::("env").map(String::as_str), project)? .context("spacetime.json not found")?; diff --git a/crates/cli/src/subcommands/container/url.rs b/crates/cli/src/subcommands/container/url.rs new file mode 100644 index 00000000000..4f319c3952b --- /dev/null +++ b/crates/cli/src/subcommands/container/url.rs @@ -0,0 +1,172 @@ +//! Anonymous discovery prints one validated public address, never a constructed +//! hostname, authentication token or private runtime address. + +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgMatches, Command}; +use reqwest::{redirect::Policy, Client, StatusCode}; +use spacetimedb_lib::{ + container::{endpoints::ContainerEndpoints, MAX_PORTS}, + Identity, +}; +use std::{collections::BTreeSet, time::Duration}; +use url::Url; + +const MAX_RESPONSE_BYTES: usize = 64 * 1024; + +pub(super) fn cli() -> Command { + Command::new("url") + .about("Print a container's published HTTPS URL") + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg( + Arg::new("port") + .long("port") + .help("Declared port name; required when several ports are published"), + ) + .arg(crate::common_args::server()) + .after_help("Discovery does not start the container or wait for readiness. No login is required.") +} + +pub(super) async fn exec(config: &crate::Config, args: &ArgMatches) -> Result<()> { + let server = args.get_one::("server").map(String::as_str); + let database = args.get_one::("database").context("database is required")?; + let port = args.get_one::("port").map(String::as_str); + let endpoints = fetch(&config.get_host_url(server)?, database).await?; + println!("{}", select(&endpoints, port)?); + Ok(()) +} + +async fn fetch(server: &str, database: &str) -> Result { + ensure!( + !database.is_empty() + && !matches!(database, "." | "..") + && database.len() <= 1024 + && !database.chars().any(char::is_control), + "invalid database name or Identity" + ); + let mut url = Url::parse(server).context("invalid server URL")?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.host().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none() + && url.path() == "/", + "server must be an HTTP or HTTPS origin without credentials or a path" + ); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid server URL"))? + .clear() + .extend(["v1", "database", database, "container", "endpoints"]); + // This endpoint is public. Do not load a login, resolve an Identity through + // another request, follow redirects, or forward saved credentials. + let client = Client::builder() + .no_proxy() + .redirect(Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(15)) + .build()?; + let mut response = client + .get(url) + .send() + .await + .map_err(|_| anyhow::anyhow!("could not reach container endpoint discovery on the selected server"))?; + let status = response.status(); + if status == StatusCode::NOT_FOUND { + bail!("database or container endpoint discovery was not found on the selected server"); + } + if status == StatusCode::SERVICE_UNAVAILABLE { + bail!("container endpoints are not available yet; retry shortly"); + } + if status != StatusCode::OK { + bail!("container endpoint discovery failed (HTTP {})", status.as_u16()); + } + ensure!( + response + .content_length() + .is_none_or(|length| length <= MAX_RESPONSE_BYTES as u64), + "container endpoint discovery response exceeds its size limit" + ); + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container endpoint discovery response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_RESPONSE_BYTES.saturating_sub(body.len()), + "container endpoint discovery response exceeds its size limit" + ); + body.extend_from_slice(&chunk); + } + let endpoints: ContainerEndpoints = + serde_json::from_slice(&body).map_err(|_| anyhow::anyhow!("invalid container endpoint discovery response"))?; + if let Ok(identity) = database.parse::() { + ensure!( + identity == endpoints.database_identity, + "container endpoint discovery returned another database Identity" + ); + } + validate(&endpoints)?; + Ok(endpoints) +} + +fn validate(endpoints: &ContainerEndpoints) -> Result<()> { + ensure!( + endpoints.endpoints.len() <= MAX_PORTS, + "too many container endpoints in discovery response" + ); + let mut names = BTreeSet::new(); + for endpoint in &endpoints.endpoints { + let name = endpoint.name.as_bytes(); + ensure!( + (1..=32).contains(&name.len()) + && name[0].is_ascii_lowercase() + && name + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') + && names.insert(&endpoint.name), + "invalid or duplicate port name in discovery response" + ); + let address = Url::parse(&endpoint.url).context("invalid container endpoint URL")?; + ensure!( + endpoint.url.len() <= 2048 + && endpoint.url.is_ascii() + && !endpoint.url.bytes().any(|byte| byte.is_ascii_control()) + && address.as_str() == endpoint.url + && address.scheme() == "https" + && matches!(address.host(), Some(url::Host::Domain(_))) + && address.username().is_empty() + && address.password().is_none() + && address.port().is_none() + && address.query().is_none() + && address.fragment().is_none() + && address.path() == "/", + "discovery returned an invalid public HTTPS endpoint" + ); + } + Ok(()) +} + +fn select<'a>(endpoints: &'a ContainerEndpoints, port: Option<&str>) -> Result<&'a str> { + match (port, endpoints.endpoints.as_slice()) { + (_, []) => bail!("this database has no declared public HTTP ports"), + (None, [endpoint]) => Ok(&endpoint.url), + (Some(port), entries) => entries + .iter() + .find(|endpoint| endpoint.name == port) + .map(|endpoint| endpoint.url.as_str()) + .context("the requested port name is not published by this database"), + (None, entries) => bail!( + "several ports are published; select one with --port: {}", + entries + .iter() + .map(|entry| entry.name.as_str()) + .collect::>() + .join(", ") + ), + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/cli/src/subcommands/container/url/tests.rs b/crates/cli/src/subcommands/container/url/tests.rs new file mode 100644 index 00000000000..37a544d8b75 --- /dev/null +++ b/crates/cli/src/subcommands/container/url/tests.rs @@ -0,0 +1,198 @@ +use super::*; +use axum::{ + body::{Body, Bytes}, + extract::Path, + http::{header, HeaderMap}, + response::{IntoResponse, Response}, + routing::get, + Json, Router, +}; +use spacetimedb_lib::container::{endpoints::ContainerEndpoint, PortProtocol}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +fn response() -> ContainerEndpoints { + ContainerEndpoints { + database_identity: Identity::ZERO, + endpoints: vec![ContainerEndpoint { + name: "http".into(), + protocol: PortProtocol::Http, + url: "https://aaaqeayeaudaocajbifqydiob4.container.example.net/".into(), + }], + } +} + +struct Server { + origin: String, + stop: Option>, + task: tokio::task::JoinHandle>, +} + +impl Server { + async fn start(router: Router) -> Result { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let origin = format!("http://{}", listener.local_addr()?); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + Ok(Self { + origin, + stop: Some(stop), + task, + }) + } + + async fn shutdown(mut self) -> Result<()> { + self.stop.take().unwrap().send(()).ok(); + tokio::time::timeout(Duration::from_secs(5), &mut self.task).await???; + Ok(()) + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[tokio::test] +async fn discovery_is_anonymous_and_encodes_the_entire_database_path_segment() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let count = requests.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/endpoints", + get(move |Path(database): Path, headers: HeaderMap| { + let count = count.clone(); + async move { + assert_eq!(database, "project/child?query=value#fragment"); + assert!(!headers.contains_key(header::AUTHORIZATION)); + assert!(!headers.contains_key(header::COOKIE)); + count.fetch_add(1, Ordering::SeqCst); + Json(response()) + } + }), + )) + .await?; + for invalid in [".", ".."] { + assert!(fetch(&server.origin, invalid).await.is_err()); + } + assert_eq!(requests.load(Ordering::SeqCst), 0); + let discovered = fetch(&server.origin, "project/child?query=value#fragment").await?; + assert_eq!(select(&discovered, None)?, response().endpoints[0].url); + assert_eq!(requests.load(Ordering::SeqCst), 1); + server.shutdown().await +} + +#[test] +fn port_selection_never_picks_an_arbitrary_endpoint() -> Result<()> { + let mut endpoints = response(); + assert_eq!(select(&endpoints, Some("http"))?, endpoints.endpoints[0].url); + assert!(select(&endpoints, Some("absent")).is_err()); + let mut second = endpoints.endpoints[0].clone(); + second.name = "metrics".into(); + second.url = "https://bbbbbbbbbbbbbbbbbbbbbbbbaa.container.example.net/".into(); + endpoints.endpoints.push(second); + assert!(select(&endpoints, None) + .unwrap_err() + .to_string() + .contains("--port: http, metrics")); + assert_eq!(select(&endpoints, Some("metrics"))?, endpoints.endpoints[1].url); + endpoints.endpoints.clear(); + assert!(select(&endpoints, None) + .unwrap_err() + .to_string() + .contains("no declared public HTTP ports")); + Ok(()) +} + +#[test] +fn endpoint_output_rejects_terminal_controls_credentials_and_ambiguous_names() { + let mut endpoints = response(); + for address in [ + "javascript:alert(1)", + "http://application.example/", + "https://user:secret@application.example/", + "https://127.0.0.1/", + "https://application.example/path", + "https://application.example/?secret=value", + "https://application.example/#fragment", + "https://application.example:8443/", + "https://application.example/\n", + "\u{1b}[2Jhttps://application.example/", + ] { + endpoints.endpoints[0].url = address.into(); + assert!(validate(&endpoints).is_err()); + } + endpoints = response(); + for name in ["", "HTTP", "1http", "http\n", "http/metrics"] { + endpoints.endpoints[0].name = name.into(); + assert!(validate(&endpoints).is_err()); + } + endpoints = response(); + endpoints.endpoints.push(endpoints.endpoints[0].clone()); + assert!(validate(&endpoints).is_err()); +} + +#[tokio::test] +async fn discovery_bounds_streams_rejects_redirects_and_preserves_pending_errors() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let count = requests.clone(); + let server = Server::start(Router::new().route( + "/v1/database/:database/container/endpoints", + get(move |Path(database): Path| { + let count = count.clone(); + async move { + count.fetch_add(1, Ordering::SeqCst); + match database.as_str() { + "pending" => (StatusCode::SERVICE_UNAVAILABLE, "private diagnostic").into_response(), + "missing" => StatusCode::NOT_FOUND.into_response(), + "redirect" => (StatusCode::FOUND, [(header::LOCATION, "/unexpected")]).into_response(), + "invalid" => Json(serde_json::json!({"private diagnostic": "do not print"})).into_response(), + "oversize" => Response::new(Body::from_stream(futures::stream::iter([ + Ok::<_, std::io::Error>(Bytes::from(vec![b' '; MAX_RESPONSE_BYTES])), + Ok(Bytes::from_static(b"x")), + ]))), + _ => Json(response()).into_response(), + } + } + }), + )) + .await?; + for (database, expected) in [ + ("pending", "not available yet"), + ("missing", "not found"), + ("redirect", "HTTP 302"), + ("invalid", "invalid container endpoint discovery response"), + ("oversize", "size limit"), + ] { + let error = format!("{:#}", fetch(&server.origin, database).await.unwrap_err()); + assert!(error.contains(expected), "{error}"); + assert!(!error.contains("private diagnostic")); + } + let other_identity = Identity::from_u256(1_u64.into()).to_string(); + assert!(fetch(&server.origin, &other_identity) + .await + .unwrap_err() + .to_string() + .contains("another database Identity")); + assert_eq!(requests.load(Ordering::SeqCst), 6); + server.shutdown().await +} + +#[test] +fn url_command_requires_database_and_parses_explicit_server_and_port() { + assert!(cli().try_get_matches_from(["url"]).is_err()); + let args = cli() + .try_get_matches_from(["url", "demo", "--server", "http://127.0.0.1:3000", "--port", "http"]) + .unwrap(); + assert_eq!(args.get_one::("database").unwrap(), "demo"); + assert_eq!(args.get_one::("server").unwrap(), "http://127.0.0.1:3000"); + assert_eq!(args.get_one::("port").unwrap(), "http"); +} diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index e7a32bc5bdb..ebb49ca5408 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -307,14 +307,14 @@ enum Error { #[error("error sending subscription queries")] Subscribe { #[source] - source: WsError, + source: Box, }, #[error("protocol error: {details}")] Protocol { details: &'static str }, #[error("websocket error: {source}")] Websocket { #[source] - source: WsError, + source: Box, }, #[error("encountered failed transaction: {reason}")] TransactionFailure { reason: Box }, @@ -343,12 +343,7 @@ enum Error { impl Error { fn is_server_closed_connection(&self) -> bool { - matches!( - self, - Self::Websocket { - source: WsError::ConnectionClosed - } - ) + matches!(self, Self::Websocket { source } if matches!(source.as_ref(), WsError::ConnectionClosed)) } } @@ -364,7 +359,9 @@ where }, ))) .unwrap(); - ws.send(msg.into()).await.map_err(|source| Error::Subscribe { source }) + ws.send(msg.into()) + .await + .map_err(|source| Error::Subscribe { source: source.into() }) } /// Send a v3 BSATN subscribe message. @@ -380,7 +377,7 @@ where let msg = bsatn::to_vec(&msg).map_err(|source| Error::BsatnEncode { source })?; ws.send(WsMessage::Binary(msg.into())) .await - .map_err(|source| Error::Subscribe { source }) + .map_err(|source| Error::Subscribe { source: source.into() }) } /// Parse a v1 text websocket message as JSON. @@ -400,7 +397,11 @@ where { const RECV_TX_UPDATE: &str = "protocol error: received transaction update before initial subscription update"; - while let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? { + while let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + { let Some(msg) = parse_msg_json(&msg) else { continue }; match msg { ws_v1::ServerMessage::InitialSubscription(sub) => { @@ -481,10 +482,14 @@ where if num.is_some_and(|n| num_received >= n) { return Ok(()); } - let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + else { eprintln!("disconnected by server"); return Err(Error::Websocket { - source: WsError::ConnectionClosed, + source: Box::new(WsError::ConnectionClosed), }); }; @@ -532,7 +537,7 @@ where let Some(msg) = next_server_message(ws, pending).await? else { eprintln!("disconnected by server"); return Err(Error::Websocket { - source: WsError::ConnectionClosed, + source: Box::new(WsError::ConnectionClosed), }); }; @@ -572,7 +577,11 @@ where return Ok(Some(msg)); } - let Some(msg) = ws.try_next().await.map_err(|source| Error::Websocket { source })? else { + let Some(msg) = ws + .try_next() + .await + .map_err(|source| Error::Websocket { source: source.into() })? + else { return Ok(None); }; let WsMessage::Binary(msg) = msg else { continue }; diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index d2202566a1a..ea9b15c12da 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -197,7 +197,6 @@ impl AuthHeader { val }) } - } pub const VALID_PROTOCOLS: [&str; 2] = ["http", "https"]; diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs index e29690d34cd..7e43abed944 100644 --- a/crates/lib/src/container.rs +++ b/crates/lib/src/container.rs @@ -7,6 +7,9 @@ use crate::{bsatn, hash_bytes, Hash, SpacetimeType}; use std::{collections::BTreeSet, fmt, str::FromStr}; +#[cfg(feature = "serde")] +pub mod endpoints; + /// Version of the normalized deployment encoding, independent of module ABI. pub const CONTAINER_SPEC_VERSION: u32 = 1; pub const MAX_ARGV_ENTRIES: usize = 256; diff --git a/crates/lib/src/container/endpoints.rs b/crates/lib/src/container/endpoints.rs new file mode 100644 index 00000000000..d5ab7964b11 --- /dev/null +++ b/crates/lib/src/container/endpoints.rs @@ -0,0 +1,57 @@ +//! Public address discovery. These addresses carry no administrative or runtime +//! authority and do not promise that an application is currently ready. + +use super::PortProtocol; +use crate::Identity; +use serde::{Deserialize, Serialize}; + +pub const ENDPOINTS_PENDING: &str = "endpoints_pending"; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerEndpoints { + pub database_identity: Identity, + pub endpoints: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerEndpoint { + pub name: String, + pub protocol: PortProtocol, + pub url: String, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn discovery_wire_contains_only_explicit_public_addresses() { + let response = ContainerEndpoints { + database_identity: Identity::ZERO, + endpoints: vec![ContainerEndpoint { + name: "http".into(), + protocol: PortProtocol::Http, + url: "https://aaaqeayeaudaocajbifqydiob4.container.example.net/".into(), + }], + }; + let encoded = serde_json::to_value(&response).unwrap(); + assert_eq!(encoded.as_object().unwrap().len(), 2); + assert_eq!( + encoded["endpoints"][0], + serde_json::json!({ + "name": "http", + "protocol": "http", + "url": "https://aaaqeayeaudaocajbifqydiob4.container.example.net/" + }) + ); + assert_eq!( + serde_json::from_value::(encoded.clone()).unwrap(), + response + ); + let mut changed = encoded; + changed["endpoints"][0]["upstream"] = serde_json::json!("10.0.0.1:8080"); + assert!(serde_json::from_value::(changed).is_err()); + } +} From a945a806327ead04e0708c0924ef2da8e5dba34d Mon Sep 17 00:00:00 2001 From: Tyler Cloutier Date: Tue, 8 Sep 2026 13:18:33 -0400 Subject: [PATCH 23/23] Add container status and idempotent lifecycle commands --- crates/cli/docs/container-operations.md | 39 +++ crates/cli/src/subcommands/container.rs | 9 +- .../src/subcommands/container/operations.rs | 315 ++++++++++++++++++ crates/cli/src/subcommands/container/url.rs | 2 +- crates/lib/src/container.rs | 2 + crates/lib/src/container/operations.rs | 199 +++++++++++ 6 files changed, 562 insertions(+), 4 deletions(-) create mode 100644 crates/cli/docs/container-operations.md create mode 100644 crates/cli/src/subcommands/container/operations.rs create mode 100644 crates/lib/src/container/operations.rs diff --git a/crates/cli/docs/container-operations.md b/crates/cli/docs/container-operations.md new file mode 100644 index 00000000000..3aa680d8167 --- /dev/null +++ b/crates/cli/docs/container-operations.md @@ -0,0 +1,39 @@ +# Container status and lifecycle requests + +`spacetime container status DATABASE` reads the current control state without +opening database storage. Viewer access is sufficient. `--json` returns the +typed status, including desired and observed state, deployment revision, +generation, fixed diagnostics, and public endpoint availability. An absent +current instance is distinct from an old instance's terminal report. Endpoint +allocation may be pending while the rest of the status is available. + +`spacetime container start DATABASE`, `stop DATABASE`, and `restart DATABASE` +require Admin access. Start requests execution, stop requests a stop, and restart +requests a new instance with a fresh environment snapshot. Acceptance records +the desired action; physical stop and readiness complete asynchronously. Use +status to inspect progress. None of these commands changes the published image +or its declaration. + +Each lifecycle request uses a UUIDv7. Before sending it, the CLI prints structured +retry parameters to stderr: the action, resolved database Identity, request ID, +and selected server URL. Keep these parameters if the response is lost. Retry +with that Identity, the same action, `--request-id UUID`, and `--server URL`. +`--request-id` requires an Identity so a changed database name cannot redirect a +retry. Never create a fresh request ID merely to resolve an unknown outcome. + +An accepted exact retry returns the original result generation, even if another +request has since advanced the database generation. Reusing an ID for another +action fails. Current Admin access is checked again on every retry. The retry +window is seven days from the UUID's timestamp. `--json` writes only a verified +receipt to stdout; its generation is a decimal string to preserve all 64 bits. + +Server selection and login use the normal CLI configuration, with an explicit +`--server` override supported on every command. Authenticated requests disable +redirects and inherited proxies. Public `container url` remains anonymous, and +local `container build` continues to run without loading server credentials. + +The HTTP counterparts are authenticated `GET +/v1/database/DATABASE/container/status` and `POST` to the `start`, `stop`, or +`restart` suffix with `{"request_id":"UUID"}`. Successful mutations return HTTP +202. Responses are not cacheable. The operational API rejects hosted container +credentials and uses current ordinary database roles. diff --git a/crates/cli/src/subcommands/container.rs b/crates/cli/src/subcommands/container.rs index 3dec5973fbf..5e66ef2eeed 100644 --- a/crates/cli/src/subcommands/container.rs +++ b/crates/cli/src/subcommands/container.rs @@ -1,5 +1,6 @@ //! Container build and operation commands. Local builds do not read saved //! server credentials; network commands use the explicitly selected server. +mod operations; mod url; use crate::{ @@ -12,7 +13,7 @@ use std::path::{Path, PathBuf}; use tokio_util::sync::CancellationToken; pub fn cli() -> Command { - Command::new("container") + let command = Command::new("container") .about("Build and manage a database's container") .subcommand_required(true) .subcommand(url::cli()) @@ -81,7 +82,8 @@ pub fn cli() -> Command { .value_name("NAME=FILE") .help("Explicit build secret file; separate from runtime env_keys"), ), - ) + ); + operations::commands(command) } pub(crate) fn select(config: &SpacetimeConfig, database: Option<&str>) -> Result { @@ -115,10 +117,11 @@ pub(crate) fn select(config: &SpacetimeConfig, database: Option<&str>) -> Result .context("selected database has no container declaration; containers are not inherited") } -pub async fn exec(config: crate::Config, args: &ArgMatches) -> Result<()> { +pub async fn exec(mut config: crate::Config, args: &ArgMatches) -> Result<()> { match args.subcommand().context("missing container command")? { ("build", args) => exec_build(args).await, ("url", args) => url::exec(&config, args).await, + (name @ ("status" | "start" | "stop" | "restart"), args) => operations::exec(&mut config, name, args).await, _ => anyhow::bail!("unsupported container command"), } } diff --git a/crates/cli/src/subcommands/container/operations.rs b/crates/cli/src/subcommands/container/operations.rs new file mode 100644 index 00000000000..3236baca42d --- /dev/null +++ b/crates/cli/src/subcommands/container/operations.rs @@ -0,0 +1,315 @@ +use anyhow::{bail, ensure, Context, Result}; +use clap::{Arg, ArgAction, ArgMatches, Command}; +use reqwest::{ + header::{HeaderValue, AUTHORIZATION}, + Client, Method, StatusCode, Url, +}; +use serde::de::DeserializeOwned; +use spacetimedb_lib::{ + container::{endpoints::ContainerEndpoints, operations::*}, + Identity, Uuid, +}; +use std::time::Duration; + +const MAX_RESPONSE: usize = 64 * 1024; +pub(super) fn commands(command: Command) -> Command { + let base = |name: &'static str, about: &'static str| { + Command::new(name) + .about(about) + .arg(Arg::new("database").required(true).help("Database name or Identity")) + .arg(crate::common_args::server()) + .arg(crate::common_args::yes()) + .arg( + Arg::new("json") + .long("json") + .action(ArgAction::SetTrue) + .help("Print the typed response as JSON"), + ) + }; + let action = |name, about| { + base(name, about) + .arg(Arg::new("request_id").long("request-id") + .help("Retry an original UUIDv7 request; DATABASE must be its recorded Identity")) + .after_help("Acceptance records the desired action; physical stop and readiness are asynchronous. A timeout must be retried with the original Identity and request ID printed on stderr.") + }; + command + .subcommand(base( + "status", + "Inspect container control state without opening its database", + )) + .subcommand(action("start", "Request container execution")) + .subcommand(action("stop", "Request container stop")) + .subcommand(action( + "restart", + "Request a new container instance and environment snapshot", + )) +} + +pub(super) async fn exec(config: &mut crate::Config, name: &str, args: &ArgMatches) -> Result<()> { + let selection = args.get_one::("server").map(String::as_str); + let database = args.get_one::("database").context("database is required")?; + validate_database(database)?; + let server = crate::container::publish::client::endpoint(&config.get_host_url(selection)?)?; + let supplied_request = args + .try_get_one::("request_id") + .ok() + .flatten() + .map(|id| Uuid::parse_str(id).context("request-id must be a UUIDv7")) + .transpose()?; + if supplied_request.is_some() { + ensure!( + database.parse::().is_ok(), + "retry with the original database Identity printed by the first command" + ); + } + let auth = crate::util::get_auth_header(config, false, selection, !args.get_flag("force")).await?; + let client = ContainerClient::new( + server, + auth.to_header().context("container operations require a login")?, + )?; + if name == "status" { + let status = client.status(database).await?; + if args.get_flag("json") { + println!("{}", serde_json::to_string(&status)?); + } else { + print_status(&status); + } + return Ok(()); + } + let action = match name { + "start" => ContainerAction::Start, + "stop" => ContainerAction::Stop, + "restart" => ContainerAction::Restart, + _ => bail!("unsupported container operation"), + }; + // Names are resolved once before mutation. The replay target is immutable. + let identity = match database.parse::() { + Ok(identity) => identity, + Err(_) => client.status(database).await?.database_identity, + }; + let request_id = supplied_request.unwrap_or_else(|| Uuid::from_u128(uuid::Uuid::now_v7().as_u128())); + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_millis(), + )?; + spacetimedb_lib::deployment::operation_expiry_ms(request_id, now) + .context("request ID is not within its seven-day retry window")?; + // Structured retry parameters preserve the selected origin without + // presenting unescaped server text as an executable shell command. + let retry = serde_json::json!({ + "command": format!("container {}", action.path()), + "database": identity.to_hex().to_string(), + "request_id": request_id.to_string(), + "server": client.server.as_str(), + }); + eprintln!("Container request retry parameters: {retry}"); + let receipt = client.operate(identity, request_id, action).await.with_context(|| { + format!("request did not return a verified receipt; retain these exact retry parameters: {retry}") + })?; + if args.get_flag("json") { + println!("{}", serde_json::to_string(&receipt)?); + } else { + println!( + "Accepted {} for {} at generation {}", + action.path(), + identity.to_hex(), + receipt.generation + ); + } + Ok(()) +} + +struct ContainerClient { + http: Client, + server: Url, + authorization: HeaderValue, +} +impl ContainerClient { + fn new(server: Url, mut authorization: HeaderValue) -> Result { + ensure!( + authorization.as_bytes().starts_with(b"Bearer "), + "container operations require ordinary Bearer authentication" + ); + authorization.set_sensitive(true); + let http = Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) + .build()?; + Ok(Self { + http, + server, + authorization, + }) + } + fn url(&self, database: &str, operation: &str) -> Result { + validate_database(database)?; + let mut url = self.server.clone(); + url.path_segments_mut() + .map_err(|_| anyhow::anyhow!("invalid server origin"))? + .pop_if_empty() + .extend(["v1", "database", database, "container", operation]); + Ok(url) + } + async fn status(&self, database: &str) -> Result { + let response = self + .http + .get(self.url(database, "status")?) + .header(AUTHORIZATION, self.authorization.clone()) + .send() + .await + .map_err(|_| anyhow::anyhow!("container status could not be reached on the selected server"))?; + let status: ContainerStatus = read_response(response, StatusCode::OK).await?; + if let Ok(identity) = database.parse::() { + ensure!( + status.database_identity == identity, + "status returned another database Identity" + ); + } + validate_status(&status)?; + Ok(status) + } + async fn operate( + &self, + identity: Identity, + request_id: Uuid, + action: ContainerAction, + ) -> Result { + let response = self + .http + .request(Method::POST, self.url(identity.to_hex().as_ref(), action.path())?) + .header(AUTHORIZATION, self.authorization.clone()) + .json(&ContainerOperationRequest { request_id }) + .send() + .await + .map_err(|_| anyhow::anyhow!("container operation response was not received"))?; + let receipt: ContainerOperationReceipt = read_response(response, StatusCode::ACCEPTED).await?; + ensure!( + receipt.database_identity == identity && receipt.request_id == request_id && receipt.action == action, + "container operation receipt does not match this request" + ); + Ok(receipt) + } +} +fn validate_database(database: &str) -> Result<()> { + ensure!( + !database.is_empty() + && database.len() <= 1024 + && !matches!(database, "." | "..") + && !database.chars().any(char::is_control), + "invalid database name or Identity" + ); + Ok(()) +} +async fn read_response(mut response: reqwest::Response, expected: StatusCode) -> Result { + let status = response.status(); + ensure!( + response.content_length().is_none_or(|size| size <= MAX_RESPONSE as u64), + "container response exceeds its size limit" + ); + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| anyhow::anyhow!("container response was interrupted"))? + { + ensure!( + chunk.len() <= MAX_RESPONSE.saturating_sub(bytes.len()), + "container response exceeds its size limit" + ); + bytes.extend_from_slice(&chunk); + } + if status != expected { + let error = serde_json::from_slice::(&bytes) + .ok() + .map(|value| value.error); + bail!( + "{} (HTTP {})", + match error { + Some(ContainerErrorCode::AccessDenied) => "database role does not permit this operation", + Some(ContainerErrorCode::NotFound) => "database was not found", + Some(ContainerErrorCode::InvalidRequest) => "container request is invalid or expired", + Some(ContainerErrorCode::Conflict) => "container request conflicts with current state", + Some(ContainerErrorCode::OutcomeUnknown) => + "container request outcome is unknown; retry the same request ID", + _ => "container service is unavailable on the selected server", + }, + status.as_u16() + ); + } + serde_json::from_slice(&bytes).map_err(|_| anyhow::anyhow!("invalid container response")) +} +fn validate_status(status: &ContainerStatus) -> Result<()> { + if let Some(state) = &status.operational + && let Some(instance) = &state.current_instance + { + ensure!( + instance.generation == state.generation, + "status instance is not the current generation" + ); + if let Some(id) = &instance.applied_env_generation { + ensure!(Uuid::parse_str(id).is_ok(), "invalid applied environment generation"); + } + } + if let EndpointStatus::Available { endpoints } = &status.endpoints { + super::url::validate(&ContainerEndpoints { + database_identity: status.database_identity, + endpoints: endpoints.clone(), + })?; + } + Ok(()) +} +fn print_status(status: &ContainerStatus) { + println!("Database: {}", status.database_identity.to_hex()); + if !status.published { + println!("Container: none published"); + } + if let Some(state) = &status.operational { + println!("Desired: {:?} (generation {})", state.desired_state, state.generation); + println!("Condition: {:?}", state.condition); + if let Some(instance) = &state.current_instance { + println!("Observed: {:?}", instance.state); + if let Some(environment) = &instance.applied_env_generation { + println!("Environment generation: {environment}"); + } + if let Some(code) = instance.exit_code { + println!("Exit code: {code}"); + } + if instance.oom_killed { + println!("Out of memory: yes"); + } + } else { + println!("Observed: no instance admitted for this generation"); + } + } + match &status.endpoints { + EndpointStatus::Available { endpoints } => { + for endpoint in endpoints { + println!("{}: {}", endpoint.name, endpoint.url); + } + } + EndpointStatus::Pending => println!("Endpoints: pending"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn target_path_encodes_one_database_and_rejects_dot_segments() { + let client = ContainerClient::new( + Url::parse("http://127.0.0.1:43123/").unwrap(), + HeaderValue::from_static("Bearer unusable"), + ) + .unwrap(); + assert_eq!( + client.url("name/child?x#y", "status").unwrap().path(), + "/v1/database/name%2Fchild%3Fx%23y/container/status" + ); + for invalid in ["", ".", "..", "name\n"] { + assert!(client.url(invalid, "status").is_err()); + } + } +} diff --git a/crates/cli/src/subcommands/container/url.rs b/crates/cli/src/subcommands/container/url.rs index 4f319c3952b..b66eee410aa 100644 --- a/crates/cli/src/subcommands/container/url.rs +++ b/crates/cli/src/subcommands/container/url.rs @@ -111,7 +111,7 @@ async fn fetch(server: &str, database: &str) -> Result { Ok(endpoints) } -fn validate(endpoints: &ContainerEndpoints) -> Result<()> { +pub(super) fn validate(endpoints: &ContainerEndpoints) -> Result<()> { ensure!( endpoints.endpoints.len() <= MAX_PORTS, "too many container endpoints in discovery response" diff --git a/crates/lib/src/container.rs b/crates/lib/src/container.rs index 7e43abed944..02d9c581824 100644 --- a/crates/lib/src/container.rs +++ b/crates/lib/src/container.rs @@ -9,6 +9,8 @@ use std::{collections::BTreeSet, fmt, str::FromStr}; #[cfg(feature = "serde")] pub mod endpoints; +#[cfg(feature = "serde")] +pub mod operations; /// Version of the normalized deployment encoding, independent of module ABI. pub const CONTAINER_SPEC_VERSION: u32 = 1; diff --git a/crates/lib/src/container/operations.rs b/crates/lib/src/container/operations.rs new file mode 100644 index 00000000000..f6b499f1db8 --- /dev/null +++ b/crates/lib/src/container/operations.rs @@ -0,0 +1,199 @@ +//! Control-plane container inspection and idempotent lifecycle messages. +use crate::{container::endpoints::ContainerEndpoint, deployment::uuid_json, Hash, Identity, Uuid}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContainerAction { + Start, + Stop, + Restart, +} +impl ContainerAction { + pub const fn path(self) -> &'static str { + match self { + Self::Start => "start", + Self::Stop => "stop", + Self::Restart => "restart", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerOperationRequest { + #[serde(with = "uuid_json")] + pub request_id: Uuid, +} + +/// Confirms admission only. It does not claim physical stop or runtime readiness. +/// An exact retry returns the generation originally recorded for this request. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerOperationReceipt { + pub database_identity: Identity, + #[serde(with = "uuid_json")] + pub request_id: Uuid, + pub action: ContainerAction, + #[serde(with = "decimal_u64")] + pub generation: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DesiredState { + Stopped, + Running, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ObservedState { + Pending, + Starting, + Running, + Ready, + Draining, + Stopped, + Completed, + Failed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Condition { + None, + Fencing, + NoLeader, + NodeUnavailable, + Capacity, + BalanceUnavailable, + BalanceExhausted, + TargetUnavailable, + PullFailed, + LaunchFailed, + ReadinessFailed, + ExitFailure, + OutOfMemory, + NodePressure, + LeaseExpired, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CurrentInstance { + #[serde(with = "decimal_u64")] + pub generation: u64, + pub state: ObservedState, + pub observed_revision: Option, + /// Application snapshot identity only. No values or hashes of values. + pub applied_env_generation: Option, + pub exit_code: Option, + pub oom_killed: bool, + pub condition: Condition, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperationalState { + pub desired_revision: Hash, + pub desired_state: DesiredState, + #[serde(with = "decimal_u64")] + pub generation: u64, + pub condition: Condition, + pub restart_pending: bool, + pub restart_attempt: u32, + /// Unix milliseconds, represented exactly for browser clients. + #[serde(with = "decimal_u64")] + pub restart_not_before_ms: u64, + /// Reports only this operational generation. A historical attempt is never + /// presented as the currently authorized replacement. + pub current_instance: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum EndpointStatus { + Available { endpoints: Vec }, + Pending, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerStatus { + pub database_identity: Identity, + pub published: bool, + pub operational: Option, + /// Address discovery is independent of readiness and remains available + /// while stopped. Pending never means an empty declaration. + pub endpoints: EndpointStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContainerErrorCode { + InvalidRequest, + AccessDenied, + NotFound, + Conflict, + Unavailable, + OutcomeUnknown, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContainerApiError { + pub error: ContainerErrorCode, +} + +mod decimal_u64 { + use serde::{Deserialize, Deserializer, Serializer}; + pub fn serialize(number: &u64, serializer: S) -> Result { + serializer.collect_str(number) + } + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result { + let text = String::deserialize(deserializer)?; + let value: u64 = text.parse().map_err(serde::de::Error::custom)?; + if value.to_string() != text { + return Err(serde::de::Error::custom("expected canonical decimal u64")); + } + Ok(value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn receipt_preserves_full_generation_and_rejects_lossy_or_extra_fields() { + let receipt = ContainerOperationReceipt { + database_identity: Identity::ONE, + request_id: Uuid::from_u128(0x01950000000070008000000000000001), + action: ContainerAction::Restart, + generation: u64::MAX, + }; + let mut value = serde_json::to_value(&receipt).unwrap(); + assert_eq!(value["generation"], u64::MAX.to_string()); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + receipt + ); + for generation in [ + serde_json::json!(1), + serde_json::json!("01"), + serde_json::json!("18446744073709551616"), + ] { + value["generation"] = generation; + assert!(serde_json::from_value::(value.clone()).is_err()); + } + value = serde_json::to_value(receipt).unwrap(); + value["credential"] = serde_json::json!("unexpected"); + assert!(serde_json::from_value::(value).is_err()); + } + #[test] + fn pending_discovery_is_distinct_from_empty() { + assert_ne!( + serde_json::to_value(EndpointStatus::Pending).unwrap(), + serde_json::to_value(EndpointStatus::Available { endpoints: vec![] }).unwrap() + ); + } +}