From cea0d0c2e73f3303297f3564ed03a7b7d9cd8d1f Mon Sep 17 00:00:00 2001 From: River Date: Tue, 29 Sep 2026 14:11:22 +0000 Subject: [PATCH 1/3] libsql-server: add namespace fence types and transition logic Add the I/O-free core of the namespace fence described in docs/NAMESPACE_FENCE.md: - state.rs: roles, fence states (including the non-durable UNFENCED, ABSENT and derived UNKNOWN_UNAVAILABLE), operation classes and the permission matrix. - outcome.rs: stable outcome codes, their admin HTTP, user HTTP, Hrana, gRPC and proxy mappings (fence denials are never 5xx, 429 or gRPC UNAVAILABLE), bounded detail reasons and FenceError. - command.rs: fence commands and requests, and the canonical SHA-256 request fingerprint over everything except command_id. - record.rs: fence records, command receipts and the on-disk marker, with a strict protobuf encoding (proto/namespace_fence.proto) whose decoder rejects unknown versions, enum values, malformed ids and self-contradictory records instead of defaulting. - transition.rs: the pure apply() and complete_drain() functions. Replay and fingerprint conflicts are decided before owner, role, state and revision; TARGET_WRITABLE has no reverse transition; adoption changes only the owner. Nothing outside the module uses it yet; the metastore, controller and protocol layers follow. Co-authored-by: Tomasz Szymczyszyn --- libsql-server/proto/namespace_fence.proto | 205 ++ .../src/generated/namespace_fence.rs | 507 +++++ libsql-server/src/namespace/fence/command.rs | 420 ++++ libsql-server/src/namespace/fence/mod.rs | 28 + libsql-server/src/namespace/fence/outcome.rs | 409 ++++ libsql-server/src/namespace/fence/record.rs | 908 +++++++++ libsql-server/src/namespace/fence/state.rs | 403 ++++ .../src/namespace/fence/transition.rs | 1794 +++++++++++++++++ libsql-server/src/namespace/mod.rs | 1 + libsql-server/tests/bootstrap.rs | 2 +- 10 files changed, 4676 insertions(+), 1 deletion(-) create mode 100644 libsql-server/proto/namespace_fence.proto create mode 100644 libsql-server/src/generated/namespace_fence.rs create mode 100644 libsql-server/src/namespace/fence/command.rs create mode 100644 libsql-server/src/namespace/fence/mod.rs create mode 100644 libsql-server/src/namespace/fence/outcome.rs create mode 100644 libsql-server/src/namespace/fence/record.rs create mode 100644 libsql-server/src/namespace/fence/state.rs create mode 100644 libsql-server/src/namespace/fence/transition.rs diff --git a/libsql-server/proto/namespace_fence.proto b/libsql-server/proto/namespace_fence.proto new file mode 100644 index 0000000000..efdee1ab81 --- /dev/null +++ b/libsql-server/proto/namespace_fence.proto @@ -0,0 +1,205 @@ +// Durable encoding of namespace fence records, command receipts and markers. +// +// See docs/NAMESPACE_FENCE.md. Every message here is stored, so fields are only ever added. +// Readers reject unknown enum values and missing required fields instead of guessing. +syntax = "proto3"; + +package namespace_fence; + +enum FenceRole { + FENCE_ROLE_UNSPECIFIED = 0; + FENCE_ROLE_SOURCE = 1; + FENCE_ROLE_TARGET = 2; +} + +// `UNFENCED` and `ABSENT` are never stored in a record; they appear as the `expected_state` of a +// request against a namespace that has no record. `UNKNOWN_UNAVAILABLE` is derived and never +// stored either. +enum FenceState { + FENCE_STATE_UNSPECIFIED = 0; + FENCE_STATE_UNFENCED = 1; + FENCE_STATE_ABSENT = 2; + FENCE_STATE_SOURCE_DRAINING = 3; + FENCE_STATE_SOURCE_WRITE_FENCED = 4; + FENCE_STATE_SOURCE_READ_DRAINING = 5; + FENCE_STATE_SOURCE_READ_FENCED = 6; + FENCE_STATE_RELEASED = 7; + FENCE_STATE_TARGET_QUARANTINED = 8; + FENCE_STATE_TARGET_IMPORT_DRAINING = 9; + FENCE_STATE_TARGET_VALIDATING = 10; + FENCE_STATE_TARGET_WRITE_FENCED = 11; + FENCE_STATE_TARGET_WRITABLE = 12; + FENCE_STATE_TARGET_ABORTED = 13; + FENCE_STATE_UNKNOWN_UNAVAILABLE = 14; +} + +enum CommandKind { + COMMAND_KIND_UNSPECIFIED = 0; + COMMAND_KIND_ACQUIRE_SOURCE_WRITE_FENCE = 1; + COMMAND_KIND_SET_SOURCE_READ_FENCE = 2; + COMMAND_KIND_CLEAR_SOURCE_READ_FENCE = 3; + COMMAND_KIND_RELEASE_SOURCE_WRITE_FENCE = 4; + COMMAND_KIND_CREATE_TARGET_QUARANTINED = 5; + COMMAND_KIND_SEAL_TARGET_IMPORT = 6; + COMMAND_KIND_RECORD_TARGET_VALIDATION = 7; + COMMAND_KIND_PUBLISH_TARGET_READABLE_WRITE_FENCED = 8; + COMMAND_KIND_ENABLE_TARGET_WRITES = 9; + COMMAND_KIND_ABORT_QUARANTINED_TARGET = 10; + COMMAND_KIND_ADOPT_FENCE = 11; +} + +// Only the outcomes a receipt can record. Errors are never stored. +enum ReceiptOutcome { + RECEIPT_OUTCOME_UNSPECIFIED = 0; + RECEIPT_OUTCOME_APPLIED = 1; + RECEIPT_OUTCOME_ALREADY_APPLIED = 2; + RECEIPT_OUTCOME_DRAINING = 3; +} + +enum OnDeadline { + ON_DEADLINE_UNSPECIFIED = 0; + ON_DEADLINE_FAIL = 1; + ON_DEADLINE_FORCE_ROLLBACK = 2; +} + +enum ValidationResult { + VALIDATION_RESULT_UNSPECIFIED = 0; + VALIDATION_RESULT_OK = 1; + VALIDATION_RESULT_FAILED = 2; +} + +message DrainPolicy { + uint64 deadline_ms = 1; + OnDeadline on_deadline = 2; +} + +message FrozenBoundary { + string log_id = 1; + uint64 frame_no = 2; +} + +message LegacyBlocks { + bool block_reads = 1; + bool block_writes = 2; + optional string block_reason = 3; +} + +message ServerIdentity { + string build = 1; + string instance_id = 2; +} + +message TargetConfig { + optional uint64 max_db_size = 1; + optional string jwt_key = 2; + optional uint64 txn_timeout_s = 3; + bool allow_attach = 4; + optional string durability_mode = 5; + optional string bottomless_db_id = 6; +} + +message ValidationSnapshot { + string log_id = 1; + uint64 frame_no = 2; + uint64 page_count = 3; +} + +message ValidationRecord { + string operation_id = 1; + string command_id = 2; + ValidationResult result = 3; + string summary = 4; + optional ValidationSnapshot snapshot = 5; + int64 recorded_at_ms = 6; +} + +message Adoption { + string previous_operation_id = 1; + string new_operation_id = 2; + string command_id = 3; + repeated string approvers = 4; + string incident_ref = 5; + string reason = 6; + int64 at_ms = 7; + uint64 revision = 8; +} + +message FenceRecord { + string namespace = 1; + FenceRole role = 2; + FenceState state = 3; + uint64 revision = 4; + string operation_id = 5; + optional string log_id = 6; + optional string target_incarnation_id = 7; + optional DrainPolicy drain_policy = 8; + optional int64 drain_started_at_ms = 9; + optional FrozenBoundary frozen_boundary = 10; + optional ValidationRecord validation = 11; + LegacyBlocks legacy_blocks = 12; + int64 created_at_ms = 13; + int64 last_transition_at_ms = 14; + string last_command_id = 15; + ServerIdentity written_by = 16; + repeated Adoption adoptions = 17; +} + +message CommandReceipt { + string namespace = 1; + string operation_id = 2; + string command_id = 3; + CommandKind command = 4; + bytes fingerprint = 5; + ReceiptOutcome outcome = 6; + uint64 revision_before = 7; + uint64 revision_after = 8; + FenceState state_after = 9; + int64 applied_at_ms = 10; + string instance_id = 11; + optional Adoption adoption = 12; +} + +// Contents of `dbs//.fence`: a copy of the last committed record (or, for +// `CreateTargetQuarantined`, of the record about to be committed). +message FenceMarker { + uint32 format_version = 1; + FenceRecord record = 2; +} + +// Canonical input of a command fingerprint: everything in the request except `command_id`. +message FingerprintInput { + string namespace = 1; + string operation_id = 2; + CommandKind kind = 3; + FenceState expected_state = 4; + uint64 expected_revision = 5; + oneof args { + AcquireSourceWriteFenceArgs acquire_source_write_fence = 10; + DrainArgs set_source_read_fence = 11; + DrainArgs seal_target_import = 12; + TargetConfig create_target_quarantined = 13; + RecordTargetValidationArgs record_target_validation = 14; + AdoptFenceArgs adopt_fence = 15; + } +} + +message AcquireSourceWriteFenceArgs { + string expected_log_id = 1; + optional DrainPolicy drain_policy = 2; +} + +message DrainArgs { + optional DrainPolicy drain_policy = 1; +} + +message RecordTargetValidationArgs { + ValidationResult result = 1; + string summary = 2; +} + +message AdoptFenceArgs { + string current_operation_id = 1; + repeated string approvers = 2; + string incident_ref = 3; + string reason = 4; +} diff --git a/libsql-server/src/generated/namespace_fence.rs b/libsql-server/src/generated/namespace_fence.rs new file mode 100644 index 0000000000..dcbbcafe96 --- /dev/null +++ b/libsql-server/src/generated/namespace_fence.rs @@ -0,0 +1,507 @@ +// This file is @generated by prost-build. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DrainPolicy { + #[prost(uint64, tag = "1")] + pub deadline_ms: u64, + #[prost(enumeration = "OnDeadline", tag = "2")] + pub on_deadline: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FrozenBoundary { + #[prost(string, tag = "1")] + pub log_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub frame_no: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LegacyBlocks { + #[prost(bool, tag = "1")] + pub block_reads: bool, + #[prost(bool, tag = "2")] + pub block_writes: bool, + #[prost(string, optional, tag = "3")] + pub block_reason: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ServerIdentity { + #[prost(string, tag = "1")] + pub build: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub instance_id: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TargetConfig { + #[prost(uint64, optional, tag = "1")] + pub max_db_size: ::core::option::Option, + #[prost(string, optional, tag = "2")] + pub jwt_key: ::core::option::Option<::prost::alloc::string::String>, + #[prost(uint64, optional, tag = "3")] + pub txn_timeout_s: ::core::option::Option, + #[prost(bool, tag = "4")] + pub allow_attach: bool, + #[prost(string, optional, tag = "5")] + pub durability_mode: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "6")] + pub bottomless_db_id: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidationSnapshot { + #[prost(string, tag = "1")] + pub log_id: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub frame_no: u64, + #[prost(uint64, tag = "3")] + pub page_count: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ValidationRecord { + #[prost(string, tag = "1")] + pub operation_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub command_id: ::prost::alloc::string::String, + #[prost(enumeration = "ValidationResult", tag = "3")] + pub result: i32, + #[prost(string, tag = "4")] + pub summary: ::prost::alloc::string::String, + #[prost(message, optional, tag = "5")] + pub snapshot: ::core::option::Option, + #[prost(int64, tag = "6")] + pub recorded_at_ms: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Adoption { + #[prost(string, tag = "1")] + pub previous_operation_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub new_operation_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub command_id: ::prost::alloc::string::String, + #[prost(string, repeated, tag = "4")] + pub approvers: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "5")] + pub incident_ref: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub reason: ::prost::alloc::string::String, + #[prost(int64, tag = "7")] + pub at_ms: i64, + #[prost(uint64, tag = "8")] + pub revision: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FenceRecord { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(enumeration = "FenceRole", tag = "2")] + pub role: i32, + #[prost(enumeration = "FenceState", tag = "3")] + pub state: i32, + #[prost(uint64, tag = "4")] + pub revision: u64, + #[prost(string, tag = "5")] + pub operation_id: ::prost::alloc::string::String, + #[prost(string, optional, tag = "6")] + pub log_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "7")] + pub target_incarnation_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, optional, tag = "8")] + pub drain_policy: ::core::option::Option, + #[prost(int64, optional, tag = "9")] + pub drain_started_at_ms: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub frozen_boundary: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub validation: ::core::option::Option, + #[prost(message, optional, tag = "12")] + pub legacy_blocks: ::core::option::Option, + #[prost(int64, tag = "13")] + pub created_at_ms: i64, + #[prost(int64, tag = "14")] + pub last_transition_at_ms: i64, + #[prost(string, tag = "15")] + pub last_command_id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "16")] + pub written_by: ::core::option::Option, + #[prost(message, repeated, tag = "17")] + pub adoptions: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommandReceipt { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub operation_id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub command_id: ::prost::alloc::string::String, + #[prost(enumeration = "CommandKind", tag = "4")] + pub command: i32, + #[prost(bytes = "vec", tag = "5")] + pub fingerprint: ::prost::alloc::vec::Vec, + #[prost(enumeration = "ReceiptOutcome", tag = "6")] + pub outcome: i32, + #[prost(uint64, tag = "7")] + pub revision_before: u64, + #[prost(uint64, tag = "8")] + pub revision_after: u64, + #[prost(enumeration = "FenceState", tag = "9")] + pub state_after: i32, + #[prost(int64, tag = "10")] + pub applied_at_ms: i64, + #[prost(string, tag = "11")] + pub instance_id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "12")] + pub adoption: ::core::option::Option, +} +/// Contents of `dbs//.fence`: a copy of the last committed record (or, for +/// `CreateTargetQuarantined`, of the record about to be committed). +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FenceMarker { + #[prost(uint32, tag = "1")] + pub format_version: u32, + #[prost(message, optional, tag = "2")] + pub record: ::core::option::Option, +} +/// Canonical input of a command fingerprint: everything in the request except `command_id`. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FingerprintInput { + #[prost(string, tag = "1")] + pub namespace: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub operation_id: ::prost::alloc::string::String, + #[prost(enumeration = "CommandKind", tag = "3")] + pub kind: i32, + #[prost(enumeration = "FenceState", tag = "4")] + pub expected_state: i32, + #[prost(uint64, tag = "5")] + pub expected_revision: u64, + #[prost(oneof = "fingerprint_input::Args", tags = "10, 11, 12, 13, 14, 15")] + pub args: ::core::option::Option, +} +/// Nested message and enum types in `FingerprintInput`. +pub mod fingerprint_input { + #[allow(clippy::derive_partial_eq_without_eq)] + #[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Args { + #[prost(message, tag = "10")] + AcquireSourceWriteFence(super::AcquireSourceWriteFenceArgs), + #[prost(message, tag = "11")] + SetSourceReadFence(super::DrainArgs), + #[prost(message, tag = "12")] + SealTargetImport(super::DrainArgs), + #[prost(message, tag = "13")] + CreateTargetQuarantined(super::TargetConfig), + #[prost(message, tag = "14")] + RecordTargetValidation(super::RecordTargetValidationArgs), + #[prost(message, tag = "15")] + AdoptFence(super::AdoptFenceArgs), + } +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AcquireSourceWriteFenceArgs { + #[prost(string, tag = "1")] + pub expected_log_id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub drain_policy: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DrainArgs { + #[prost(message, optional, tag = "1")] + pub drain_policy: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RecordTargetValidationArgs { + #[prost(enumeration = "ValidationResult", tag = "1")] + pub result: i32, + #[prost(string, tag = "2")] + pub summary: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdoptFenceArgs { + #[prost(string, tag = "1")] + pub current_operation_id: ::prost::alloc::string::String, + #[prost(string, repeated, tag = "2")] + pub approvers: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "3")] + pub incident_ref: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub reason: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FenceRole { + Unspecified = 0, + Source = 1, + Target = 2, +} +impl FenceRole { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + FenceRole::Unspecified => "FENCE_ROLE_UNSPECIFIED", + FenceRole::Source => "FENCE_ROLE_SOURCE", + FenceRole::Target => "FENCE_ROLE_TARGET", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "FENCE_ROLE_UNSPECIFIED" => Some(Self::Unspecified), + "FENCE_ROLE_SOURCE" => Some(Self::Source), + "FENCE_ROLE_TARGET" => Some(Self::Target), + _ => None, + } + } +} +/// `UNFENCED` and `ABSENT` are never stored in a record; they appear as the `expected_state` of a +/// request against a namespace that has no record. `UNKNOWN_UNAVAILABLE` is derived and never +/// stored either. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum FenceState { + Unspecified = 0, + Unfenced = 1, + Absent = 2, + SourceDraining = 3, + SourceWriteFenced = 4, + SourceReadDraining = 5, + SourceReadFenced = 6, + Released = 7, + TargetQuarantined = 8, + TargetImportDraining = 9, + TargetValidating = 10, + TargetWriteFenced = 11, + TargetWritable = 12, + TargetAborted = 13, + UnknownUnavailable = 14, +} +impl FenceState { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + FenceState::Unspecified => "FENCE_STATE_UNSPECIFIED", + FenceState::Unfenced => "FENCE_STATE_UNFENCED", + FenceState::Absent => "FENCE_STATE_ABSENT", + FenceState::SourceDraining => "FENCE_STATE_SOURCE_DRAINING", + FenceState::SourceWriteFenced => "FENCE_STATE_SOURCE_WRITE_FENCED", + FenceState::SourceReadDraining => "FENCE_STATE_SOURCE_READ_DRAINING", + FenceState::SourceReadFenced => "FENCE_STATE_SOURCE_READ_FENCED", + FenceState::Released => "FENCE_STATE_RELEASED", + FenceState::TargetQuarantined => "FENCE_STATE_TARGET_QUARANTINED", + FenceState::TargetImportDraining => "FENCE_STATE_TARGET_IMPORT_DRAINING", + FenceState::TargetValidating => "FENCE_STATE_TARGET_VALIDATING", + FenceState::TargetWriteFenced => "FENCE_STATE_TARGET_WRITE_FENCED", + FenceState::TargetWritable => "FENCE_STATE_TARGET_WRITABLE", + FenceState::TargetAborted => "FENCE_STATE_TARGET_ABORTED", + FenceState::UnknownUnavailable => "FENCE_STATE_UNKNOWN_UNAVAILABLE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "FENCE_STATE_UNSPECIFIED" => Some(Self::Unspecified), + "FENCE_STATE_UNFENCED" => Some(Self::Unfenced), + "FENCE_STATE_ABSENT" => Some(Self::Absent), + "FENCE_STATE_SOURCE_DRAINING" => Some(Self::SourceDraining), + "FENCE_STATE_SOURCE_WRITE_FENCED" => Some(Self::SourceWriteFenced), + "FENCE_STATE_SOURCE_READ_DRAINING" => Some(Self::SourceReadDraining), + "FENCE_STATE_SOURCE_READ_FENCED" => Some(Self::SourceReadFenced), + "FENCE_STATE_RELEASED" => Some(Self::Released), + "FENCE_STATE_TARGET_QUARANTINED" => Some(Self::TargetQuarantined), + "FENCE_STATE_TARGET_IMPORT_DRAINING" => Some(Self::TargetImportDraining), + "FENCE_STATE_TARGET_VALIDATING" => Some(Self::TargetValidating), + "FENCE_STATE_TARGET_WRITE_FENCED" => Some(Self::TargetWriteFenced), + "FENCE_STATE_TARGET_WRITABLE" => Some(Self::TargetWritable), + "FENCE_STATE_TARGET_ABORTED" => Some(Self::TargetAborted), + "FENCE_STATE_UNKNOWN_UNAVAILABLE" => Some(Self::UnknownUnavailable), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum CommandKind { + Unspecified = 0, + AcquireSourceWriteFence = 1, + SetSourceReadFence = 2, + ClearSourceReadFence = 3, + ReleaseSourceWriteFence = 4, + CreateTargetQuarantined = 5, + SealTargetImport = 6, + RecordTargetValidation = 7, + PublishTargetReadableWriteFenced = 8, + EnableTargetWrites = 9, + AbortQuarantinedTarget = 10, + AdoptFence = 11, +} +impl CommandKind { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + CommandKind::Unspecified => "COMMAND_KIND_UNSPECIFIED", + CommandKind::AcquireSourceWriteFence => { + "COMMAND_KIND_ACQUIRE_SOURCE_WRITE_FENCE" + } + CommandKind::SetSourceReadFence => "COMMAND_KIND_SET_SOURCE_READ_FENCE", + CommandKind::ClearSourceReadFence => "COMMAND_KIND_CLEAR_SOURCE_READ_FENCE", + CommandKind::ReleaseSourceWriteFence => { + "COMMAND_KIND_RELEASE_SOURCE_WRITE_FENCE" + } + CommandKind::CreateTargetQuarantined => { + "COMMAND_KIND_CREATE_TARGET_QUARANTINED" + } + CommandKind::SealTargetImport => "COMMAND_KIND_SEAL_TARGET_IMPORT", + CommandKind::RecordTargetValidation => { + "COMMAND_KIND_RECORD_TARGET_VALIDATION" + } + CommandKind::PublishTargetReadableWriteFenced => { + "COMMAND_KIND_PUBLISH_TARGET_READABLE_WRITE_FENCED" + } + CommandKind::EnableTargetWrites => "COMMAND_KIND_ENABLE_TARGET_WRITES", + CommandKind::AbortQuarantinedTarget => { + "COMMAND_KIND_ABORT_QUARANTINED_TARGET" + } + CommandKind::AdoptFence => "COMMAND_KIND_ADOPT_FENCE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "COMMAND_KIND_UNSPECIFIED" => Some(Self::Unspecified), + "COMMAND_KIND_ACQUIRE_SOURCE_WRITE_FENCE" => { + Some(Self::AcquireSourceWriteFence) + } + "COMMAND_KIND_SET_SOURCE_READ_FENCE" => Some(Self::SetSourceReadFence), + "COMMAND_KIND_CLEAR_SOURCE_READ_FENCE" => Some(Self::ClearSourceReadFence), + "COMMAND_KIND_RELEASE_SOURCE_WRITE_FENCE" => { + Some(Self::ReleaseSourceWriteFence) + } + "COMMAND_KIND_CREATE_TARGET_QUARANTINED" => { + Some(Self::CreateTargetQuarantined) + } + "COMMAND_KIND_SEAL_TARGET_IMPORT" => Some(Self::SealTargetImport), + "COMMAND_KIND_RECORD_TARGET_VALIDATION" => Some(Self::RecordTargetValidation), + "COMMAND_KIND_PUBLISH_TARGET_READABLE_WRITE_FENCED" => { + Some(Self::PublishTargetReadableWriteFenced) + } + "COMMAND_KIND_ENABLE_TARGET_WRITES" => Some(Self::EnableTargetWrites), + "COMMAND_KIND_ABORT_QUARANTINED_TARGET" => Some(Self::AbortQuarantinedTarget), + "COMMAND_KIND_ADOPT_FENCE" => Some(Self::AdoptFence), + _ => None, + } + } +} +/// Only the outcomes a receipt can record. Errors are never stored. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ReceiptOutcome { + Unspecified = 0, + Applied = 1, + AlreadyApplied = 2, + Draining = 3, +} +impl ReceiptOutcome { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ReceiptOutcome::Unspecified => "RECEIPT_OUTCOME_UNSPECIFIED", + ReceiptOutcome::Applied => "RECEIPT_OUTCOME_APPLIED", + ReceiptOutcome::AlreadyApplied => "RECEIPT_OUTCOME_ALREADY_APPLIED", + ReceiptOutcome::Draining => "RECEIPT_OUTCOME_DRAINING", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "RECEIPT_OUTCOME_UNSPECIFIED" => Some(Self::Unspecified), + "RECEIPT_OUTCOME_APPLIED" => Some(Self::Applied), + "RECEIPT_OUTCOME_ALREADY_APPLIED" => Some(Self::AlreadyApplied), + "RECEIPT_OUTCOME_DRAINING" => Some(Self::Draining), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum OnDeadline { + Unspecified = 0, + Fail = 1, + ForceRollback = 2, +} +impl OnDeadline { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + OnDeadline::Unspecified => "ON_DEADLINE_UNSPECIFIED", + OnDeadline::Fail => "ON_DEADLINE_FAIL", + OnDeadline::ForceRollback => "ON_DEADLINE_FORCE_ROLLBACK", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "ON_DEADLINE_UNSPECIFIED" => Some(Self::Unspecified), + "ON_DEADLINE_FAIL" => Some(Self::Fail), + "ON_DEADLINE_FORCE_ROLLBACK" => Some(Self::ForceRollback), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum ValidationResult { + Unspecified = 0, + Ok = 1, + Failed = 2, +} +impl ValidationResult { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + ValidationResult::Unspecified => "VALIDATION_RESULT_UNSPECIFIED", + ValidationResult::Ok => "VALIDATION_RESULT_OK", + ValidationResult::Failed => "VALIDATION_RESULT_FAILED", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "VALIDATION_RESULT_UNSPECIFIED" => Some(Self::Unspecified), + "VALIDATION_RESULT_OK" => Some(Self::Ok), + "VALIDATION_RESULT_FAILED" => Some(Self::Failed), + _ => None, + } + } +} diff --git a/libsql-server/src/namespace/fence/command.rs b/libsql-server/src/namespace/fence/command.rs new file mode 100644 index 0000000000..718fc78d83 --- /dev/null +++ b/libsql-server/src/namespace/fence/command.rs @@ -0,0 +1,420 @@ +//! Fence commands, requests and their canonical fingerprint (`docs/NAMESPACE_FENCE.md` +//! sections 4.2, 4.4 and 5.3). + +use std::fmt; + +use prost::Message as _; +use sha2::{Digest as _, Sha256}; +use uuid::Uuid; + +use crate::namespace::NamespaceName; + +use super::proto; +use super::record::codec; +use super::state::FenceState; + +/// Largest accepted `RecordTargetValidation` summary, in bytes. +pub const MAX_VALIDATION_SUMMARY_BYTES: usize = 4096; + +/// What happens when a drain deadline passes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OnDeadline { + /// Answer `DRAINING`; the durable state stays draining and admission stays closed. + Fail, + /// Roll back (or cancel, for reads) the work still holding the drain open, then keep + /// waiting for it to actually end. + ForceRollback, +} + +impl OnDeadline { + pub const fn as_str(self) -> &'static str { + match self { + OnDeadline::Fail => "fail", + OnDeadline::ForceRollback => "force_rollback", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DrainPolicy { + pub deadline_ms: u64, + pub on_deadline: OnDeadline, +} + +/// The subset of namespace configuration `CreateTargetQuarantined` accepts. Restore options +/// and dump URLs are not part of it: import goes through the migration capability. +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct TargetConfig { + pub max_db_size: Option, + pub jwt_key: Option, + pub txn_timeout_s: Option, + pub allow_attach: bool, + pub durability_mode: Option, + pub bottomless_db_id: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ValidationResult { + Ok, + Failed, +} + +impl ValidationResult { + pub const fn as_str(self) -> &'static str { + match self { + ValidationResult::Ok => "ok", + ValidationResult::Failed => "failed", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AdoptArgs { + /// The operation that currently owns the record, as the adopter believes it. + pub current_operation_id: Uuid, + /// Two distinct, non-empty identities. Recorded, not verified (section 12). + pub approvers: Vec, + pub incident_ref: String, + pub reason: String, +} + +/// A mutating fence command and its command-specific arguments. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum FenceCommand { + AcquireSourceWriteFence { + /// The replication log id the caller observed on the source. + expected_log_id: Uuid, + drain_policy: Option, + }, + SetSourceReadFence { + drain_policy: Option, + }, + ClearSourceReadFence, + ReleaseSourceWriteFence, + CreateTargetQuarantined { + config: TargetConfig, + }, + SealTargetImport { + drain_policy: Option, + }, + RecordTargetValidation { + result: ValidationResult, + summary: String, + }, + PublishTargetReadableWriteFenced, + EnableTargetWrites, + AbortQuarantinedTarget, + AdoptFence(AdoptArgs), +} + +/// The kind of a command, without its arguments. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CommandKind { + AcquireSourceWriteFence, + SetSourceReadFence, + ClearSourceReadFence, + ReleaseSourceWriteFence, + CreateTargetQuarantined, + SealTargetImport, + RecordTargetValidation, + PublishTargetReadableWriteFenced, + EnableTargetWrites, + AbortQuarantinedTarget, + AdoptFence, +} + +impl CommandKind { + pub const ALL: [CommandKind; 11] = [ + CommandKind::AcquireSourceWriteFence, + CommandKind::SetSourceReadFence, + CommandKind::ClearSourceReadFence, + CommandKind::ReleaseSourceWriteFence, + CommandKind::CreateTargetQuarantined, + CommandKind::SealTargetImport, + CommandKind::RecordTargetValidation, + CommandKind::PublishTargetReadableWriteFenced, + CommandKind::EnableTargetWrites, + CommandKind::AbortQuarantinedTarget, + CommandKind::AdoptFence, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + CommandKind::AcquireSourceWriteFence => "AcquireSourceWriteFence", + CommandKind::SetSourceReadFence => "SetSourceReadFence", + CommandKind::ClearSourceReadFence => "ClearSourceReadFence", + CommandKind::ReleaseSourceWriteFence => "ReleaseSourceWriteFence", + CommandKind::CreateTargetQuarantined => "CreateTargetQuarantined", + CommandKind::SealTargetImport => "SealTargetImport", + CommandKind::RecordTargetValidation => "RecordTargetValidation", + CommandKind::PublishTargetReadableWriteFenced => "PublishTargetReadableWriteFenced", + CommandKind::EnableTargetWrites => "EnableTargetWrites", + CommandKind::AbortQuarantinedTarget => "AbortQuarantinedTarget", + CommandKind::AdoptFence => "AdoptFence", + } + } + + /// The state a successful command finally leaves the record in, where that is a single + /// state. A command from the owner whose goal state the record is already in is + /// `ALREADY_APPLIED`. `RecordTargetValidation` and `AdoptFence` do not move the state and + /// have none. + pub const fn goal_state(self) -> Option { + match self { + CommandKind::AcquireSourceWriteFence => Some(FenceState::SourceWriteFenced), + CommandKind::SetSourceReadFence => Some(FenceState::SourceReadFenced), + CommandKind::ClearSourceReadFence => Some(FenceState::SourceWriteFenced), + CommandKind::ReleaseSourceWriteFence => Some(FenceState::Released), + CommandKind::CreateTargetQuarantined => Some(FenceState::TargetQuarantined), + CommandKind::SealTargetImport => Some(FenceState::TargetValidating), + CommandKind::PublishTargetReadableWriteFenced => Some(FenceState::TargetWriteFenced), + CommandKind::EnableTargetWrites => Some(FenceState::TargetWritable), + CommandKind::AbortQuarantinedTarget => Some(FenceState::TargetAborted), + CommandKind::RecordTargetValidation | CommandKind::AdoptFence => None, + } + } +} + +impl fmt::Display for CommandKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FenceCommand { + pub fn kind(&self) -> CommandKind { + match self { + FenceCommand::AcquireSourceWriteFence { .. } => CommandKind::AcquireSourceWriteFence, + FenceCommand::SetSourceReadFence { .. } => CommandKind::SetSourceReadFence, + FenceCommand::ClearSourceReadFence => CommandKind::ClearSourceReadFence, + FenceCommand::ReleaseSourceWriteFence => CommandKind::ReleaseSourceWriteFence, + FenceCommand::CreateTargetQuarantined { .. } => CommandKind::CreateTargetQuarantined, + FenceCommand::SealTargetImport { .. } => CommandKind::SealTargetImport, + FenceCommand::RecordTargetValidation { .. } => CommandKind::RecordTargetValidation, + FenceCommand::PublishTargetReadableWriteFenced => { + CommandKind::PublishTargetReadableWriteFenced + } + FenceCommand::EnableTargetWrites => CommandKind::EnableTargetWrites, + FenceCommand::AbortQuarantinedTarget => CommandKind::AbortQuarantinedTarget, + FenceCommand::AdoptFence(_) => CommandKind::AdoptFence, + } + } +} + +/// One mutating request: the common fields of section 4.2 and the command. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FenceRequest { + pub namespace: NamespaceName, + pub operation_id: Uuid, + pub command_id: Uuid, + /// `Unfenced` or `Absent` for a namespace with no record. + pub expected_state: FenceState, + /// `0` for a namespace with no record. + pub expected_revision: u64, + pub command: FenceCommand, +} + +impl FenceRequest { + /// SHA-256 of the deterministic protobuf encoding of everything in the request except + /// `command_id` (section 5.3). Two requests with the same `(operation_id, command_id)` and + /// different fingerprints are a `FENCE_COMMAND_CONFLICT`. + pub fn fingerprint(&self) -> Fingerprint { + let input = proto::FingerprintInput { + namespace: self.namespace.as_str().to_string(), + operation_id: self.operation_id.to_string(), + kind: codec::command_kind_to_proto(self.command.kind()) as i32, + expected_state: codec::state_to_proto(self.expected_state) as i32, + expected_revision: self.expected_revision, + args: fingerprint_args(&self.command), + }; + Fingerprint(Sha256::digest(input.encode_to_vec()).into()) + } +} + +fn fingerprint_args(command: &FenceCommand) -> Option { + use proto::fingerprint_input::Args; + + let drain = |p: &Option| proto::DrainArgs { + drain_policy: p.as_ref().map(codec::drain_policy_to_proto), + }; + + match command { + FenceCommand::AcquireSourceWriteFence { + expected_log_id, + drain_policy, + } => Some(Args::AcquireSourceWriteFence( + proto::AcquireSourceWriteFenceArgs { + expected_log_id: expected_log_id.to_string(), + drain_policy: drain_policy.as_ref().map(codec::drain_policy_to_proto), + }, + )), + FenceCommand::SetSourceReadFence { drain_policy } => { + Some(Args::SetSourceReadFence(drain(drain_policy))) + } + FenceCommand::SealTargetImport { drain_policy } => { + Some(Args::SealTargetImport(drain(drain_policy))) + } + FenceCommand::CreateTargetQuarantined { config } => Some(Args::CreateTargetQuarantined( + codec::target_config_to_proto(config), + )), + FenceCommand::RecordTargetValidation { result, summary } => Some( + Args::RecordTargetValidation(proto::RecordTargetValidationArgs { + result: codec::validation_result_to_proto(*result) as i32, + summary: summary.clone(), + }), + ), + FenceCommand::AdoptFence(args) => Some(Args::AdoptFence(proto::AdoptFenceArgs { + current_operation_id: args.current_operation_id.to_string(), + approvers: args.approvers.clone(), + incident_ref: args.incident_ref.clone(), + reason: args.reason.clone(), + })), + FenceCommand::ClearSourceReadFence + | FenceCommand::ReleaseSourceWriteFence + | FenceCommand::PublishTargetReadableWriteFenced + | FenceCommand::EnableTargetWrites + | FenceCommand::AbortQuarantinedTarget => None, + } +} + +/// A canonical request fingerprint. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct Fingerprint(pub [u8; 32]); + +impl Fingerprint { + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +impl fmt::Display for Fingerprint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("sha256:")?; + for b in self.0 { + write!(f, "{b:02x}")?; + } + Ok(()) + } +} + +impl fmt::Debug for Fingerprint { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request(command: FenceCommand) -> FenceRequest { + FenceRequest { + namespace: NamespaceName::from("db1"), + operation_id: Uuid::from_u128(1), + command_id: Uuid::from_u128(2), + expected_state: FenceState::Unfenced, + expected_revision: 0, + command, + } + } + + fn acquire() -> FenceCommand { + FenceCommand::AcquireSourceWriteFence { + expected_log_id: Uuid::from_u128(3), + drain_policy: Some(DrainPolicy { + deadline_ms: 1000, + on_deadline: OnDeadline::Fail, + }), + } + } + + #[test] + fn fingerprint_ignores_command_id() { + let a = request(acquire()); + let mut b = a.clone(); + b.command_id = Uuid::from_u128(99); + assert_eq!(a.fingerprint(), b.fingerprint()); + } + + #[test] + fn fingerprint_covers_every_other_field() { + let base = request(acquire()); + let fp = base.fingerprint(); + + let mut changed = Vec::new(); + let mut r = base.clone(); + r.namespace = NamespaceName::from("db2"); + changed.push(r); + let mut r = base.clone(); + r.operation_id = Uuid::from_u128(7); + changed.push(r); + let mut r = base.clone(); + r.expected_state = FenceState::Released; + changed.push(r); + let mut r = base.clone(); + r.expected_revision = 1; + changed.push(r); + let mut r = base.clone(); + r.command = FenceCommand::AcquireSourceWriteFence { + expected_log_id: Uuid::from_u128(4), + drain_policy: Some(DrainPolicy { + deadline_ms: 1000, + on_deadline: OnDeadline::Fail, + }), + }; + changed.push(r); + let mut r = base.clone(); + r.command = FenceCommand::AcquireSourceWriteFence { + expected_log_id: Uuid::from_u128(3), + drain_policy: Some(DrainPolicy { + deadline_ms: 1000, + on_deadline: OnDeadline::ForceRollback, + }), + }; + changed.push(r); + let mut r = base.clone(); + r.command = FenceCommand::AcquireSourceWriteFence { + expected_log_id: Uuid::from_u128(3), + drain_policy: None, + }; + changed.push(r); + + for r in changed { + assert_ne!(r.fingerprint(), fp, "{r:?}"); + } + } + + #[test] + fn fingerprint_distinguishes_argumentless_commands() { + let kinds = [ + FenceCommand::ClearSourceReadFence, + FenceCommand::ReleaseSourceWriteFence, + FenceCommand::PublishTargetReadableWriteFenced, + FenceCommand::EnableTargetWrites, + FenceCommand::AbortQuarantinedTarget, + FenceCommand::SetSourceReadFence { drain_policy: None }, + FenceCommand::SealTargetImport { drain_policy: None }, + ]; + let fps: std::collections::HashSet<_> = kinds + .into_iter() + .map(|c| request(c).fingerprint()) + .collect(); + assert_eq!(fps.len(), 7); + } + + /// The fingerprint is stored in receipts and compared on every replay, so its encoding must + /// never change: a change would turn every stored receipt into a command conflict. + #[test] + fn fingerprint_is_stable() { + assert_eq!( + request(acquire()).fingerprint().to_string(), + "sha256:c585d3570b5eb5a3ef9b8efb89eca26e2336c7e19a561fa3db667c4cde905515" + ); + } + + #[test] + fn every_kind_has_a_name() { + let names: std::collections::HashSet<_> = + CommandKind::ALL.iter().map(|k| k.as_str()).collect(); + assert_eq!(names.len(), CommandKind::ALL.len()); + } +} diff --git a/libsql-server/src/namespace/fence/mod.rs b/libsql-server/src/namespace/fence/mod.rs new file mode 100644 index 0000000000..3ffe2746f8 --- /dev/null +++ b/libsql-server/src/namespace/fence/mod.rs @@ -0,0 +1,28 @@ +//! Namespace fence: a durable, operation-owned control record that an external operation (for +//! example, moving a database between servers) uses as the data-plane authority boundary for +//! one namespace. +//! +//! `docs/NAMESPACE_FENCE.md` is the contract and the design. This module holds the parts with +//! no I/O: the states and permission matrix ([`state`]), the stable outcome codes and their +//! protocol mappings ([`outcome`]), commands and their canonical fingerprint ([`command`]), +//! records, receipts and markers with their strict durable encoding ([`record`]), and the pure +//! transition function ([`transition`]). + +// The persistence, controller and protocol layers that consume these types land in the +// following commits of this series; until then most of the module is unused by the rest of +// the crate. This attribute is removed once they are wired. +#![allow(dead_code)] + +pub mod command; +pub mod outcome; +pub mod record; +pub mod state; +pub mod transition; + +#[allow(clippy::all)] +pub(crate) mod proto { + include!("../../generated/namespace_fence.rs"); +} + +/// Version of the fence admin protocol reported by capability discovery. +pub const FENCE_PROTOCOL_VERSION: u32 = 1; diff --git a/libsql-server/src/namespace/fence/outcome.rs b/libsql-server/src/namespace/fence/outcome.rs new file mode 100644 index 0000000000..6a9b6a8b57 --- /dev/null +++ b/libsql-server/src/namespace/fence/outcome.rs @@ -0,0 +1,409 @@ +//! Stable outcome codes and their protocol mappings (`docs/NAMESPACE_FENCE.md` section 6). + +use std::fmt; +use std::str::FromStr; + +use hyper::StatusCode; + +/// gRPC metadata key carrying the stable code of a fence denial. +pub const GRPC_FENCE_CODE_METADATA: &str = "x-libsql-fence-code"; + +/// Every machine-readable outcome a fence command or a fenced data-plane operation can report. +/// Clients match on [`FenceOutcome::as_str`], never on a message. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FenceOutcome { + Applied, + AlreadyApplied, + Draining, + MigrationWriteFenced, + MigrationReadFenced, + MigrationTargetQuarantined, + FenceStateUnavailable, + OperationCapabilityRequired, + FenceOwnedByAnotherOperation, + FenceRevisionMismatch, + InvalidFenceTransition, + FenceCommandConflict, + FenceCommitIndeterminate, + FencePreconditionFailed, +} + +/// What kind of answer an outcome is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OutcomeKind { + Success, + InProgress, + /// A denial of ordinary data-plane or lifecycle work because of the fence. + DataPlane, + /// A refusal of a fence command (or of capability work). + Control, +} + +impl FenceOutcome { + pub const ALL: [FenceOutcome; 14] = [ + FenceOutcome::Applied, + FenceOutcome::AlreadyApplied, + FenceOutcome::Draining, + FenceOutcome::MigrationWriteFenced, + FenceOutcome::MigrationReadFenced, + FenceOutcome::MigrationTargetQuarantined, + FenceOutcome::FenceStateUnavailable, + FenceOutcome::OperationCapabilityRequired, + FenceOutcome::FenceOwnedByAnotherOperation, + FenceOutcome::FenceRevisionMismatch, + FenceOutcome::InvalidFenceTransition, + FenceOutcome::FenceCommandConflict, + FenceOutcome::FenceCommitIndeterminate, + FenceOutcome::FencePreconditionFailed, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + FenceOutcome::Applied => "APPLIED", + FenceOutcome::AlreadyApplied => "ALREADY_APPLIED", + FenceOutcome::Draining => "DRAINING", + FenceOutcome::MigrationWriteFenced => "MIGRATION_WRITE_FENCED", + FenceOutcome::MigrationReadFenced => "MIGRATION_READ_FENCED", + FenceOutcome::MigrationTargetQuarantined => "MIGRATION_TARGET_QUARANTINED", + FenceOutcome::FenceStateUnavailable => "FENCE_STATE_UNAVAILABLE", + FenceOutcome::OperationCapabilityRequired => "OPERATION_CAPABILITY_REQUIRED", + FenceOutcome::FenceOwnedByAnotherOperation => "FENCE_OWNED_BY_ANOTHER_OPERATION", + FenceOutcome::FenceRevisionMismatch => "FENCE_REVISION_MISMATCH", + FenceOutcome::InvalidFenceTransition => "INVALID_FENCE_TRANSITION", + FenceOutcome::FenceCommandConflict => "FENCE_COMMAND_CONFLICT", + FenceOutcome::FenceCommitIndeterminate => "FENCE_COMMIT_INDETERMINATE", + FenceOutcome::FencePreconditionFailed => "FENCE_PRECONDITION_FAILED", + } + } + + pub const fn kind(self) -> OutcomeKind { + match self { + FenceOutcome::Applied | FenceOutcome::AlreadyApplied => OutcomeKind::Success, + FenceOutcome::Draining => OutcomeKind::InProgress, + FenceOutcome::MigrationWriteFenced + | FenceOutcome::MigrationReadFenced + | FenceOutcome::MigrationTargetQuarantined + | FenceOutcome::FenceStateUnavailable => OutcomeKind::DataPlane, + FenceOutcome::OperationCapabilityRequired + | FenceOutcome::FenceOwnedByAnotherOperation + | FenceOutcome::FenceRevisionMismatch + | FenceOutcome::InvalidFenceTransition + | FenceOutcome::FenceCommandConflict + | FenceOutcome::FenceCommitIndeterminate + | FenceOutcome::FencePreconditionFailed => OutcomeKind::Control, + } + } + + pub const fn is_error(self) -> bool { + matches!(self.kind(), OutcomeKind::DataPlane | OutcomeKind::Control) + } + + /// Status code on the admin API. + pub fn admin_http_status(self) -> StatusCode { + match self { + FenceOutcome::Applied | FenceOutcome::AlreadyApplied => StatusCode::OK, + FenceOutcome::Draining => StatusCode::ACCEPTED, + FenceOutcome::MigrationWriteFenced + | FenceOutcome::MigrationReadFenced + | FenceOutcome::MigrationTargetQuarantined + | FenceOutcome::FenceStateUnavailable => StatusCode::LOCKED, + FenceOutcome::OperationCapabilityRequired => StatusCode::FORBIDDEN, + FenceOutcome::FenceOwnedByAnotherOperation + | FenceOutcome::FenceRevisionMismatch + | FenceOutcome::InvalidFenceTransition + | FenceOutcome::FenceCommandConflict + | FenceOutcome::FenceCommitIndeterminate => StatusCode::CONFLICT, + FenceOutcome::FencePreconditionFailed => StatusCode::PRECONDITION_FAILED, + } + } + + /// Status code on the user HTTP API (`/`, `/v1`, `/v2`, `/v3`, `/dump`). Only data-plane + /// denials reach it. + pub fn user_http_status(self) -> Option { + match self.kind() { + OutcomeKind::DataPlane => Some(StatusCode::LOCKED), + _ => None, + } + } + + /// The Hrana error `code`. Only data-plane denials reach Hrana. + pub fn hrana_code(self) -> Option<&'static str> { + match self.kind() { + OutcomeKind::DataPlane => Some(self.as_str()), + _ => None, + } + } + + /// The gRPC status code on RPC, proxy connection and replication services. Never + /// `UNAVAILABLE`, which the write proxy retries without bound. + pub fn grpc_code(self) -> Option { + match self { + FenceOutcome::MigrationWriteFenced + | FenceOutcome::MigrationReadFenced + | FenceOutcome::MigrationTargetQuarantined + | FenceOutcome::FenceStateUnavailable + | FenceOutcome::OperationCapabilityRequired => Some(tonic::Code::FailedPrecondition), + _ => None, + } + } + + /// The value of the proxy protocol's `Error.stable_code` field. + pub fn proxy_stable_code(self) -> Option<&'static str> { + match self.kind() { + OutcomeKind::DataPlane => Some(self.as_str()), + _ => None, + } + } +} + +impl fmt::Display for FenceOutcome { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unknown fence outcome `{0}`")] +pub struct UnknownFenceOutcome(pub String); + +impl FromStr for FenceOutcome { + type Err = UnknownFenceOutcome; + + fn from_str(s: &str) -> Result { + FenceOutcome::ALL + .iter() + .copied() + .find(|o| o.as_str() == s) + .ok_or_else(|| UnknownFenceOutcome(s.to_string())) + } +} + +/// The bounded `detail` reason of an error outcome. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FenceDetail { + // FENCE_PRECONDITION_FAILED + AdminAuthRequired, + FenceDisabled, + NotPrimary, + SharedSchemaUnsupported, + NamespaceIdentityMismatch, + NamespaceExists, + ValidationReceiptRequired, + RestoreNotAllowed, + AdoptionNotAuthorised, + InvalidArgument, + // INVALID_FENCE_TRANSITION + RoleMismatch, + OperationFinished, + // FENCE_STATE_UNAVAILABLE + CorruptRecord, + UnsupportedFormatVersion, + IncompleteTargetCreation, + MetastoreBehindMarker, + IndeterminateCommit, +} + +impl FenceDetail { + pub const fn as_str(self) -> &'static str { + match self { + FenceDetail::AdminAuthRequired => "admin_auth_required", + FenceDetail::FenceDisabled => "fence_disabled", + FenceDetail::NotPrimary => "not_primary", + FenceDetail::SharedSchemaUnsupported => "shared_schema_unsupported", + FenceDetail::NamespaceIdentityMismatch => "namespace_identity_mismatch", + FenceDetail::NamespaceExists => "namespace_exists", + FenceDetail::ValidationReceiptRequired => "validation_receipt_required", + FenceDetail::RestoreNotAllowed => "restore_not_allowed", + FenceDetail::AdoptionNotAuthorised => "adoption_not_authorised", + FenceDetail::InvalidArgument => "invalid_argument", + FenceDetail::RoleMismatch => "role_mismatch", + FenceDetail::OperationFinished => "operation_finished", + FenceDetail::CorruptRecord => "corrupt_record", + FenceDetail::UnsupportedFormatVersion => "unsupported_format_version", + FenceDetail::IncompleteTargetCreation => "incomplete_target_creation", + FenceDetail::MetastoreBehindMarker => "metastore_behind_marker", + FenceDetail::IndeterminateCommit => "indeterminate_commit", + } + } +} + +impl fmt::Display for FenceDetail { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// An error outcome with its bounded detail and a human message. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{outcome}: {message}")] +pub struct FenceError { + outcome: FenceOutcome, + detail: Option, + message: String, +} + +impl FenceError { + /// # Panics + /// + /// If `outcome` is not an error outcome. That is a programming error, never input. + pub fn new(outcome: FenceOutcome, message: impl Into) -> Self { + assert!(outcome.is_error(), "{outcome} is not an error outcome"); + Self { + outcome, + detail: None, + message: message.into(), + } + } + + pub fn with_detail(mut self, detail: FenceDetail) -> Self { + self.detail = Some(detail); + self + } + + pub fn outcome(&self) -> FenceOutcome { + self.outcome + } + + pub fn detail(&self) -> Option { + self.detail + } + + pub fn message(&self) -> &str { + &self.message + } + + /// A gRPC status for this error, if the outcome has a gRPC mapping. The code is in the + /// [`GRPC_FENCE_CODE_METADATA`] entry and prefixes the message. + pub fn to_grpc_status(&self) -> Option { + let code = self.outcome.grpc_code()?; + let mut status = tonic::Status::new(code, format!("{}: {}", self.outcome, self.message)); + status.metadata_mut().insert( + GRPC_FENCE_CODE_METADATA, + tonic::metadata::MetadataValue::from_static(self.outcome.as_str()), + ); + Some(status) + } + + /// The stable code carried by a gRPC status produced by [`FenceError::to_grpc_status`]. + pub fn outcome_from_grpc_status(status: &tonic::Status) -> Option { + status + .metadata() + .get(GRPC_FENCE_CODE_METADATA)? + .to_str() + .ok()? + .parse() + .ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codes_round_trip() { + for outcome in FenceOutcome::ALL { + assert_eq!(outcome.as_str().parse::().unwrap(), outcome); + } + assert!("applied".parse::().is_err()); + } + + /// Section 6: data-plane denials are never 500, 503, 429 or gRPC UNAVAILABLE. + #[test] + fn denials_are_never_retryable_statuses() { + let retryable = [ + StatusCode::INTERNAL_SERVER_ERROR, + StatusCode::SERVICE_UNAVAILABLE, + StatusCode::TOO_MANY_REQUESTS, + StatusCode::BAD_GATEWAY, + StatusCode::GATEWAY_TIMEOUT, + ]; + for outcome in FenceOutcome::ALL { + assert!( + !retryable.contains(&outcome.admin_http_status()), + "{outcome}" + ); + if let Some(status) = outcome.user_http_status() { + assert!(!retryable.contains(&status), "{outcome}"); + } + assert_ne!(outcome.grpc_code(), Some(tonic::Code::Unavailable)); + } + } + + #[test] + fn protocol_table() { + use FenceOutcome as O; + let rows = [ + (O::Applied, 200, None, None), + (O::AlreadyApplied, 200, None, None), + (O::Draining, 202, None, None), + ( + O::MigrationWriteFenced, + 423, + Some(423), + Some("MIGRATION_WRITE_FENCED"), + ), + ( + O::MigrationReadFenced, + 423, + Some(423), + Some("MIGRATION_READ_FENCED"), + ), + ( + O::MigrationTargetQuarantined, + 423, + Some(423), + Some("MIGRATION_TARGET_QUARANTINED"), + ), + ( + O::FenceStateUnavailable, + 423, + Some(423), + Some("FENCE_STATE_UNAVAILABLE"), + ), + (O::OperationCapabilityRequired, 403, None, None), + (O::FenceOwnedByAnotherOperation, 409, None, None), + (O::FenceRevisionMismatch, 409, None, None), + (O::InvalidFenceTransition, 409, None, None), + (O::FenceCommandConflict, 409, None, None), + (O::FenceCommitIndeterminate, 409, None, None), + (O::FencePreconditionFailed, 412, None, None), + ]; + assert_eq!(rows.len(), FenceOutcome::ALL.len()); + for (outcome, admin, user, hrana) in rows { + assert_eq!(outcome.admin_http_status().as_u16(), admin, "{outcome}"); + assert_eq!( + outcome.user_http_status().map(|s| s.as_u16()), + user, + "{outcome}" + ); + assert_eq!(outcome.hrana_code(), hrana, "{outcome}"); + assert_eq!(outcome.proxy_stable_code(), hrana, "{outcome}"); + } + } + + #[test] + fn grpc_status_carries_code() { + let err = FenceError::new(FenceOutcome::MigrationReadFenced, "reads are fenced"); + let status = err.to_grpc_status().unwrap(); + assert_eq!(status.code(), tonic::Code::FailedPrecondition); + assert!(status.message().starts_with("MIGRATION_READ_FENCED: ")); + assert_eq!( + FenceError::outcome_from_grpc_status(&status), + Some(FenceOutcome::MigrationReadFenced) + ); + assert_eq!( + FenceError::outcome_from_grpc_status(&tonic::Status::unavailable("x")), + None + ); + + let control = FenceError::new(FenceOutcome::FenceRevisionMismatch, "stale"); + assert!(control.to_grpc_status().is_none()); + } + + #[test] + #[should_panic] + fn success_is_not_an_error() { + FenceError::new(FenceOutcome::Applied, "nope"); + } +} diff --git a/libsql-server/src/namespace/fence/record.rs b/libsql-server/src/namespace/fence/record.rs new file mode 100644 index 0000000000..9599f2a849 --- /dev/null +++ b/libsql-server/src/namespace/fence/record.rs @@ -0,0 +1,908 @@ +//! Fence records, command receipts, the on-disk marker, and their durable encoding +//! (`docs/NAMESPACE_FENCE.md` sections 5.1, 5.2 and 5.6). +//! +//! Decoding is strict: an unknown format version, an unknown enum value, a missing required +//! field, a malformed id or a record that contradicts itself is an error, which the store turns +//! into `UNKNOWN_UNAVAILABLE`. Nothing is defaulted. + +use prost::Message as _; +use uuid::Uuid; + +use crate::namespace::NamespaceName; + +use super::command::{CommandKind, DrainPolicy, Fingerprint, TargetConfig, ValidationResult}; +use super::outcome::FenceOutcome; +use super::proto; +use super::state::{Admission, FenceState, Role}; + +/// The only `format_version` this server writes and reads. +pub const FENCE_FORMAT_VERSION: u32 = 1; + +/// Identity of the namespace copy a record is about. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct NamespaceIdentity { + /// Replication log id: for a source, captured at acquisition and checked against the + /// caller's expectation; for a target, known once the namespace exists. + pub log_id: Option, + /// Server-generated id of a target created by `CreateTargetQuarantined`. + pub target_incarnation_id: Option, +} + +/// The source's replication position once no writer can commit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrozenBoundary { + pub log_id: Uuid, + pub frame_no: u64, +} + +/// The pre-fence values of the legacy `block_*` configuration fields, restored when the +/// operation finishes (section 13.2). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LegacyBlocks { + pub block_reads: bool, + pub block_writes: bool, + pub block_reason: Option, +} + +/// The server process that wrote something. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServerIdentity { + pub build: String, + pub instance_id: Uuid, +} + +/// What the server observed of a target when a validation result was recorded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ValidationSnapshot { + pub log_id: Uuid, + pub frame_no: u64, + pub page_count: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ValidationRecord { + pub operation_id: Uuid, + pub command_id: Uuid, + pub result: ValidationResult, + pub summary: String, + pub snapshot: Option, + pub recorded_at_ms: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Adoption { + pub previous_operation_id: Uuid, + pub new_operation_id: Uuid, + pub command_id: Uuid, + pub approvers: Vec, + pub incident_ref: String, + pub reason: String, + pub at_ms: i64, + /// The revision the adoption produced. + pub revision: u64, +} + +/// The durable control record of one fenced namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NamespaceFenceRecord { + pub namespace: NamespaceName, + pub role: Role, + /// Always a durable state whose role is `role`. + pub state: FenceState, + /// Starts at 1 and increases by one on every applied transition. Survives restart. + pub revision: u64, + pub operation_id: Uuid, + pub identity: NamespaceIdentity, + pub drain_policy: Option, + /// When the current drain was requested, if the record is draining. + pub drain_started_at_ms: Option, + pub frozen_boundary: Option, + /// The most recent validation result of the owning operation (target). + pub validation: Option, + pub legacy_blocks: LegacyBlocks, + pub created_at_ms: i64, + pub last_transition_at_ms: i64, + /// The command that produced the current revision. + pub last_command_id: Uuid, + pub written_by: ServerIdentity, + pub adoptions: Vec, +} + +impl NamespaceFenceRecord { + pub fn write_admission(&self) -> Admission { + self.state.write_admission() + } + + pub fn read_admission(&self) -> Admission { + self.state.read_admission() + } + + /// Values of the legacy `block_*` configuration fields while this record is in force: the + /// fence state mirrored for an older binary, or the pre-fence values once the operation + /// has released the namespace. + pub fn legacy_mirror(&self) -> LegacyBlocks { + match self.state { + FenceState::Released | FenceState::TargetWritable => self.legacy_blocks.clone(), + state => LegacyBlocks { + block_reads: !state.read_admission().is_open(), + block_writes: !state.write_admission().is_open(), + block_reason: Some(format!( + "namespace fence: {state} (operation {})", + self.operation_id + )), + }, + } + } + + pub fn encode(&self) -> Vec { + codec::record_to_proto(self).encode_to_vec() + } + + /// Decode a stored record. `format_version` and `revision` are the columns stored beside + /// the payload; both must agree with it. + pub fn decode( + format_version: u32, + revision: u64, + bytes: &[u8], + ) -> Result { + check_format_version(format_version)?; + let msg = proto::FenceRecord::decode(bytes)?; + let record = codec::record_from_proto(msg)?; + if record.revision != revision { + return Err(FenceDecodeError::Invalid( + "revision column disagrees with payload", + )); + } + Ok(record) + } +} + +/// The durable result of one applied command, keyed by `(namespace, operation_id, command_id)`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommandReceipt { + pub namespace: NamespaceName, + pub operation_id: Uuid, + pub command_id: Uuid, + pub command: CommandKind, + pub fingerprint: Fingerprint, + /// `APPLIED`, `ALREADY_APPLIED` or `DRAINING`. Errors are never stored. + pub outcome: FenceOutcome, + pub revision_before: u64, + pub revision_after: u64, + pub state_after: FenceState, + pub applied_at_ms: i64, + pub instance_id: Uuid, + pub adoption: Option, +} + +impl CommandReceipt { + /// Whether the receipt holds the command's final answer, as opposed to a drain that is + /// still to be completed. + pub fn is_final(&self) -> bool { + self.outcome != FenceOutcome::Draining + } + + pub fn encode(&self) -> Vec { + codec::receipt_to_proto(self).encode_to_vec() + } + + pub fn decode(format_version: u32, bytes: &[u8]) -> Result { + check_format_version(format_version)?; + let msg = proto::CommandReceipt::decode(bytes)?; + codec::receipt_from_proto(msg) + } +} + +/// Contents of the per-namespace marker file: a copy of the last committed record. Written +/// after each metastore commit, and before the metastore transaction of +/// `CreateTargetQuarantined`, so recovery can tell a fenced namespace from a legacy one and +/// detect a metastore that went backwards (section 5.6). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FenceMarker { + pub record: NamespaceFenceRecord, +} + +impl FenceMarker { + pub fn for_record(record: &NamespaceFenceRecord) -> Self { + Self { + record: record.clone(), + } + } + + pub fn encode(&self) -> Vec { + proto::FenceMarker { + format_version: FENCE_FORMAT_VERSION, + record: Some(codec::record_to_proto(&self.record)), + } + .encode_to_vec() + } + + pub fn decode(bytes: &[u8]) -> Result { + let msg = proto::FenceMarker::decode(bytes)?; + check_format_version(msg.format_version)?; + let record = msg + .record + .ok_or(FenceDecodeError::Invalid("marker without record"))?; + Ok(Self { + record: codec::record_from_proto(record)?, + }) + } +} + +/// Why stored fence state could not be read. Every variant means the namespace is +/// `UNKNOWN_UNAVAILABLE`. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum FenceDecodeError { + #[error("unsupported fence format version {0}")] + UnsupportedFormatVersion(u32), + #[error("undecodable fence payload: {0}")] + Undecodable(String), + #[error("invalid fence payload: {0}")] + Invalid(&'static str), +} + +impl From for FenceDecodeError { + fn from(e: prost::DecodeError) -> Self { + FenceDecodeError::Undecodable(e.to_string()) + } +} + +fn check_format_version(v: u32) -> Result<(), FenceDecodeError> { + if v != FENCE_FORMAT_VERSION { + return Err(FenceDecodeError::UnsupportedFormatVersion(v)); + } + Ok(()) +} + +/// Conversions between the domain types and their protobuf encoding. +pub(super) mod codec { + use super::*; + use crate::namespace::fence::command::OnDeadline; + + type R = Result; + + pub fn parse_uuid(s: &str) -> R { + // Only the canonical hyphenated form is ever written. + let id = Uuid::parse_str(s).map_err(|_| FenceDecodeError::Invalid("malformed id"))?; + if id.hyphenated().to_string() != s { + return Err(FenceDecodeError::Invalid("non-canonical id")); + } + Ok(id) + } + + fn parse_opt_uuid(s: Option<&str>) -> R> { + s.map(parse_uuid).transpose() + } + + fn namespace(s: String) -> R { + NamespaceName::from_string(s).map_err(|_| FenceDecodeError::Invalid("invalid namespace")) + } + + pub fn role_to_proto(role: Role) -> proto::FenceRole { + match role { + Role::Source => proto::FenceRole::Source, + Role::Target => proto::FenceRole::Target, + } + } + + pub fn role_from_proto(v: i32) -> R { + match proto::FenceRole::try_from(v) { + Ok(proto::FenceRole::Source) => Ok(Role::Source), + Ok(proto::FenceRole::Target) => Ok(Role::Target), + _ => Err(FenceDecodeError::Invalid("unknown role")), + } + } + + pub fn state_to_proto(state: FenceState) -> proto::FenceState { + use proto::FenceState as P; + match state { + FenceState::Unfenced => P::Unfenced, + FenceState::Absent => P::Absent, + FenceState::SourceDraining => P::SourceDraining, + FenceState::SourceWriteFenced => P::SourceWriteFenced, + FenceState::SourceReadDraining => P::SourceReadDraining, + FenceState::SourceReadFenced => P::SourceReadFenced, + FenceState::Released => P::Released, + FenceState::TargetQuarantined => P::TargetQuarantined, + FenceState::TargetImportDraining => P::TargetImportDraining, + FenceState::TargetValidating => P::TargetValidating, + FenceState::TargetWriteFenced => P::TargetWriteFenced, + FenceState::TargetWritable => P::TargetWritable, + FenceState::TargetAborted => P::TargetAborted, + FenceState::UnknownUnavailable => P::UnknownUnavailable, + } + } + + pub fn state_from_proto(v: i32) -> R { + use proto::FenceState as P; + let p = P::try_from(v).map_err(|_| FenceDecodeError::Invalid("unknown state"))?; + Ok(match p { + P::Unspecified => return Err(FenceDecodeError::Invalid("unspecified state")), + P::Unfenced => FenceState::Unfenced, + P::Absent => FenceState::Absent, + P::SourceDraining => FenceState::SourceDraining, + P::SourceWriteFenced => FenceState::SourceWriteFenced, + P::SourceReadDraining => FenceState::SourceReadDraining, + P::SourceReadFenced => FenceState::SourceReadFenced, + P::Released => FenceState::Released, + P::TargetQuarantined => FenceState::TargetQuarantined, + P::TargetImportDraining => FenceState::TargetImportDraining, + P::TargetValidating => FenceState::TargetValidating, + P::TargetWriteFenced => FenceState::TargetWriteFenced, + P::TargetWritable => FenceState::TargetWritable, + P::TargetAborted => FenceState::TargetAborted, + P::UnknownUnavailable => FenceState::UnknownUnavailable, + }) + } + + /// A state stored in a record or marker: durable, and of the stated role. + pub fn durable_state_from_proto(v: i32, role: Role) -> R { + let state = state_from_proto(v)?; + match state.role() { + Some(r) if r == role => Ok(state), + Some(_) => Err(FenceDecodeError::Invalid("state does not match role")), + None => Err(FenceDecodeError::Invalid("state is not durable")), + } + } + + pub fn command_kind_to_proto(kind: CommandKind) -> proto::CommandKind { + use proto::CommandKind as P; + match kind { + CommandKind::AcquireSourceWriteFence => P::AcquireSourceWriteFence, + CommandKind::SetSourceReadFence => P::SetSourceReadFence, + CommandKind::ClearSourceReadFence => P::ClearSourceReadFence, + CommandKind::ReleaseSourceWriteFence => P::ReleaseSourceWriteFence, + CommandKind::CreateTargetQuarantined => P::CreateTargetQuarantined, + CommandKind::SealTargetImport => P::SealTargetImport, + CommandKind::RecordTargetValidation => P::RecordTargetValidation, + CommandKind::PublishTargetReadableWriteFenced => P::PublishTargetReadableWriteFenced, + CommandKind::EnableTargetWrites => P::EnableTargetWrites, + CommandKind::AbortQuarantinedTarget => P::AbortQuarantinedTarget, + CommandKind::AdoptFence => P::AdoptFence, + } + } + + fn command_kind_from_proto(v: i32) -> R { + use proto::CommandKind as P; + let p = P::try_from(v).map_err(|_| FenceDecodeError::Invalid("unknown command"))?; + Ok(match p { + P::Unspecified => return Err(FenceDecodeError::Invalid("unspecified command")), + P::AcquireSourceWriteFence => CommandKind::AcquireSourceWriteFence, + P::SetSourceReadFence => CommandKind::SetSourceReadFence, + P::ClearSourceReadFence => CommandKind::ClearSourceReadFence, + P::ReleaseSourceWriteFence => CommandKind::ReleaseSourceWriteFence, + P::CreateTargetQuarantined => CommandKind::CreateTargetQuarantined, + P::SealTargetImport => CommandKind::SealTargetImport, + P::RecordTargetValidation => CommandKind::RecordTargetValidation, + P::PublishTargetReadableWriteFenced => CommandKind::PublishTargetReadableWriteFenced, + P::EnableTargetWrites => CommandKind::EnableTargetWrites, + P::AbortQuarantinedTarget => CommandKind::AbortQuarantinedTarget, + P::AdoptFence => CommandKind::AdoptFence, + }) + } + + fn outcome_to_proto(outcome: FenceOutcome) -> proto::ReceiptOutcome { + match outcome { + FenceOutcome::Applied => proto::ReceiptOutcome::Applied, + FenceOutcome::AlreadyApplied => proto::ReceiptOutcome::AlreadyApplied, + FenceOutcome::Draining => proto::ReceiptOutcome::Draining, + other => unreachable!("receipts never store {other}"), + } + } + + fn outcome_from_proto(v: i32) -> R { + match proto::ReceiptOutcome::try_from(v) { + Ok(proto::ReceiptOutcome::Applied) => Ok(FenceOutcome::Applied), + Ok(proto::ReceiptOutcome::AlreadyApplied) => Ok(FenceOutcome::AlreadyApplied), + Ok(proto::ReceiptOutcome::Draining) => Ok(FenceOutcome::Draining), + _ => Err(FenceDecodeError::Invalid("unknown receipt outcome")), + } + } + + pub fn drain_policy_to_proto(p: &DrainPolicy) -> proto::DrainPolicy { + proto::DrainPolicy { + deadline_ms: p.deadline_ms, + on_deadline: match p.on_deadline { + OnDeadline::Fail => proto::OnDeadline::Fail, + OnDeadline::ForceRollback => proto::OnDeadline::ForceRollback, + } as i32, + } + } + + fn drain_policy_from_proto(p: proto::DrainPolicy) -> R { + let on_deadline = match proto::OnDeadline::try_from(p.on_deadline) { + Ok(proto::OnDeadline::Fail) => OnDeadline::Fail, + Ok(proto::OnDeadline::ForceRollback) => OnDeadline::ForceRollback, + _ => return Err(FenceDecodeError::Invalid("unknown drain deadline policy")), + }; + Ok(DrainPolicy { + deadline_ms: p.deadline_ms, + on_deadline, + }) + } + + pub fn validation_result_to_proto(r: ValidationResult) -> proto::ValidationResult { + match r { + ValidationResult::Ok => proto::ValidationResult::Ok, + ValidationResult::Failed => proto::ValidationResult::Failed, + } + } + + fn validation_result_from_proto(v: i32) -> R { + match proto::ValidationResult::try_from(v) { + Ok(proto::ValidationResult::Ok) => Ok(ValidationResult::Ok), + Ok(proto::ValidationResult::Failed) => Ok(ValidationResult::Failed), + _ => Err(FenceDecodeError::Invalid("unknown validation result")), + } + } + + pub fn target_config_to_proto(c: &TargetConfig) -> proto::TargetConfig { + proto::TargetConfig { + max_db_size: c.max_db_size, + jwt_key: c.jwt_key.clone(), + txn_timeout_s: c.txn_timeout_s, + allow_attach: c.allow_attach, + durability_mode: c.durability_mode.clone(), + bottomless_db_id: c.bottomless_db_id.clone(), + } + } + + fn validation_to_proto(v: &ValidationRecord) -> proto::ValidationRecord { + proto::ValidationRecord { + operation_id: v.operation_id.to_string(), + command_id: v.command_id.to_string(), + result: validation_result_to_proto(v.result) as i32, + summary: v.summary.clone(), + snapshot: v.snapshot.map(|s| proto::ValidationSnapshot { + log_id: s.log_id.to_string(), + frame_no: s.frame_no, + page_count: s.page_count, + }), + recorded_at_ms: v.recorded_at_ms, + } + } + + fn validation_from_proto(v: proto::ValidationRecord) -> R { + Ok(ValidationRecord { + operation_id: parse_uuid(&v.operation_id)?, + command_id: parse_uuid(&v.command_id)?, + result: validation_result_from_proto(v.result)?, + summary: v.summary, + snapshot: v + .snapshot + .map(|s| { + Ok::<_, FenceDecodeError>(ValidationSnapshot { + log_id: parse_uuid(&s.log_id)?, + frame_no: s.frame_no, + page_count: s.page_count, + }) + }) + .transpose()?, + recorded_at_ms: v.recorded_at_ms, + }) + } + + fn adoption_to_proto(a: &Adoption) -> proto::Adoption { + proto::Adoption { + previous_operation_id: a.previous_operation_id.to_string(), + new_operation_id: a.new_operation_id.to_string(), + command_id: a.command_id.to_string(), + approvers: a.approvers.clone(), + incident_ref: a.incident_ref.clone(), + reason: a.reason.clone(), + at_ms: a.at_ms, + revision: a.revision, + } + } + + fn adoption_from_proto(a: proto::Adoption) -> R { + Ok(Adoption { + previous_operation_id: parse_uuid(&a.previous_operation_id)?, + new_operation_id: parse_uuid(&a.new_operation_id)?, + command_id: parse_uuid(&a.command_id)?, + approvers: a.approvers, + incident_ref: a.incident_ref, + reason: a.reason, + at_ms: a.at_ms, + revision: a.revision, + }) + } + + pub fn record_to_proto(r: &NamespaceFenceRecord) -> proto::FenceRecord { + proto::FenceRecord { + namespace: r.namespace.as_str().to_string(), + role: role_to_proto(r.role) as i32, + state: state_to_proto(r.state) as i32, + revision: r.revision, + operation_id: r.operation_id.to_string(), + log_id: r.identity.log_id.map(|id| id.to_string()), + target_incarnation_id: r.identity.target_incarnation_id.map(|id| id.to_string()), + drain_policy: r.drain_policy.as_ref().map(drain_policy_to_proto), + drain_started_at_ms: r.drain_started_at_ms, + frozen_boundary: r.frozen_boundary.map(|b| proto::FrozenBoundary { + log_id: b.log_id.to_string(), + frame_no: b.frame_no, + }), + validation: r.validation.as_ref().map(validation_to_proto), + legacy_blocks: Some(proto::LegacyBlocks { + block_reads: r.legacy_blocks.block_reads, + block_writes: r.legacy_blocks.block_writes, + block_reason: r.legacy_blocks.block_reason.clone(), + }), + created_at_ms: r.created_at_ms, + last_transition_at_ms: r.last_transition_at_ms, + last_command_id: r.last_command_id.to_string(), + written_by: Some(proto::ServerIdentity { + build: r.written_by.build.clone(), + instance_id: r.written_by.instance_id.to_string(), + }), + adoptions: r.adoptions.iter().map(adoption_to_proto).collect(), + } + } + + pub fn record_from_proto(m: proto::FenceRecord) -> R { + let role = role_from_proto(m.role)?; + let state = durable_state_from_proto(m.state, role)?; + if m.revision == 0 { + return Err(FenceDecodeError::Invalid("record revision is zero")); + } + let legacy = m + .legacy_blocks + .ok_or(FenceDecodeError::Invalid("missing legacy blocks"))?; + let written_by = m + .written_by + .ok_or(FenceDecodeError::Invalid("missing server identity"))?; + let record = NamespaceFenceRecord { + namespace: namespace(m.namespace)?, + role, + state, + revision: m.revision, + operation_id: parse_uuid(&m.operation_id)?, + identity: NamespaceIdentity { + log_id: parse_opt_uuid(m.log_id.as_deref())?, + target_incarnation_id: parse_opt_uuid(m.target_incarnation_id.as_deref())?, + }, + drain_policy: m.drain_policy.map(drain_policy_from_proto).transpose()?, + drain_started_at_ms: m.drain_started_at_ms, + frozen_boundary: m + .frozen_boundary + .map(|b| { + Ok::<_, FenceDecodeError>(FrozenBoundary { + log_id: parse_uuid(&b.log_id)?, + frame_no: b.frame_no, + }) + }) + .transpose()?, + validation: m.validation.map(validation_from_proto).transpose()?, + legacy_blocks: LegacyBlocks { + block_reads: legacy.block_reads, + block_writes: legacy.block_writes, + block_reason: legacy.block_reason, + }, + created_at_ms: m.created_at_ms, + last_transition_at_ms: m.last_transition_at_ms, + last_command_id: parse_uuid(&m.last_command_id)?, + written_by: ServerIdentity { + build: written_by.build, + instance_id: parse_uuid(&written_by.instance_id)?, + }, + adoptions: m + .adoptions + .into_iter() + .map(adoption_from_proto) + .collect::>()?, + }; + check_record_invariants(&record)?; + Ok(record) + } + + /// Facts every record written by `apply` satisfies. A record that breaks one was not + /// written by this server and is not trusted. + fn check_record_invariants(r: &NamespaceFenceRecord) -> R<()> { + use FenceState::*; + let frozen_required = matches!( + r.state, + SourceWriteFenced | SourceReadDraining | SourceReadFenced + ); + if frozen_required && r.frozen_boundary.is_none() { + return Err(FenceDecodeError::Invalid( + "fenced source without frozen boundary", + )); + } + if r.role == Role::Source && r.identity.log_id.is_none() { + return Err(FenceDecodeError::Invalid( + "source without namespace identity", + )); + } + if r.role == Role::Target && r.identity.target_incarnation_id.is_none() { + return Err(FenceDecodeError::Invalid("target without incarnation id")); + } + if matches!(r.state, TargetWriteFenced | TargetWritable) + && !r + .validation + .as_ref() + .is_some_and(|v| v.result == ValidationResult::Ok) + { + return Err(FenceDecodeError::Invalid( + "published target without validation", + )); + } + Ok(()) + } + + pub fn receipt_to_proto(r: &CommandReceipt) -> proto::CommandReceipt { + proto::CommandReceipt { + namespace: r.namespace.as_str().to_string(), + operation_id: r.operation_id.to_string(), + command_id: r.command_id.to_string(), + command: command_kind_to_proto(r.command) as i32, + fingerprint: r.fingerprint.as_bytes().to_vec(), + outcome: outcome_to_proto(r.outcome) as i32, + revision_before: r.revision_before, + revision_after: r.revision_after, + state_after: state_to_proto(r.state_after) as i32, + applied_at_ms: r.applied_at_ms, + instance_id: r.instance_id.to_string(), + adoption: r.adoption.as_ref().map(adoption_to_proto), + } + } + + pub fn receipt_from_proto(m: proto::CommandReceipt) -> R { + let fingerprint: [u8; 32] = m + .fingerprint + .as_slice() + .try_into() + .map_err(|_| FenceDecodeError::Invalid("fingerprint is not 32 bytes"))?; + let state_after = state_from_proto(m.state_after)?; + if !state_after.is_durable() { + return Err(FenceDecodeError::Invalid("receipt state is not durable")); + } + if m.revision_after < m.revision_before { + return Err(FenceDecodeError::Invalid("receipt revision went backwards")); + } + Ok(CommandReceipt { + namespace: namespace(m.namespace)?, + operation_id: parse_uuid(&m.operation_id)?, + command_id: parse_uuid(&m.command_id)?, + command: command_kind_from_proto(m.command)?, + fingerprint: Fingerprint(fingerprint), + outcome: outcome_from_proto(m.outcome)?, + revision_before: m.revision_before, + revision_after: m.revision_after, + state_after, + applied_at_ms: m.applied_at_ms, + instance_id: parse_uuid(&m.instance_id)?, + adoption: m.adoption.map(adoption_from_proto).transpose()?, + }) + } +} + +#[cfg(test)] +pub(super) mod tests { + use super::*; + use crate::namespace::fence::command::OnDeadline; + + pub fn sample_record() -> NamespaceFenceRecord { + NamespaceFenceRecord { + namespace: NamespaceName::from("db1"), + role: Role::Source, + state: FenceState::SourceWriteFenced, + revision: 2, + operation_id: Uuid::from_u128(1), + identity: NamespaceIdentity { + log_id: Some(Uuid::from_u128(10)), + target_incarnation_id: None, + }, + drain_policy: Some(DrainPolicy { + deadline_ms: 5000, + on_deadline: OnDeadline::ForceRollback, + }), + drain_started_at_ms: Some(100), + frozen_boundary: Some(FrozenBoundary { + log_id: Uuid::from_u128(10), + frame_no: 1234, + }), + validation: None, + legacy_blocks: LegacyBlocks { + block_reads: false, + block_writes: true, + block_reason: Some("maintenance".into()), + }, + created_at_ms: 100, + last_transition_at_ms: 200, + last_command_id: Uuid::from_u128(2), + written_by: ServerIdentity { + build: "test".into(), + instance_id: Uuid::from_u128(99), + }, + adoptions: vec![Adoption { + previous_operation_id: Uuid::from_u128(5), + new_operation_id: Uuid::from_u128(1), + command_id: Uuid::from_u128(6), + approvers: vec!["a".into(), "b".into()], + incident_ref: "inc".into(), + reason: "lost control plane".into(), + at_ms: 150, + revision: 2, + }], + } + } + + fn sample_receipt() -> CommandReceipt { + CommandReceipt { + namespace: NamespaceName::from("db1"), + operation_id: Uuid::from_u128(1), + command_id: Uuid::from_u128(2), + command: CommandKind::AcquireSourceWriteFence, + fingerprint: Fingerprint([7; 32]), + outcome: FenceOutcome::Applied, + revision_before: 0, + revision_after: 2, + state_after: FenceState::SourceWriteFenced, + applied_at_ms: 200, + instance_id: Uuid::from_u128(99), + adoption: None, + } + } + + #[test] + fn record_round_trips() { + let record = sample_record(); + let bytes = record.encode(); + let decoded = NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, 2, &bytes).unwrap(); + assert_eq!(decoded, record); + } + + #[test] + fn receipt_round_trips() { + let receipt = sample_receipt(); + let decoded = CommandReceipt::decode(FENCE_FORMAT_VERSION, &receipt.encode()).unwrap(); + assert_eq!(decoded, receipt); + } + + #[test] + fn marker_round_trips() { + let marker = FenceMarker::for_record(&sample_record()); + assert_eq!(FenceMarker::decode(&marker.encode()).unwrap(), marker); + } + + #[test] + fn unknown_format_version_is_rejected() { + let bytes = sample_record().encode(); + assert_eq!( + NamespaceFenceRecord::decode(2, 2, &bytes), + Err(FenceDecodeError::UnsupportedFormatVersion(2)) + ); + assert!(matches!( + CommandReceipt::decode(0, &sample_receipt().encode()), + Err(FenceDecodeError::UnsupportedFormatVersion(0)) + )); + let mut marker = proto::FenceMarker::decode( + FenceMarker::for_record(&sample_record()) + .encode() + .as_slice(), + ) + .unwrap(); + marker.format_version = 7; + assert_eq!( + FenceMarker::decode(&marker.encode_to_vec()), + Err(FenceDecodeError::UnsupportedFormatVersion(7)) + ); + } + + #[test] + fn garbage_is_rejected() { + assert!(matches!( + NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, 2, &[0xff, 0xff, 0xff]), + Err(FenceDecodeError::Undecodable(_)) + )); + // An empty payload decodes as an all-default message, which is not a valid record. + assert!(NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, 0, &[]).is_err()); + assert!(CommandReceipt::decode(FENCE_FORMAT_VERSION, &[]).is_err()); + assert!(FenceMarker::decode(&[]).is_err()); + } + + #[test] + fn revision_column_must_match() { + let bytes = sample_record().encode(); + assert!(matches!( + NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, 3, &bytes), + Err(FenceDecodeError::Invalid(_)) + )); + } + + fn mutate( + f: impl FnOnce(&mut proto::FenceRecord), + ) -> Result { + let mut m = codec::record_to_proto(&sample_record()); + f(&mut m); + let revision = m.revision; + NamespaceFenceRecord::decode(FENCE_FORMAT_VERSION, revision, &m.encode_to_vec()) + } + + #[test] + fn invalid_records_are_rejected() { + assert!(mutate(|m| m.state = 999).is_err(), "unknown state"); + assert!( + mutate(|m| m.state = proto::FenceState::Unfenced as i32).is_err(), + "non-durable" + ); + assert!( + mutate(|m| m.state = proto::FenceState::UnknownUnavailable as i32).is_err(), + "derived state stored" + ); + assert!( + mutate(|m| m.state = proto::FenceState::TargetWritable as i32).is_err(), + "state/role mismatch" + ); + assert!(mutate(|m| m.role = 0).is_err(), "unspecified role"); + assert!(mutate(|m| m.operation_id = "not-a-uuid".into()).is_err()); + assert!( + mutate(|m| m.operation_id = m.operation_id.replace('-', "")).is_err(), + "non-canonical id" + ); + assert!(mutate(|m| m.namespace = String::new()).is_err()); + assert!(mutate(|m| m.legacy_blocks = None).is_err()); + assert!(mutate(|m| m.written_by = None).is_err()); + assert!( + mutate(|m| m.frozen_boundary = None).is_err(), + "fenced without boundary" + ); + assert!( + mutate(|m| m.log_id = None).is_err(), + "source without identity" + ); + assert!( + mutate(|m| { + m.revision = 0; + }) + .is_err(), + "revision zero" + ); + assert!( + mutate(|m| m.drain_policy.as_mut().unwrap().on_deadline = 0).is_err(), + "unspecified drain policy" + ); + } + + #[test] + fn invalid_receipts_are_rejected() { + let enc = |f: &dyn Fn(&mut proto::CommandReceipt)| { + let mut m = codec::receipt_to_proto(&sample_receipt()); + f(&mut m); + CommandReceipt::decode(FENCE_FORMAT_VERSION, &m.encode_to_vec()) + }; + assert!(enc(&|m| m.fingerprint = vec![1; 31]).is_err()); + assert!(enc(&|m| m.outcome = 0).is_err()); + assert!(enc(&|m| m.command = 0).is_err()); + assert!(enc(&|m| m.state_after = proto::FenceState::Unfenced as i32).is_err()); + assert!( + enc(&|m| m.revision_before = 5).is_err(), + "revision went backwards" + ); + } + + #[test] + fn legacy_mirror_follows_state() { + let mut record = sample_record(); + let m = record.legacy_mirror(); + assert!(!m.block_reads); + assert!(m.block_writes); + assert!(m.block_reason.unwrap().contains("SOURCE_WRITE_FENCED")); + + record.state = FenceState::SourceReadFenced; + let m = record.legacy_mirror(); + assert!(m.block_reads && m.block_writes); + + record.state = FenceState::Released; + assert_eq!(record.legacy_mirror(), record.legacy_blocks); + + record.role = Role::Target; + record.state = FenceState::TargetQuarantined; + let m = record.legacy_mirror(); + assert!(m.block_reads && m.block_writes); + record.state = FenceState::TargetWriteFenced; + let m = record.legacy_mirror(); + assert!(!m.block_reads && m.block_writes); + } +} diff --git a/libsql-server/src/namespace/fence/state.rs b/libsql-server/src/namespace/fence/state.rs new file mode 100644 index 0000000000..53a681775c --- /dev/null +++ b/libsql-server/src/namespace/fence/state.rs @@ -0,0 +1,403 @@ +//! Roles, states, operation classes and the permission matrix of `docs/NAMESPACE_FENCE.md` +//! sections 3 and 7.3. + +use std::fmt; +use std::str::FromStr; + +use super::outcome::FenceOutcome; + +/// Which side of a move a fence record belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Role { + Source, + Target, +} + +impl Role { + pub const fn as_str(self) -> &'static str { + match self { + Role::Source => "SOURCE", + Role::Target => "TARGET", + } + } +} + +impl fmt::Display for Role { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The state of a namespace as the fence sees it. +/// +/// `Unfenced` and `Absent` describe a namespace with no record (an ordinary namespace, or no +/// namespace at all). `UnknownUnavailable` is derived when the server cannot establish the +/// control state. None of those three is ever stored in a record. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FenceState { + Unfenced, + Absent, + SourceDraining, + SourceWriteFenced, + SourceReadDraining, + SourceReadFenced, + Released, + TargetQuarantined, + TargetImportDraining, + TargetValidating, + TargetWriteFenced, + TargetWritable, + TargetAborted, + UnknownUnavailable, +} + +impl FenceState { + pub const ALL: [FenceState; 14] = [ + FenceState::Unfenced, + FenceState::Absent, + FenceState::SourceDraining, + FenceState::SourceWriteFenced, + FenceState::SourceReadDraining, + FenceState::SourceReadFenced, + FenceState::Released, + FenceState::TargetQuarantined, + FenceState::TargetImportDraining, + FenceState::TargetValidating, + FenceState::TargetWriteFenced, + FenceState::TargetWritable, + FenceState::TargetAborted, + FenceState::UnknownUnavailable, + ]; + + pub const fn as_str(self) -> &'static str { + match self { + FenceState::Unfenced => "UNFENCED", + FenceState::Absent => "ABSENT", + FenceState::SourceDraining => "SOURCE_DRAINING", + FenceState::SourceWriteFenced => "SOURCE_WRITE_FENCED", + FenceState::SourceReadDraining => "SOURCE_READ_DRAINING", + FenceState::SourceReadFenced => "SOURCE_READ_FENCED", + FenceState::Released => "RELEASED", + FenceState::TargetQuarantined => "TARGET_QUARANTINED", + FenceState::TargetImportDraining => "TARGET_IMPORT_DRAINING", + FenceState::TargetValidating => "TARGET_VALIDATING", + FenceState::TargetWriteFenced => "TARGET_WRITE_FENCED", + FenceState::TargetWritable => "TARGET_WRITABLE", + FenceState::TargetAborted => "TARGET_ABORTED", + FenceState::UnknownUnavailable => "UNKNOWN_UNAVAILABLE", + } + } + + /// The role a record in this state has, if the state belongs to one. + pub const fn role(self) -> Option { + match self { + FenceState::SourceDraining + | FenceState::SourceWriteFenced + | FenceState::SourceReadDraining + | FenceState::SourceReadFenced + | FenceState::Released => Some(Role::Source), + FenceState::TargetQuarantined + | FenceState::TargetImportDraining + | FenceState::TargetValidating + | FenceState::TargetWriteFenced + | FenceState::TargetWritable + | FenceState::TargetAborted => Some(Role::Target), + FenceState::Unfenced | FenceState::Absent | FenceState::UnknownUnavailable => None, + } + } + + /// Whether this state can be stored in a fence record. + pub const fn is_durable(self) -> bool { + self.role().is_some() + } + + /// Whether the operation that owns a record in this state has finished with it. + pub const fn is_operation_finished(self) -> bool { + matches!( + self, + FenceState::Released | FenceState::TargetWritable | FenceState::TargetAborted + ) + } + + /// Whether a record in this state counts as an active fence (section 4.4, + /// `active_fences`): anything except an ordinary namespace, a released source or a + /// published target. + pub const fn is_active(self) -> bool { + !matches!( + self, + FenceState::Unfenced + | FenceState::Absent + | FenceState::Released + | FenceState::TargetWritable + ) + } + + /// A state in which the server is waiting for work admitted earlier to end. + pub const fn is_draining(self) -> bool { + matches!( + self, + FenceState::SourceDraining + | FenceState::SourceReadDraining + | FenceState::TargetImportDraining + ) + } + + /// The permission-matrix decision for work of `class` (section 3.3). + /// + /// For the capability classes this decides only whether the state admits capability work + /// at all; whether a particular capability is valid is the controller's decision. + pub fn permits(self, class: OperationClass) -> Result<(), FenceOutcome> { + use FenceState::*; + use OperationClass::*; + + match class { + Maintenance | Observability => return Ok(()), + _ => (), + } + + if self == UnknownUnavailable { + return Err(FenceOutcome::FenceStateUnavailable); + } + + match class { + NormalRead | Stream => match self { + Unfenced | Absent | Released | SourceDraining | SourceWriteFenced + | TargetWriteFenced | TargetWritable => Ok(()), + SourceReadDraining | SourceReadFenced => Err(FenceOutcome::MigrationReadFenced), + TargetQuarantined | TargetImportDraining | TargetValidating | TargetAborted => { + Err(FenceOutcome::MigrationTargetQuarantined) + } + UnknownUnavailable => unreachable!(), + }, + NormalWrite | Vacuum | Lifecycle => match self { + Unfenced | Absent | Released | TargetWritable => Ok(()), + SourceDraining | SourceWriteFenced | SourceReadDraining | SourceReadFenced + | TargetWriteFenced => Err(FenceOutcome::MigrationWriteFenced), + TargetQuarantined | TargetImportDraining | TargetValidating | TargetAborted => { + Err(FenceOutcome::MigrationTargetQuarantined) + } + UnknownUnavailable => unreachable!(), + }, + CapabilityImport => match self { + TargetQuarantined => Ok(()), + // Existing import writers may finish while draining, but no new one starts: + // that distinction is the controller's, which tracks issued capabilities. + TargetImportDraining => Ok(()), + _ => Err(FenceOutcome::OperationCapabilityRequired), + }, + CapabilityValidate => match self { + TargetValidating | TargetWriteFenced => Ok(()), + _ => Err(FenceOutcome::OperationCapabilityRequired), + }, + Maintenance | Observability => unreachable!(), + } + } + + /// Whether normal write admission is open in this state. + pub fn write_admission(self) -> Admission { + Admission::from(self.permits(OperationClass::NormalWrite)) + } + + /// Whether normal read admission is open in this state. + pub fn read_admission(self) -> Admission { + Admission::from(self.permits(OperationClass::NormalRead)) + } +} + +impl fmt::Display for FenceState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("unknown fence state `{0}`")] +pub struct UnknownFenceState(pub String); + +impl FromStr for FenceState { + type Err = UnknownFenceState; + + fn from_str(s: &str) -> Result { + FenceState::ALL + .iter() + .copied() + .find(|state| state.as_str() == s) + .ok_or_else(|| UnknownFenceState(s.to_string())) + } +} + +/// Whether an admission path is open, and if it is closed, the code a denial carries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Admission { + Open, + Closed(FenceOutcome), +} + +impl Admission { + pub fn is_open(self) -> bool { + matches!(self, Admission::Open) + } + + pub fn as_str(self) -> &'static str { + match self { + Admission::Open => "open", + Admission::Closed(_) => "closed", + } + } +} + +impl From> for Admission { + fn from(value: Result<(), FenceOutcome>) -> Self { + match value { + Ok(()) => Admission::Open, + Err(code) => Admission::Closed(code), + } + } +} + +/// The class of a piece of work, which the permission matrix is keyed by (section 7.3). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OperationClass { + /// Any logical write that is not operation-owned: SQL over every protocol, the admin + /// shell, schema migration, dump load outside an import capability. + NormalWrite, + /// Work that cannot change logical contents: `TRUNCATE` checkpoints, the storage monitor, + /// bottomless WAL upload, the replication logger's own connection. + Maintenance, + /// `VACUUM`. It takes a write transaction and produces replicated frames, so it is not + /// maintenance and is skipped wherever normal writes are denied. + Vacuum, + /// Generic lifecycle and configuration: config mutation, delete, reset, fork, create over + /// an existing record, restore, dump load, shared-schema linking. + Lifecycle, + /// Writes of an import session holding a `MigrationCapability`. + CapabilityImport, + /// Read-only validation through a `MigrationCapability`. + CapabilityValidate, + /// SQL programs, Hrana cursors, `/beta/listen`, ATTACH of the namespace. + NormalRead, + /// `/dump` and replication streams (`hello`, `log_entries`, `batch_log_entries`, + /// `snapshot`). + Stream, + /// Stats, jobs and metrics. Never a read lease. + Observability, +} + +impl OperationClass { + pub const ALL: [OperationClass; 9] = [ + OperationClass::NormalWrite, + OperationClass::Maintenance, + OperationClass::Vacuum, + OperationClass::Lifecycle, + OperationClass::CapabilityImport, + OperationClass::CapabilityValidate, + OperationClass::NormalRead, + OperationClass::Stream, + OperationClass::Observability, + ]; +} + +#[cfg(test)] +mod tests { + use super::*; + + use FenceOutcome as O; + use FenceState as S; + use OperationClass as C; + + #[test] + fn state_names_round_trip() { + for state in FenceState::ALL { + assert_eq!(state.as_str().parse::().unwrap(), state); + } + assert!("source_draining".parse::().is_err()); + assert!("".parse::().is_err()); + } + + #[test] + fn durable_states_have_a_role() { + for state in FenceState::ALL { + let expected = !matches!(state, S::Unfenced | S::Absent | S::UnknownUnavailable); + assert_eq!(state.is_durable(), expected, "{state}"); + } + } + + /// The whole permission matrix of section 3.3, row by row. + #[test] + fn permission_matrix() { + let allow = Ok(()); + let wf = Err(O::MigrationWriteFenced); + let rf = Err(O::MigrationReadFenced); + let tq = Err(O::MigrationTargetQuarantined); + let un = Err(O::FenceStateUnavailable); + let cap = Err(O::OperationCapabilityRequired); + + // state: (read, stream, write, lifecycle, import, validate) + let rows = [ + (S::Unfenced, [allow, allow, allow, allow, cap, cap]), + (S::Released, [allow, allow, allow, allow, cap, cap]), + (S::SourceDraining, [allow, allow, wf, wf, cap, cap]), + (S::SourceWriteFenced, [allow, allow, wf, wf, cap, cap]), + (S::SourceReadDraining, [rf, rf, wf, wf, cap, cap]), + (S::SourceReadFenced, [rf, rf, wf, wf, cap, cap]), + (S::TargetQuarantined, [tq, tq, tq, tq, allow, cap]), + (S::TargetImportDraining, [tq, tq, tq, tq, allow, cap]), + (S::TargetValidating, [tq, tq, tq, tq, cap, allow]), + (S::TargetWriteFenced, [allow, allow, wf, wf, cap, allow]), + (S::TargetWritable, [allow, allow, allow, allow, cap, cap]), + (S::TargetAborted, [tq, tq, tq, tq, cap, cap]), + (S::UnknownUnavailable, [un, un, un, un, un, un]), + ]; + + for (state, expected) in rows { + let classes = [ + C::NormalRead, + C::Stream, + C::NormalWrite, + C::Lifecycle, + C::CapabilityImport, + C::CapabilityValidate, + ]; + for (class, expected) in classes.into_iter().zip(expected) { + assert_eq!(state.permits(class), expected, "{state} {class:?}"); + } + // Vacuum follows the normal write column. + assert_eq!( + state.permits(C::Vacuum), + state.permits(C::NormalWrite), + "{state}" + ); + // Maintenance and observability continue in every state. + assert_eq!(state.permits(C::Maintenance), Ok(()), "{state}"); + assert_eq!(state.permits(C::Observability), Ok(()), "{state}"); + } + } + + #[test] + fn denials_are_data_plane_codes() { + for state in FenceState::ALL { + for class in OperationClass::ALL { + if let Err(code) = state.permits(class) { + assert!(code.is_error(), "{state} {class:?} {code}"); + } + } + } + } + + #[test] + fn active_and_finished() { + assert!(!S::Unfenced.is_active()); + assert!(!S::Released.is_active()); + assert!(!S::TargetWritable.is_active()); + assert!(S::TargetAborted.is_active()); + assert!(S::UnknownUnavailable.is_active()); + assert!(S::SourceDraining.is_active()); + + for state in FenceState::ALL { + assert_eq!( + state.is_operation_finished(), + matches!(state, S::Released | S::TargetWritable | S::TargetAborted) + ); + } + } +} diff --git a/libsql-server/src/namespace/fence/transition.rs b/libsql-server/src/namespace/fence/transition.rs new file mode 100644 index 0000000000..f1de43f747 --- /dev/null +++ b/libsql-server/src/namespace/fence/transition.rs @@ -0,0 +1,1794 @@ +//! The pure fence transition function (`docs/NAMESPACE_FENCE.md` sections 3.2 and 5.3). +//! +//! [`apply`] decides what a command does to a namespace's fence, given everything the store +//! read inside its transaction. It performs no I/O, takes no locks and reads no clock: the +//! store supplies the current record, the stored receipt for the request's +//! `(operation_id, command_id)` if there is one, and the facts in [`ApplyEnv`]. The store +//! persists whatever [`Decision::Apply`] returns, in one transaction, before anything is +//! published or answered. +//! +//! Checks run in this order, and the order is part of the contract: +//! +//! 1. **Replay.** A stored receipt with the same fingerprint is answered from the receipt +//! (`Replay`, or `Resume` for a drain still in progress), whatever has happened to the +//! record since. A stored receipt with a different fingerprint is `FENCE_COMMAND_CONFLICT`. +//! 2. **Unavailable state.** A record the server cannot establish refuses everything with +//! `FENCE_STATE_UNAVAILABLE`, except the two commands that can reconcile it: a replay of the +//! `CreateTargetQuarantined` that left the marker, and an adoption after a metastore +//! rollback. +//! 3. **Owner.** An unfinished record owned by another operation is +//! `FENCE_OWNED_BY_ANOTHER_OPERATION`. +//! 4. **Already applied.** A command from the owner whose goal state the record is already in +//! is `ALREADY_APPLIED`: a receipt is stored, the record and its revision do not change. +//! This is checked before the revision, because the caller's stated expectation is +//! typically the state before a response it never received. +//! 5. **Transition.** Role, then legality of the transition from the current state +//! (`INVALID_FENCE_TRANSITION`). +//! 6. **Expectation.** `expected_state` and `expected_revision` (`FENCE_REVISION_MISMATCH`). +//! 7. **Preconditions** of the command (`FENCE_PRECONDITION_FAILED`). +//! +//! A drain that starts in `apply` (`DRAINING`) is finished by [`complete_drain`], once the +//! controller has proven that the work admitted earlier has ended. + +use uuid::Uuid; + +use super::command::{ + AdoptArgs, CommandKind, FenceCommand, FenceRequest, ValidationResult, + MAX_VALIDATION_SUMMARY_BYTES, +}; +use super::outcome::{FenceDetail, FenceError, FenceOutcome}; +use super::record::{ + Adoption, CommandReceipt, FrozenBoundary, LegacyBlocks, NamespaceFenceRecord, + NamespaceIdentity, ServerIdentity, ValidationRecord, ValidationSnapshot, +}; +use super::state::{FenceState, Role}; + +/// What the store knows about a namespace's fence. +#[derive(Debug, Clone, Copy)] +pub enum CurrentFence<'a> { + /// No record. `namespace_exists` distinguishes `UNFENCED` from `ABSENT`. + None { + namespace_exists: bool, + }, + Record(&'a NamespaceFenceRecord), + /// The control state cannot be established. `marker` is the record the namespace's marker + /// file holds, when it has a readable one. + Unavailable { + detail: FenceDetail, + marker: Option<&'a NamespaceFenceRecord>, + }, +} + +impl CurrentFence<'_> { + pub fn state(&self) -> FenceState { + match self { + CurrentFence::None { + namespace_exists: true, + } => FenceState::Unfenced, + CurrentFence::None { + namespace_exists: false, + } => FenceState::Absent, + CurrentFence::Record(r) => r.state, + CurrentFence::Unavailable { .. } => FenceState::UnknownUnavailable, + } + } + + pub fn revision(&self) -> u64 { + match self { + CurrentFence::None { .. } => 0, + CurrentFence::Record(r) => r.revision, + CurrentFence::Unavailable { marker, .. } => marker.map_or(0, |m| m.revision), + } + } +} + +/// Facts `apply` needs that are not in the record. The store fills them from inside the same +/// transaction and the live namespace. +#[derive(Debug, Clone)] +pub struct ApplyEnv { + /// Wall-clock time, in milliseconds since the Unix epoch. Informational only: nothing in + /// the fence expires. + pub now_ms: i64, + pub server: ServerIdentity, + /// The namespace's current replication log id, if it exists and has one. + pub namespace_log_id: Option, + /// Whether the namespace is a shared schema or linked to one. + pub shared_schema: bool, + /// The namespace config's current `block_*` values, saved when a source is acquired and + /// restored when it is released. + pub legacy_blocks: LegacyBlocks, + /// A fresh id, used as `target_incarnation_id` by `CreateTargetQuarantined`. + pub new_incarnation_id: Uuid, + /// Whether the request carried the configured adoption key. + pub adoption_authorised: bool, + /// What the server observed of the target, for `RecordTargetValidation`. + pub validation_snapshot: Option, +} + +/// What a command does. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Decision { + /// The command was applied before and its answer is final: return this receipt, with + /// `replayed: true`. Nothing is written. + Replay(CommandReceipt), + /// The same command started a drain that has not been completed: resume it. Nothing is + /// written. + Resume(CommandReceipt), + /// Persist `record` (when `Some`; `None` leaves the record as it is) and `receipt` in one + /// transaction, then answer `receipt.outcome`. A `DRAINING` receipt means the controller + /// must now run the drain and finish it with [`complete_drain`]. + Apply { + record: Option, + receipt: CommandReceipt, + }, +} + +/// Evidence, gathered by the controller, that a drain is complete. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DrainCompletion { + /// No normal writer holds or can take the write slot; `boundary` was read with the slot + /// free, after the last commit published its frame. + SourceWrites { boundary: FrozenBoundary }, + /// Every SQL, dump and replication read lease has been released. + SourceReads, + /// Every import writer has finished and no capability writer holds the slot. + TargetImport, +} + +fn err(outcome: FenceOutcome, message: impl Into) -> FenceError { + FenceError::new(outcome, message) +} + +fn invalid(message: impl Into) -> FenceError { + err(FenceOutcome::InvalidFenceTransition, message) +} + +fn precondition(detail: FenceDetail, message: impl Into) -> FenceError { + err(FenceOutcome::FencePreconditionFailed, message).with_detail(detail) +} + +/// The role a command acts on. `None` for adoption, which acts on either. +fn command_role(kind: CommandKind) -> Option { + match kind { + CommandKind::AcquireSourceWriteFence + | CommandKind::SetSourceReadFence + | CommandKind::ClearSourceReadFence + | CommandKind::ReleaseSourceWriteFence => Some(Role::Source), + CommandKind::CreateTargetQuarantined + | CommandKind::SealTargetImport + | CommandKind::RecordTargetValidation + | CommandKind::PublishTargetReadableWriteFenced + | CommandKind::EnableTargetWrites + | CommandKind::AbortQuarantinedTarget => Some(Role::Target), + CommandKind::AdoptFence => None, + } +} + +/// The state a legal command moves a record from `from` to, and the receipt outcome. This is +/// the transition graph of section 3.2, minus the drain completions and adoption. +fn transition(kind: CommandKind, from: FenceState) -> Option<(FenceState, FenceOutcome)> { + use CommandKind as K; + use FenceOutcome::{Applied, Draining}; + use FenceState as S; + + Some(match (kind, from) { + (K::AcquireSourceWriteFence, S::Unfenced | S::Released | S::TargetWritable) => { + (S::SourceDraining, Draining) + } + (K::SetSourceReadFence, S::SourceWriteFenced) => (S::SourceReadDraining, Draining), + (K::ClearSourceReadFence, S::SourceReadDraining | S::SourceReadFenced) => { + (S::SourceWriteFenced, Applied) + } + (K::ReleaseSourceWriteFence, S::SourceDraining | S::SourceWriteFenced) => { + (S::Released, Applied) + } + (K::CreateTargetQuarantined, S::Absent) => (S::TargetQuarantined, Applied), + (K::SealTargetImport, S::TargetQuarantined) => (S::TargetImportDraining, Draining), + (K::RecordTargetValidation, S::TargetValidating) => (S::TargetValidating, Applied), + (K::PublishTargetReadableWriteFenced, S::TargetValidating) => { + (S::TargetWriteFenced, Applied) + } + (K::EnableTargetWrites, S::TargetWriteFenced) => (S::TargetWritable, Applied), + ( + K::AbortQuarantinedTarget, + S::TargetQuarantined + | S::TargetImportDraining + | S::TargetValidating + | S::TargetWriteFenced, + ) => (S::TargetAborted, Applied), + _ => return None, + }) +} + +/// The draining state a command's drain runs in, for commands that drain. +fn drain_state(kind: CommandKind) -> Option { + match kind { + CommandKind::AcquireSourceWriteFence => Some(FenceState::SourceDraining), + CommandKind::SetSourceReadFence => Some(FenceState::SourceReadDraining), + CommandKind::SealTargetImport => Some(FenceState::TargetImportDraining), + _ => None, + } +} + +fn receipt( + request: &FenceRequest, + env: &ApplyEnv, + outcome: FenceOutcome, + revision_before: u64, + revision_after: u64, + state_after: FenceState, +) -> CommandReceipt { + CommandReceipt { + namespace: request.namespace.clone(), + operation_id: request.operation_id, + command_id: request.command_id, + command: request.command.kind(), + fingerprint: request.fingerprint(), + outcome, + revision_before, + revision_after, + state_after, + applied_at_ms: env.now_ms, + instance_id: env.server.instance_id, + adoption: None, + } +} + +/// Decide what `request` does to the namespace's fence. See the module documentation for the +/// order of the checks. +/// +/// `existing` is the stored receipt for `(request.namespace, request.operation_id, +/// request.command_id)`, if any. +pub fn apply( + current: CurrentFence<'_>, + existing: Option<&CommandReceipt>, + request: &FenceRequest, + env: &ApplyEnv, +) -> Result { + let kind = request.command.kind(); + + // 1. Replay, before anything about the record is looked at. + if let Some(existing) = existing { + debug_assert_eq!(existing.operation_id, request.operation_id); + debug_assert_eq!(existing.command_id, request.command_id); + if existing.fingerprint != request.fingerprint() { + return Err(err( + FenceOutcome::FenceCommandConflict, + format!( + "command {} was already used for a different request", + request.command_id + ), + )); + } + return Ok(if existing.is_final() { + Decision::Replay(existing.clone()) + } else { + Decision::Resume(existing.clone()) + }); + } + + // 2. A state the server cannot establish. + let record = match current { + CurrentFence::None { .. } => None, + CurrentFence::Record(record) => Some(record), + CurrentFence::Unavailable { detail, marker } => { + return apply_unavailable(detail, marker, request, env); + } + }; + + if let FenceCommand::AdoptFence(args) = &request.command { + return apply_adopt(current, record, args, request, env); + } + + if let Some(record) = record { + // 3. Owner. + let finished = record.state.is_operation_finished(); + if !finished && record.operation_id != request.operation_id { + return Err(err( + FenceOutcome::FenceOwnedByAnotherOperation, + format!( + "namespace fence is owned by operation {}", + record.operation_id + ), + )); + } + + let own = record.operation_id == request.operation_id; + + // 4. Already applied. + if own && kind.goal_state() == Some(record.state) { + let receipt = receipt( + request, + env, + FenceOutcome::AlreadyApplied, + record.revision, + record.revision, + record.state, + ); + return Ok(Decision::Apply { + record: None, + receipt, + }); + } + + // The owner joining its own drain under a new command id (for example after + // adoption, or after losing the original command id): nothing changes but the receipt, + // and the controller resumes the drain. + if own && drain_state(kind) == Some(record.state) { + check_expectation(current, request)?; + let receipt = receipt( + request, + env, + FenceOutcome::Draining, + record.revision, + record.revision, + record.state, + ); + return Ok(Decision::Apply { + record: None, + receipt, + }); + } + + if own && finished { + return Err(invalid(format!( + "operation {} has finished with this namespace ({})", + record.operation_id, record.state + )) + .with_detail(FenceDetail::OperationFinished)); + } + } + + // `CreateTargetQuarantined` needs a name nobody uses, whatever the record says. + if kind == CommandKind::CreateTargetQuarantined && current.state() != FenceState::Absent { + return Err(precondition( + FenceDetail::NamespaceExists, + "the namespace already exists", + )); + } + + // 5. Role and transition. + let from = current.state(); + let role = command_role(kind).expect("adoption is handled above"); + let current_role = match from { + // A published target, or a released source, may be acquired as a source by a new + // operation. + FenceState::Released | FenceState::TargetWritable | FenceState::Unfenced => { + Some(Role::Source) + } + FenceState::Absent => Some(Role::Target), + other => other.role(), + }; + if current_role != Some(role) { + return Err( + invalid(format!("{kind} does not apply to a namespace in {from}")) + .with_detail(FenceDetail::RoleMismatch), + ); + } + let Some((to, outcome)) = transition(kind, from) else { + return Err(invalid(format!("{kind} is not a transition from {from}"))); + }; + + // 6. Expectation. + check_expectation(current, request)?; + + // 7. Preconditions, and the next record. + let revision_before = current.revision(); + let revision_after = revision_before + 1; + let mut next = match record { + Some(record) if !record.state.is_operation_finished() => record.clone(), + _ => fresh_record(request, env, role), + }; + + match &request.command { + FenceCommand::AcquireSourceWriteFence { + expected_log_id, + drain_policy, + } => { + if env.shared_schema { + return Err(precondition( + FenceDetail::SharedSchemaUnsupported, + "shared-schema namespaces cannot be fenced", + )); + } + if env.namespace_log_id != Some(*expected_log_id) { + return Err(precondition( + FenceDetail::NamespaceIdentityMismatch, + "the namespace's replication log id is not the one the caller observed", + )); + } + next.identity = NamespaceIdentity { + log_id: Some(*expected_log_id), + target_incarnation_id: None, + }; + next.legacy_blocks = env.legacy_blocks.clone(); + next.drain_policy = *drain_policy; + next.drain_started_at_ms = Some(env.now_ms); + } + FenceCommand::SetSourceReadFence { drain_policy } + | FenceCommand::SealTargetImport { drain_policy } => { + next.drain_policy = *drain_policy; + next.drain_started_at_ms = Some(env.now_ms); + } + FenceCommand::ClearSourceReadFence + | FenceCommand::ReleaseSourceWriteFence + | FenceCommand::EnableTargetWrites + | FenceCommand::AbortQuarantinedTarget => { + next.drain_policy = None; + next.drain_started_at_ms = None; + } + FenceCommand::CreateTargetQuarantined { .. } => { + next.identity = NamespaceIdentity { + log_id: None, + target_incarnation_id: Some(env.new_incarnation_id), + }; + next.legacy_blocks = LegacyBlocks::default(); + } + FenceCommand::RecordTargetValidation { result, summary } => { + if summary.len() > MAX_VALIDATION_SUMMARY_BYTES { + return Err(precondition( + FenceDetail::InvalidArgument, + format!( + "validation summary is longer than {MAX_VALIDATION_SUMMARY_BYTES} bytes" + ), + )); + } + next.validation = Some(ValidationRecord { + operation_id: request.operation_id, + command_id: request.command_id, + result: *result, + summary: summary.clone(), + snapshot: env.validation_snapshot, + recorded_at_ms: env.now_ms, + }); + } + FenceCommand::PublishTargetReadableWriteFenced => { + let validated = next.validation.as_ref().is_some_and(|v| { + v.result == ValidationResult::Ok && v.operation_id == next.operation_id + }); + if !validated { + return Err(precondition( + FenceDetail::ValidationReceiptRequired, + "publication requires a successful validation receipt from the owning operation", + )); + } + } + FenceCommand::AdoptFence(_) => unreachable!("handled above"), + } + + next.state = to; + next.revision = revision_after; + next.last_transition_at_ms = env.now_ms; + next.last_command_id = request.command_id; + next.written_by = env.server.clone(); + + let receipt = receipt(request, env, outcome, revision_before, revision_after, to); + Ok(Decision::Apply { + record: Some(next), + receipt, + }) +} + +/// A record for an operation that is starting on this namespace. +fn fresh_record(request: &FenceRequest, env: &ApplyEnv, role: Role) -> NamespaceFenceRecord { + NamespaceFenceRecord { + namespace: request.namespace.clone(), + role, + state: FenceState::Unfenced, + revision: 0, + operation_id: request.operation_id, + identity: NamespaceIdentity::default(), + drain_policy: None, + drain_started_at_ms: None, + frozen_boundary: None, + validation: None, + legacy_blocks: LegacyBlocks::default(), + created_at_ms: env.now_ms, + last_transition_at_ms: env.now_ms, + last_command_id: request.command_id, + written_by: env.server.clone(), + adoptions: Vec::new(), + } +} + +fn check_expectation(current: CurrentFence<'_>, request: &FenceRequest) -> Result<(), FenceError> { + let (state, revision) = (current.state(), current.revision()); + if request.expected_state != state || request.expected_revision != revision { + return Err(err( + FenceOutcome::FenceRevisionMismatch, + format!( + "expected {} at revision {}, found {} at revision {}", + request.expected_state, request.expected_revision, state, revision + ), + )); + } + Ok(()) +} + +fn apply_unavailable( + detail: FenceDetail, + marker: Option<&NamespaceFenceRecord>, + request: &FenceRequest, + env: &ApplyEnv, +) -> Result { + let unavailable = || { + err( + FenceOutcome::FenceStateUnavailable, + "the namespace's fence state cannot be established", + ) + .with_detail(detail) + }; + + match (&request.command, detail, marker) { + // A crash between writing the marker and committing the target's rows: only the same + // command completes the creation. + ( + FenceCommand::CreateTargetQuarantined { .. }, + FenceDetail::IncompleteTargetCreation, + Some(m), + ) if m.state == FenceState::TargetQuarantined + && m.revision == 1 + && m.operation_id == request.operation_id + && m.last_command_id == request.command_id => + { + let decision = apply( + CurrentFence::None { + namespace_exists: false, + }, + None, + request, + &ApplyEnv { + new_incarnation_id: m + .identity + .target_incarnation_id + .unwrap_or(env.new_incarnation_id), + ..env.clone() + }, + )?; + Ok(decision) + } + // The metastore went backwards: adoption re-establishes the record the marker last + // recorded (it is written only after a commit), under the adopting operation. + (FenceCommand::AdoptFence(args), FenceDetail::MetastoreBehindMarker, Some(m)) => { + apply_adopt(CurrentFence::Record(m), Some(m), args, request, env) + } + _ => Err(unavailable()), + } +} + +fn apply_adopt( + current: CurrentFence<'_>, + record: Option<&NamespaceFenceRecord>, + args: &AdoptArgs, + request: &FenceRequest, + env: &ApplyEnv, +) -> Result { + let Some(record) = record else { + return Err(invalid("there is no fence to adopt")); + }; + if record.state.is_operation_finished() { + return Err( + invalid(format!("a fence in {} cannot be adopted", record.state)) + .with_detail(FenceDetail::OperationFinished), + ); + } + if args.current_operation_id != record.operation_id { + return Err(err( + FenceOutcome::FenceOwnedByAnotherOperation, + format!( + "namespace fence is owned by operation {}", + record.operation_id + ), + )); + } + if request.operation_id == record.operation_id { + return Err(invalid("an operation cannot adopt its own fence")); + } + check_expectation(current, request)?; + if !env.adoption_authorised { + return Err(precondition( + FenceDetail::AdoptionNotAuthorised, + "adoption requires the configured adoption key", + )); + } + let approvers: Vec<&str> = args.approvers.iter().map(|a| a.trim()).collect(); + let two_distinct = approvers.len() == 2 + && approvers.iter().all(|a| !a.is_empty()) + && approvers[0] != approvers[1]; + if !two_distinct || args.incident_ref.trim().is_empty() || args.reason.trim().is_empty() { + return Err(precondition( + FenceDetail::AdoptionNotAuthorised, + "adoption requires two distinct approvers, an incident reference and a reason", + )); + } + + let revision_after = record.revision + 1; + let adoption = Adoption { + previous_operation_id: record.operation_id, + new_operation_id: request.operation_id, + command_id: request.command_id, + approvers: args.approvers.clone(), + incident_ref: args.incident_ref.clone(), + reason: args.reason.clone(), + at_ms: env.now_ms, + revision: revision_after, + }; + + // Ownership changes and nothing else: the state, and so every gate, stays as it was. + let mut next = record.clone(); + next.operation_id = request.operation_id; + next.revision = revision_after; + next.last_transition_at_ms = env.now_ms; + next.last_command_id = request.command_id; + next.written_by = env.server.clone(); + next.adoptions.push(adoption.clone()); + + let mut receipt = receipt( + request, + env, + FenceOutcome::Applied, + record.revision, + revision_after, + record.state, + ); + receipt.adoption = Some(adoption); + Ok(Decision::Apply { + record: Some(next), + receipt, + }) +} + +/// Finish a drain that `apply` started. `receipt` is the owning operation's `DRAINING` receipt +/// for it; the returned receipt replaces it (same key) with the final `APPLIED` answer. +pub fn complete_drain( + record: &NamespaceFenceRecord, + receipt: &CommandReceipt, + completion: DrainCompletion, + env: &ApplyEnv, +) -> Result<(NamespaceFenceRecord, CommandReceipt), FenceError> { + if receipt.outcome != FenceOutcome::Draining || receipt.operation_id != record.operation_id { + return Err(invalid( + "there is no drain of the owning operation to complete", + )); + } + + let (from, to) = match completion { + DrainCompletion::SourceWrites { .. } => { + (FenceState::SourceDraining, FenceState::SourceWriteFenced) + } + DrainCompletion::SourceReads => { + (FenceState::SourceReadDraining, FenceState::SourceReadFenced) + } + DrainCompletion::TargetImport => ( + FenceState::TargetImportDraining, + FenceState::TargetValidating, + ), + }; + if record.state != from || drain_state(receipt.command) != Some(from) { + return Err(invalid(format!( + "{} cannot complete a drain from {}", + receipt.command, record.state + ))); + } + + let mut next = record.clone(); + if let DrainCompletion::SourceWrites { boundary } = completion { + if record.identity.log_id != Some(boundary.log_id) { + return Err(precondition( + FenceDetail::NamespaceIdentityMismatch, + "the frozen boundary belongs to a different replication log", + )); + } + next.frozen_boundary = Some(boundary); + } + next.state = to; + next.revision = record.revision + 1; + next.drain_policy = None; + next.drain_started_at_ms = None; + next.last_transition_at_ms = env.now_ms; + next.last_command_id = receipt.command_id; + next.written_by = env.server.clone(); + + let mut final_receipt = receipt.clone(); + final_receipt.outcome = FenceOutcome::Applied; + final_receipt.revision_after = next.revision; + final_receipt.state_after = to; + final_receipt.applied_at_ms = env.now_ms; + final_receipt.instance_id = env.server.instance_id; + + Ok((next, final_receipt)) +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::namespace::fence::command::{DrainPolicy, OnDeadline, TargetConfig}; + use crate::namespace::fence::record::{FenceMarker, FENCE_FORMAT_VERSION}; + use crate::namespace::NamespaceName; + + use FenceOutcome as O; + use FenceState as S; + + const LOG: Uuid = Uuid::from_u128(0x10); + const INCARNATION: Uuid = Uuid::from_u128(0x20); + const OP: Uuid = Uuid::from_u128(0xa); + const OTHER_OP: Uuid = Uuid::from_u128(0xb); + + fn env() -> ApplyEnv { + ApplyEnv { + now_ms: 1_000, + server: ServerIdentity { + build: "test".into(), + instance_id: Uuid::from_u128(0x99), + }, + namespace_log_id: Some(LOG), + shared_schema: false, + legacy_blocks: LegacyBlocks::default(), + new_incarnation_id: INCARNATION, + adoption_authorised: false, + validation_snapshot: None, + } + } + + fn policy() -> Option { + Some(DrainPolicy { + deadline_ms: 1_000, + on_deadline: OnDeadline::Fail, + }) + } + + fn command(kind: CommandKind) -> FenceCommand { + match kind { + CommandKind::AcquireSourceWriteFence => FenceCommand::AcquireSourceWriteFence { + expected_log_id: LOG, + drain_policy: policy(), + }, + CommandKind::SetSourceReadFence => FenceCommand::SetSourceReadFence { + drain_policy: policy(), + }, + CommandKind::ClearSourceReadFence => FenceCommand::ClearSourceReadFence, + CommandKind::ReleaseSourceWriteFence => FenceCommand::ReleaseSourceWriteFence, + CommandKind::CreateTargetQuarantined => FenceCommand::CreateTargetQuarantined { + config: TargetConfig::default(), + }, + CommandKind::SealTargetImport => FenceCommand::SealTargetImport { + drain_policy: policy(), + }, + CommandKind::RecordTargetValidation => FenceCommand::RecordTargetValidation { + result: ValidationResult::Ok, + summary: "row counts match".into(), + }, + CommandKind::PublishTargetReadableWriteFenced => { + FenceCommand::PublishTargetReadableWriteFenced + } + CommandKind::EnableTargetWrites => FenceCommand::EnableTargetWrites, + CommandKind::AbortQuarantinedTarget => FenceCommand::AbortQuarantinedTarget, + CommandKind::AdoptFence => FenceCommand::AdoptFence(AdoptArgs { + current_operation_id: OP, + approvers: vec!["alice".into(), "bob".into()], + incident_ref: "INC-1".into(), + reason: "control plane lost".into(), + }), + } + } + + /// A little driver that plays the store: it keeps the record and receipts and applies + /// decisions the way the store will. + #[derive(Default)] + struct Harness { + namespace_exists: bool, + record: Option, + receipts: Vec, + next_command: u128, + } + + impl Harness { + fn source() -> Self { + Self { + namespace_exists: true, + ..Default::default() + } + } + + fn target() -> Self { + Self::default() + } + + fn current(&self) -> CurrentFence<'_> { + match &self.record { + Some(r) => CurrentFence::Record(r), + None => CurrentFence::None { + namespace_exists: self.namespace_exists, + }, + } + } + + fn request(&mut self, op: Uuid, command: FenceCommand) -> FenceRequest { + self.next_command += 1; + FenceRequest { + namespace: NamespaceName::from("db1"), + operation_id: op, + command_id: Uuid::from_u128(0x1000 + self.next_command), + expected_state: self.current().state(), + expected_revision: self.current().revision(), + command, + } + } + + fn lookup(&self, request: &FenceRequest) -> Option<&CommandReceipt> { + self.receipts.iter().find(|r| { + r.operation_id == request.operation_id && r.command_id == request.command_id + }) + } + + fn decide(&self, request: &FenceRequest, env: &ApplyEnv) -> Result { + apply(self.current(), self.lookup(request), request, env) + } + + fn persist(&mut self, decision: &Decision) { + if let Decision::Apply { record, receipt } = decision { + if let Some(record) = record { + // Everything apply writes must survive the durable encoding. + let decoded = NamespaceFenceRecord::decode( + FENCE_FORMAT_VERSION, + record.revision, + &record.encode(), + ) + .unwrap(); + assert_eq!(&decoded, record); + if let Some(old) = &self.record { + assert!(record.revision > old.revision, "revision must increase"); + } + self.record = Some(record.clone()); + } + self.store_receipt(receipt.clone()); + } + } + + fn store_receipt(&mut self, receipt: CommandReceipt) { + self.receipts.retain(|r| { + !(r.operation_id == receipt.operation_id && r.command_id == receipt.command_id) + }); + self.receipts.push(receipt); + } + + /// Send a fresh command with correct expectations and persist the result. + fn run(&mut self, op: Uuid, kind: CommandKind) -> Result { + let request = self.request(op, command(kind)); + self.run_request(&request, &env()) + } + + fn run_request( + &mut self, + request: &FenceRequest, + env: &ApplyEnv, + ) -> Result { + let decision = self.decide(request, env)?; + self.persist(&decision); + Ok(decision) + } + + fn complete(&mut self, completion: DrainCompletion) { + let record = self.record.clone().unwrap(); + let receipt = self + .receipts + .iter() + .find(|r| r.outcome == O::Draining && r.command_id == record.last_command_id) + .or_else(|| { + self.receipts + .iter() + .rev() + .find(|r| r.outcome == O::Draining) + }) + .unwrap() + .clone(); + let (next, receipt) = complete_drain(&record, &receipt, completion, &env()).unwrap(); + assert_eq!(next.revision, record.revision + 1); + self.record = Some(next); + self.store_receipt(receipt); + } + + fn state(&self) -> FenceState { + self.current().state() + } + + fn revision(&self) -> u64 { + self.current().revision() + } + + /// Drive the harness into `state` with operation `OP`. + fn in_state(state: FenceState) -> Self { + use CommandKind as K; + let boundary = DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 42, + }, + }; + let mut h = if state.role() == Some(Role::Target) || state == S::Absent { + Self::target() + } else { + Self::source() + }; + let steps: &[&dyn Fn(&mut Harness)] = match state { + S::Unfenced | S::Absent => &[], + S::SourceDraining => &[&|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap())], + S::SourceWriteFenced => &[ + &|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap()), + &|h| h.complete(boundary), + ], + S::SourceReadDraining => &[ + &|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap()), + &|h| h.complete(boundary), + &|h| drop(h.run(OP, K::SetSourceReadFence).unwrap()), + ], + S::SourceReadFenced => &[ + &|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap()), + &|h| h.complete(boundary), + &|h| drop(h.run(OP, K::SetSourceReadFence).unwrap()), + &|h| h.complete(DrainCompletion::SourceReads), + ], + S::Released => &[ + &|h| drop(h.run(OP, K::AcquireSourceWriteFence).unwrap()), + &|h| h.complete(boundary), + &|h| drop(h.run(OP, K::ReleaseSourceWriteFence).unwrap()), + ], + S::TargetQuarantined => { + &[&|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap())] + } + S::TargetImportDraining => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::SealTargetImport).unwrap()), + ], + S::TargetValidating => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::SealTargetImport).unwrap()), + &|h| h.complete(DrainCompletion::TargetImport), + ], + S::TargetWriteFenced => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::SealTargetImport).unwrap()), + &|h| h.complete(DrainCompletion::TargetImport), + &|h| drop(h.run(OP, K::RecordTargetValidation).unwrap()), + &|h| drop(h.run(OP, K::PublishTargetReadableWriteFenced).unwrap()), + ], + S::TargetWritable => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::SealTargetImport).unwrap()), + &|h| h.complete(DrainCompletion::TargetImport), + &|h| drop(h.run(OP, K::RecordTargetValidation).unwrap()), + &|h| drop(h.run(OP, K::PublishTargetReadableWriteFenced).unwrap()), + &|h| drop(h.run(OP, K::EnableTargetWrites).unwrap()), + ], + S::TargetAborted => &[ + &|h| drop(h.run(OP, K::CreateTargetQuarantined).unwrap()), + &|h| drop(h.run(OP, K::AbortQuarantinedTarget).unwrap()), + ], + S::UnknownUnavailable => panic!("not reachable by transitions"), + }; + for step in steps { + step(&mut h); + } + assert_eq!(h.state(), state); + h + } + } + + fn outcome_of(result: &Result) -> FenceOutcome { + match result { + Ok(Decision::Apply { receipt, .. }) => receipt.outcome, + Ok(Decision::Replay(r)) | Ok(Decision::Resume(r)) => r.outcome, + Err(e) => e.outcome(), + } + } + + fn state_after(result: &Result) -> Option { + match result { + Ok(Decision::Apply { receipt, .. }) => Some(receipt.state_after), + _ => None, + } + } + + const RECORD_STATES: [FenceState; 13] = [ + S::Unfenced, + S::Absent, + S::SourceDraining, + S::SourceWriteFenced, + S::SourceReadDraining, + S::SourceReadFenced, + S::Released, + S::TargetQuarantined, + S::TargetImportDraining, + S::TargetValidating, + S::TargetWriteFenced, + S::TargetWritable, + S::TargetAborted, + ]; + + /// Every (state, command) pair from the owning operation, with correct expectations, + /// against the full expected table: the legal transitions of section 3.2, the + /// `ALREADY_APPLIED` goal states, drain joins, and a refusal for everything else. + #[test] + fn exhaustive_owner_commands() { + use CommandKind as K; + + let expected = + |state: FenceState, kind: CommandKind| -> (FenceOutcome, Option) { + if kind == K::AdoptFence { + // Adopting one's own fence is never allowed; finished fences cannot be + // adopted; no record has nothing to adopt. + return (O::InvalidFenceTransition, None); + } + if kind == K::CreateTargetQuarantined { + return match state { + S::Absent => (O::Applied, Some(S::TargetQuarantined)), + S::TargetQuarantined => (O::AlreadyApplied, Some(S::TargetQuarantined)), + // The owner has finished with the namespace. + s if s.is_operation_finished() => (O::InvalidFenceTransition, None), + // The name is in use. + _ => (O::FencePreconditionFailed, None), + }; + } + if kind.goal_state() == Some(state) { + return (O::AlreadyApplied, Some(state)); + } + if drain_state(kind) == Some(state) { + return (O::Draining, Some(state)); + } + if state.is_operation_finished() && state != S::Unfenced { + return (O::InvalidFenceTransition, None); + } + match transition(kind, state) { + Some((to, outcome)) => { + if kind == K::PublishTargetReadableWriteFenced { + // No validation receipt has been recorded on this path. + (O::FencePreconditionFailed, None) + } else { + (outcome, Some(to)) + } + } + None => (O::InvalidFenceTransition, None), + } + }; + + for state in RECORD_STATES { + for kind in CommandKind::ALL { + // A target driven to TARGET_VALIDATING has no validation receipt yet. + let mut h = Harness::in_state(state); + let result = h.run(OP, kind); + let (outcome, to) = expected(state, kind); + assert_eq!(outcome_of(&result), outcome, "{state} {kind}: {result:?}"); + assert_eq!(state_after(&result), to, "{state} {kind}"); + } + } + } + + #[test] + fn source_happy_path() { + let mut h = Harness::source(); + let r = h.run(OP, CommandKind::AcquireSourceWriteFence).unwrap(); + let Decision::Apply { record, receipt } = r else { + panic!() + }; + let record = record.unwrap(); + assert_eq!(record.state, S::SourceDraining); + assert_eq!(record.revision, 1); + assert_eq!(receipt.outcome, O::Draining); + assert_eq!((receipt.revision_before, receipt.revision_after), (0, 1)); + assert_eq!(record.identity.log_id, Some(LOG)); + assert!(!record.write_admission().is_open()); + assert!(record.read_admission().is_open()); + + h.complete(DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 7, + }, + }); + assert_eq!(h.state(), S::SourceWriteFenced); + assert_eq!(h.revision(), 2); + assert_eq!( + h.record.as_ref().unwrap().frozen_boundary.unwrap().frame_no, + 7 + ); + let acquire_receipt = h + .receipts + .iter() + .find(|r| r.command == CommandKind::AcquireSourceWriteFence) + .unwrap(); + assert_eq!(acquire_receipt.outcome, O::Applied); + assert_eq!(acquire_receipt.revision_after, 2); + + h.run(OP, CommandKind::SetSourceReadFence).unwrap(); + assert_eq!((h.state(), h.revision()), (S::SourceReadDraining, 3)); + h.complete(DrainCompletion::SourceReads); + assert_eq!((h.state(), h.revision()), (S::SourceReadFenced, 4)); + h.run(OP, CommandKind::ClearSourceReadFence).unwrap(); + assert_eq!((h.state(), h.revision()), (S::SourceWriteFenced, 5)); + h.run(OP, CommandKind::ReleaseSourceWriteFence).unwrap(); + assert_eq!((h.state(), h.revision()), (S::Released, 6)); + assert!(h.record.as_ref().unwrap().write_admission().is_open()); + + // A new operation may acquire the released namespace; the revision keeps counting. + let r = h + .run(OTHER_OP, CommandKind::AcquireSourceWriteFence) + .unwrap(); + assert!(matches!(r, Decision::Apply { .. })); + let record = h.record.as_ref().unwrap(); + assert_eq!( + (record.state, record.revision, record.operation_id), + (S::SourceDraining, 7, OTHER_OP) + ); + assert!(record.frozen_boundary.is_none()); + } + + #[test] + fn target_happy_path() { + let mut h = Harness::target(); + h.run(OP, CommandKind::CreateTargetQuarantined).unwrap(); + let record = h.record.clone().unwrap(); + assert_eq!( + (record.role, record.state, record.revision), + (Role::Target, S::TargetQuarantined, 1) + ); + assert_eq!(record.identity.target_incarnation_id, Some(INCARNATION)); + assert!(!record.read_admission().is_open()); + + h.run(OP, CommandKind::SealTargetImport).unwrap(); + assert_eq!((h.state(), h.revision()), (S::TargetImportDraining, 2)); + h.complete(DrainCompletion::TargetImport); + assert_eq!((h.state(), h.revision()), (S::TargetValidating, 3)); + + // Publication needs a successful validation receipt first. + let err = h + .run(OP, CommandKind::PublishTargetReadableWriteFenced) + .unwrap_err(); + assert_eq!(err.outcome(), O::FencePreconditionFailed); + assert_eq!(err.detail(), Some(FenceDetail::ValidationReceiptRequired)); + + let failed = h.request( + OP, + FenceCommand::RecordTargetValidation { + result: ValidationResult::Failed, + summary: "mismatch".into(), + }, + ); + h.run_request(&failed, &env()).unwrap(); + assert_eq!((h.state(), h.revision()), (S::TargetValidating, 4)); + let err = h + .run(OP, CommandKind::PublishTargetReadableWriteFenced) + .unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::ValidationReceiptRequired)); + + let snapshot = ValidationSnapshot { + log_id: LOG, + frame_no: 9, + page_count: 3, + }; + let ok = h.request(OP, command(CommandKind::RecordTargetValidation)); + h.run_request( + &ok, + &ApplyEnv { + validation_snapshot: Some(snapshot), + ..env() + }, + ) + .unwrap(); + assert_eq!(h.revision(), 5); + assert_eq!( + h.record + .as_ref() + .unwrap() + .validation + .as_ref() + .unwrap() + .snapshot, + Some(snapshot) + ); + + h.run(OP, CommandKind::PublishTargetReadableWriteFenced) + .unwrap(); + assert_eq!((h.state(), h.revision()), (S::TargetWriteFenced, 6)); + assert!(h.record.as_ref().unwrap().read_admission().is_open()); + assert!(!h.record.as_ref().unwrap().write_admission().is_open()); + + h.run(OP, CommandKind::EnableTargetWrites).unwrap(); + assert_eq!((h.state(), h.revision()), (S::TargetWritable, 7)); + assert!(h.record.as_ref().unwrap().write_admission().is_open()); + } + + #[test] + fn validation_summary_is_bounded() { + let mut h = Harness::in_state(S::TargetValidating); + let request = h.request( + OP, + FenceCommand::RecordTargetValidation { + result: ValidationResult::Ok, + summary: "x".repeat(MAX_VALIDATION_SUMMARY_BYTES + 1), + }, + ); + let err = h.run_request(&request, &env()).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::InvalidArgument)); + } + + /// Section 2.8: nothing moves a published target back, for the owning operation. + #[test] + fn target_writable_is_irreversible() { + for kind in CommandKind::ALL { + let mut h = Harness::in_state(S::TargetWritable); + let before = h.record.clone(); + let result = h.run(OP, kind); + match kind { + CommandKind::EnableTargetWrites => { + assert_eq!(outcome_of(&result), O::AlreadyApplied) + } + _ => assert_eq!(outcome_of(&result), O::InvalidFenceTransition, "{kind}"), + } + assert_eq!(h.record, before, "{kind} changed a published target"); + } + + // Another operation cannot move it to a frozen, aborted or absent target state + // either; it can only start a new move with the namespace as a source. + for kind in CommandKind::ALL { + let mut h = Harness::in_state(S::TargetWritable); + let result = h.run(OTHER_OP, kind); + if kind == CommandKind::AcquireSourceWriteFence { + assert_eq!(state_after(&result), Some(S::SourceDraining)); + } else { + assert!(outcome_of(&result).is_error(), "{kind}: {result:?}"); + assert_eq!(h.state(), S::TargetWritable); + } + } + } + + #[test] + fn exact_replay_after_revision_advanced() { + let mut h = Harness::source(); + let acquire = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + h.run_request(&acquire, &env()).unwrap(); + + // While draining, a replay resumes the same drain. + let d = h.decide(&acquire, &env()).unwrap(); + assert!(matches!(&d, Decision::Resume(r) if r.command_id == acquire.command_id)); + + h.complete(DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 1, + }, + }); + h.run(OP, CommandKind::SetSourceReadFence).unwrap(); + h.complete(DrainCompletion::SourceReads); + assert_eq!(h.revision(), 4); + + // The original acquire's expectations (UNFENCED, 0) are long stale, but a replay is + // answered from its receipt before any revision check. + let d = h.decide(&acquire, &env()).unwrap(); + let Decision::Replay(receipt) = d else { + panic!("{d:?}") + }; + assert_eq!(receipt.outcome, O::Applied); + assert_eq!(receipt.state_after, S::SourceWriteFenced); + assert_eq!(receipt.revision_after, 2); + } + + #[test] + fn command_id_reuse_with_different_fingerprint_conflicts() { + let mut h = Harness::source(); + let acquire = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + h.run_request(&acquire, &env()).unwrap(); + let before = (h.record.clone(), h.receipts.clone()); + + let mut reused = acquire.clone(); + reused.command = FenceCommand::ReleaseSourceWriteFence; + reused.expected_state = S::SourceDraining; + reused.expected_revision = 1; + let err = h.decide(&reused, &env()).unwrap_err(); + assert_eq!(err.outcome(), O::FenceCommandConflict); + + let mut reused = acquire.clone(); + reused.command = FenceCommand::AcquireSourceWriteFence { + expected_log_id: LOG, + drain_policy: None, + }; + assert_eq!( + h.decide(&reused, &env()).unwrap_err().outcome(), + O::FenceCommandConflict + ); + + assert_eq!((h.record.clone(), h.receipts.clone()), before); + } + + #[test] + fn wrong_owner_is_refused() { + for state in RECORD_STATES { + if !state.is_durable() || state.is_operation_finished() { + continue; + } + for kind in CommandKind::ALL { + if kind == CommandKind::AdoptFence { + continue; + } + let mut h = Harness::in_state(state); + let before = h.record.clone(); + let result = h.run(OTHER_OP, kind); + // The owner is checked before anything about the command. + assert_eq!( + outcome_of(&result), + O::FenceOwnedByAnotherOperation, + "{state} {kind}" + ); + assert_eq!(h.record, before); + } + } + } + + #[test] + fn stale_revision_is_refused() { + let mut h = Harness::in_state(S::SourceWriteFenced); + let mut request = h.request(OP, command(CommandKind::SetSourceReadFence)); + request.expected_revision -= 1; + assert_eq!( + h.decide(&request, &env()).unwrap_err().outcome(), + O::FenceRevisionMismatch + ); + + let mut request = h.request(OP, command(CommandKind::SetSourceReadFence)); + request.expected_state = S::SourceDraining; + assert_eq!( + h.decide(&request, &env()).unwrap_err().outcome(), + O::FenceRevisionMismatch + ); + + let mut request = h.request(OP, command(CommandKind::SetSourceReadFence)); + request.expected_revision += 1; + assert_eq!( + h.decide(&request, &env()).unwrap_err().outcome(), + O::FenceRevisionMismatch + ); + } + + #[test] + fn role_mismatch() { + let mut h = Harness::in_state(S::SourceWriteFenced); + let err = h.run(OP, CommandKind::SealTargetImport).unwrap_err(); + assert_eq!(err.outcome(), O::InvalidFenceTransition); + assert_eq!(err.detail(), Some(FenceDetail::RoleMismatch)); + + let mut h = Harness::in_state(S::TargetQuarantined); + let err = h.run(OP, CommandKind::SetSourceReadFence).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::RoleMismatch)); + + let mut h = Harness::source(); + let err = h.run(OP, CommandKind::EnableTargetWrites).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::RoleMismatch)); + + let mut h = Harness::target(); + let err = h.run(OP, CommandKind::AcquireSourceWriteFence).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::RoleMismatch)); + } + + /// The pure half of `acquire_race_single_owner`: two operations race to acquire the same + /// namespace; the store's serialised transactions mean the second decides against the + /// first's committed record and loses with a typed conflict. + #[test] + fn acquire_race_single_owner() { + let mut h = Harness::source(); + let a = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + let b = h.request(OTHER_OP, command(CommandKind::AcquireSourceWriteFence)); + h.run_request(&a, &env()).unwrap(); + let err = h.run_request(&b, &env()).unwrap_err(); + assert_eq!(err.outcome(), O::FenceOwnedByAnotherOperation); + assert_eq!(h.record.as_ref().unwrap().operation_id, OP); + } + + #[test] + fn acquire_preconditions() { + let mut h = Harness::source(); + let request = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + let err = h + .decide( + &request, + &ApplyEnv { + namespace_log_id: Some(Uuid::from_u128(0x11)), + ..env() + }, + ) + .unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::NamespaceIdentityMismatch)); + + let err = h + .decide( + &request, + &ApplyEnv { + shared_schema: true, + ..env() + }, + ) + .unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::SharedSchemaUnsupported)); + } + + #[test] + fn acquire_saves_and_release_restores_legacy_blocks() { + let saved = LegacyBlocks { + block_reads: false, + block_writes: true, + block_reason: Some("maintenance".into()), + }; + let mut h = Harness::source(); + let request = h.request(OP, command(CommandKind::AcquireSourceWriteFence)); + h.run_request( + &request, + &ApplyEnv { + legacy_blocks: saved.clone(), + ..env() + }, + ) + .unwrap(); + let mirror = h.record.as_ref().unwrap().legacy_mirror(); + assert!(mirror.block_writes); + assert!(mirror + .block_reason + .unwrap() + .starts_with("namespace fence: SOURCE_DRAINING")); + + h.run(OP, CommandKind::ReleaseSourceWriteFence).unwrap(); + assert_eq!(h.record.as_ref().unwrap().legacy_mirror(), saved); + } + + #[test] + fn release_from_draining_is_a_precommit_rollback() { + let mut h = Harness::in_state(S::SourceDraining); + h.run(OP, CommandKind::ReleaseSourceWriteFence).unwrap(); + assert_eq!((h.state(), h.revision()), (S::Released, 2)); + } + + #[test] + fn owner_joins_its_own_drain_with_a_new_command() { + let mut h = Harness::in_state(S::SourceDraining); + let d = h.run(OP, CommandKind::AcquireSourceWriteFence).unwrap(); + let Decision::Apply { record, receipt } = d else { + panic!() + }; + assert!(record.is_none()); + assert_eq!(receipt.outcome, O::Draining); + assert_eq!(h.revision(), 1); + + // The drain completes through the new command's receipt. + let record = h.record.clone().unwrap(); + let (next, final_receipt) = complete_drain( + &record, + &receipt, + DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 3, + }, + }, + &env(), + ) + .unwrap(); + assert_eq!(next.state, S::SourceWriteFenced); + assert_eq!(final_receipt.command_id, receipt.command_id); + assert_eq!(final_receipt.outcome, O::Applied); + } + + #[test] + fn complete_drain_checks() { + let h = Harness::in_state(S::SourceDraining); + let record = h.record.clone().unwrap(); + let receipt = h.receipts[0].clone(); + + // Wrong kind of completion for the state. + assert!(complete_drain(&record, &receipt, DrainCompletion::SourceReads, &env()).is_err()); + assert!(complete_drain(&record, &receipt, DrainCompletion::TargetImport, &env()).is_err()); + + // A boundary from another log. + let err = complete_drain( + &record, + &receipt, + DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: Uuid::from_u128(0x77), + frame_no: 1, + }, + }, + &env(), + ) + .unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::NamespaceIdentityMismatch)); + + // A final receipt, or another operation's. + let mut final_receipt = receipt.clone(); + final_receipt.outcome = O::Applied; + let boundary = DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 1, + }, + }; + assert!(complete_drain(&record, &final_receipt, boundary, &env()).is_err()); + let mut other = receipt.clone(); + other.operation_id = OTHER_OP; + assert!(complete_drain(&record, &other, boundary, &env()).is_err()); + + // Not draining any more. + let h = Harness::in_state(S::SourceWriteFenced); + let record = h.record.clone().unwrap(); + assert!(complete_drain(&record, &receipt, boundary, &env()).is_err()); + } + + #[test] + fn unavailable_refuses_everything_else() { + let marker = Harness::in_state(S::SourceWriteFenced).record.unwrap(); + for detail in [ + FenceDetail::CorruptRecord, + FenceDetail::UnsupportedFormatVersion, + FenceDetail::MetastoreBehindMarker, + FenceDetail::IncompleteTargetCreation, + FenceDetail::IndeterminateCommit, + ] { + for kind in CommandKind::ALL { + if kind == CommandKind::AdoptFence && detail == FenceDetail::MetastoreBehindMarker { + continue; + } + let current = CurrentFence::Unavailable { + detail, + marker: Some(&marker), + }; + let request = FenceRequest { + namespace: NamespaceName::from("db1"), + operation_id: OP, + command_id: Uuid::from_u128(0x5000), + expected_state: S::UnknownUnavailable, + expected_revision: marker.revision, + command: command(kind), + }; + let err = apply(current, None, &request, &env()).unwrap_err(); + assert_eq!(err.outcome(), O::FenceStateUnavailable, "{detail} {kind}"); + assert_eq!(err.detail(), Some(detail)); + } + } + } + + #[test] + fn incomplete_target_creation_is_completed_only_by_the_same_command() { + let mut h = Harness::target(); + let create = h.request(OP, command(CommandKind::CreateTargetQuarantined)); + let Decision::Apply { record, .. } = h.decide(&create, &env()).unwrap() else { + panic!() + }; + // The store writes this marker, then crashes before the metastore commit. + let marker = FenceMarker::for_record(record.as_ref().unwrap()); + let marker = FenceMarker::decode(&marker.encode()).unwrap().record; + let current = CurrentFence::Unavailable { + detail: FenceDetail::IncompleteTargetCreation, + marker: Some(&marker), + }; + + // Another command id, even from the same operation, cannot complete it. + let mut other = create.clone(); + other.command_id = Uuid::from_u128(0x6000); + let err = apply(current, None, &other, &env()).unwrap_err(); + assert_eq!(err.outcome(), O::FenceStateUnavailable); + + // The same command does, with the incarnation id the marker recorded. + let d = apply( + current, + None, + &create, + &ApplyEnv { + new_incarnation_id: Uuid::from_u128(0x7777), + ..env() + }, + ) + .unwrap(); + let Decision::Apply { + record: Some(record), + receipt, + } = d + else { + panic!() + }; + assert_eq!(record, marker); + assert_eq!(receipt.outcome, O::Applied); + } + + fn adopt_request(h: &mut Harness, args: AdoptArgs) -> FenceRequest { + h.request(OTHER_OP, FenceCommand::AdoptFence(args)) + } + + fn adopt_args() -> AdoptArgs { + match command(CommandKind::AdoptFence) { + FenceCommand::AdoptFence(args) => args, + _ => unreachable!(), + } + } + + fn authorised() -> ApplyEnv { + ApplyEnv { + adoption_authorised: true, + ..env() + } + } + + #[test] + fn adopt_requires_key_and_two_approvers() { + let mut h = Harness::in_state(S::SourceWriteFenced); + let request = adopt_request(&mut h, adopt_args()); + let err = h.decide(&request, &env()).unwrap_err(); + assert_eq!(err.detail(), Some(FenceDetail::AdoptionNotAuthorised)); + + let bad = [ + AdoptArgs { + approvers: vec!["alice".into()], + ..adopt_args() + }, + AdoptArgs { + approvers: vec!["alice".into(), " alice ".into()], + ..adopt_args() + }, + AdoptArgs { + approvers: vec!["alice".into(), "".into()], + ..adopt_args() + }, + AdoptArgs { + approvers: vec!["a".into(), "b".into(), "c".into()], + ..adopt_args() + }, + AdoptArgs { + incident_ref: " ".into(), + ..adopt_args() + }, + AdoptArgs { + reason: "".into(), + ..adopt_args() + }, + ]; + for args in bad { + let request = adopt_request(&mut h, args.clone()); + let err = h.decide(&request, &authorised()).unwrap_err(); + assert_eq!( + err.detail(), + Some(FenceDetail::AdoptionNotAuthorised), + "{args:?}" + ); + } + + let wrong_owner = AdoptArgs { + current_operation_id: Uuid::from_u128(0xdead), + ..adopt_args() + }; + let request = adopt_request(&mut h, wrong_owner); + assert_eq!( + h.decide(&request, &authorised()).unwrap_err().outcome(), + O::FenceOwnedByAnotherOperation + ); + } + + #[test] + fn adopt_keeps_gates_closed() { + for state in [ + S::SourceDraining, + S::SourceWriteFenced, + S::SourceReadDraining, + S::SourceReadFenced, + S::TargetQuarantined, + S::TargetImportDraining, + S::TargetValidating, + S::TargetWriteFenced, + ] { + let mut h = Harness::in_state(state); + let before = h.record.clone().unwrap(); + let request = adopt_request(&mut h, adopt_args()); + h.run_request(&request, &authorised()).unwrap(); + let after = h.record.clone().unwrap(); + assert_eq!(after.state, state); + assert_eq!(after.write_admission(), before.write_admission()); + assert_eq!(after.read_admission(), before.read_admission()); + assert_eq!(after.revision, before.revision + 1); + assert_eq!(after.operation_id, OTHER_OP); + assert_eq!(after.adoptions.len(), 1); + assert_eq!(after.adoptions[0].approvers, vec!["alice", "bob"]); + let receipt = h.receipts.last().unwrap(); + assert_eq!(receipt.adoption.as_ref().unwrap().previous_operation_id, OP); + + // The old owner is now locked out. + let result = h.run(OP, CommandKind::ReleaseSourceWriteFence); + assert_eq!(outcome_of(&result), O::FenceOwnedByAnotherOperation); + } + } + + #[test] + fn adopt_cannot_touch_finished_fences() { + for state in [S::TargetWritable, S::TargetAborted, S::Released] { + let mut h = Harness::in_state(state); + let request = adopt_request(&mut h, adopt_args()); + let err = h.decide(&request, &authorised()).unwrap_err(); + assert_eq!(err.outcome(), O::InvalidFenceTransition, "{state}"); + assert_eq!(err.detail(), Some(FenceDetail::OperationFinished)); + } + let mut h = Harness::source(); + let request = adopt_request(&mut h, adopt_args()); + assert_eq!( + h.decide(&request, &authorised()).unwrap_err().outcome(), + O::InvalidFenceTransition + ); + } + + #[test] + fn adopted_owner_can_finish_the_drain() { + let mut h = Harness::in_state(S::SourceDraining); + let request = adopt_request(&mut h, adopt_args()); + h.run_request(&request, &authorised()).unwrap(); + let d = h + .run(OTHER_OP, CommandKind::AcquireSourceWriteFence) + .unwrap(); + assert!( + matches!(d, Decision::Apply { record: None, ref receipt } if receipt.outcome == O::Draining) + ); + h.complete(DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 5, + }, + }); + assert_eq!(h.state(), S::SourceWriteFenced); + assert_eq!(h.record.as_ref().unwrap().operation_id, OTHER_OP); + } + + #[test] + fn adopt_after_metastore_rollback_restores_marker_record() { + let marker = Harness::in_state(S::SourceReadFenced).record.unwrap(); + let current = CurrentFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + marker: Some(&marker), + }; + let request = FenceRequest { + namespace: NamespaceName::from("db1"), + operation_id: OTHER_OP, + command_id: Uuid::from_u128(0x8000), + expected_state: S::SourceReadFenced, + expected_revision: marker.revision, + command: FenceCommand::AdoptFence(adopt_args()), + }; + let d = apply(current, None, &request, &authorised()).unwrap(); + let Decision::Apply { + record: Some(record), + .. + } = d + else { + panic!() + }; + assert_eq!(record.state, S::SourceReadFenced); + assert_eq!(record.revision, marker.revision + 1); + assert_eq!(record.operation_id, OTHER_OP); + assert_eq!(record.frozen_boundary, marker.frozen_boundary); + } + + #[test] + fn revision_increases_by_one_per_applied_transition() { + let h = Harness::in_state(S::TargetWritable); + let mut receipts = h.receipts.clone(); + receipts.sort_by_key(|r| r.revision_after); + let mut last = 0; + for r in receipts { + if r.outcome == O::AlreadyApplied { + continue; + } + assert_eq!( + r.revision_after, + last + if r.command == CommandKind::SealTargetImport { + 2 + } else { + 1 + }, + "{r:?}" + ); + last = r.revision_after; + } + assert_eq!(h.revision(), 6); + } +} diff --git a/libsql-server/src/namespace/mod.rs b/libsql-server/src/namespace/mod.rs index ec45b50445..cba4030090 100644 --- a/libsql-server/src/namespace/mod.rs +++ b/libsql-server/src/namespace/mod.rs @@ -20,6 +20,7 @@ pub use self::store::NamespaceStore; pub mod broadcasters; pub(crate) mod configurator; +pub mod fence; pub mod meta_store; mod name; pub mod replication_wal; diff --git a/libsql-server/tests/bootstrap.rs b/libsql-server/tests/bootstrap.rs index a464f53288..912015e1f9 100644 --- a/libsql-server/tests/bootstrap.rs +++ b/libsql-server/tests/bootstrap.rs @@ -3,7 +3,7 @@ use std::process::Command; #[test] fn bootstrap() { - let iface_files = &["proto/admin_shell.proto"]; + let iface_files = &["proto/admin_shell.proto", "proto/namespace_fence.proto"]; let dirs = &["proto"]; let out_dir = PathBuf::from(std::env!("CARGO_MANIFEST_DIR")) From 7f1b4e982f3eacec9717d4b615eb58b32149fd1a Mon Sep 17 00:00:00 2001 From: River Date: Tue, 29 Sep 2026 14:32:58 +0000 Subject: [PATCH 2/3] libsql-server: persist namespace fences in the metastore Add the additive `namespace_fences` and `namespace_fence_receipts` tables (created with --enable-namespace-fence, loaded and enforced whenever they exist) and run every fence command as a compare-and-swap in one BEGIN IMMEDIATE metastore transaction: read the record, the command's receipt and the config row, decide with the pure transition function, then commit the record, the receipt and the legacy block_* mirror together. The revision is stored, so it survives restart. Stored state is read strictly: an unknown format version, an undecodable payload, a revision column that disagrees with its payload, or a per-namespace `.fence` marker that is ahead of the metastore makes the namespace UNKNOWN_UNAVAILABLE instead of guessing. The marker is written after each commit (before it, for target creation) and rewritten on load when it fell behind. Ordinary config writes and deletes now take BEGIN IMMEDIATE and refuse while the fence denies lifecycle operations, and a config write that failed to persist is no longer published to the in-memory config. Receipts of finished operations are pruned after a retention period. Co-authored-by: Tomasz Szymczyszyn --- libsql-server/src/config.rs | 6 + libsql-server/src/error.rs | 3 + libsql-server/src/main.rs | 14 + libsql-server/src/namespace/fence/mod.rs | 6 +- libsql-server/src/namespace/fence/record.rs | 2 +- libsql-server/src/namespace/fence/store.rs | 951 +++++++++++++ libsql-server/src/namespace/meta_store.rs | 1381 ++++++++++++++++++- 7 files changed, 2342 insertions(+), 21 deletions(-) create mode 100644 libsql-server/src/namespace/fence/store.rs diff --git a/libsql-server/src/config.rs b/libsql-server/src/config.rs index 2c3c302a6d..9ac7add98b 100644 --- a/libsql-server/src/config.rs +++ b/libsql-server/src/config.rs @@ -187,6 +187,12 @@ pub struct MetaStoreConfig { pub allow_recover_from_fs: bool, /// Destroy the metastore if there is a restore error pub destroy_on_error: bool, + /// Allow namespace fences to be used: creates the fence tables. Fences that already exist + /// are loaded and enforced whether or not this is set. + pub namespace_fence: bool, + /// How long receipts of finished fence operations are kept. `None` is the default of + /// 30 days. + pub namespace_fence_receipt_retention: Option, } #[derive(Debug, Clone)] diff --git a/libsql-server/src/error.rs b/libsql-server/src/error.rs index bfe67f47c7..f0cb631769 100644 --- a/libsql-server/src/error.rs +++ b/libsql-server/src/error.rs @@ -128,6 +128,8 @@ pub enum Error { RuntimeTaskJoinError(#[from] tokio::task::JoinError), #[error("database is not a primary")] NotAPrimary, + #[error(transparent)] + NamespaceFence(#[from] crate::namespace::fence::outcome::FenceError), } impl AsRef for Error { @@ -224,6 +226,7 @@ impl IntoResponse for &Error { AttachInMigration => self.format_err(StatusCode::BAD_REQUEST), RuntimeTaskJoinError(_) => self.format_err(StatusCode::INTERNAL_SERVER_ERROR), NotAPrimary => self.format_err(StatusCode::BAD_REQUEST), + NamespaceFence(e) => self.format_err(e.outcome().admin_http_status()), } } } diff --git a/libsql-server/src/main.rs b/libsql-server/src/main.rs index 307d5482fe..5d738a1ed5 100644 --- a/libsql-server/src/main.rs +++ b/libsql-server/src/main.rs @@ -258,6 +258,16 @@ struct Cli { #[clap(long, env = "SQLD_ALLOW_METASTORE_RECOVERY")] allow_metastore_recovery: bool, + /// Allow namespace fences to be used (see `docs/NAMESPACE_FENCE.md`). Off by default. + /// Fences that already exist in the metastore are enforced either way. + #[clap(long, env = "SQLD_ENABLE_NAMESPACE_FENCE")] + enable_namespace_fence: bool, + + /// How long, in seconds, receipts of finished namespace-fence operations are kept. + /// Defaults to 30 days. + #[clap(long, env = "SQLD_NAMESPACE_FENCE_RECEIPT_RETENTION_S")] + namespace_fence_receipt_retention_s: Option, + /// Shutdown timeout duration in seconds, defaults to 30 seconds. #[clap(long, env = "SQLD_SHUTDOWN_TIMEOUT")] shutdown_timeout: Option, @@ -650,6 +660,10 @@ fn make_meta_store_config(config: &Cli) -> anyhow::Result { bottomless, allow_recover_from_fs: config.allow_metastore_recovery, destroy_on_error: config.meta_store_destroy_on_error, + namespace_fence: config.enable_namespace_fence, + namespace_fence_receipt_retention: config + .namespace_fence_receipt_retention_s + .map(Duration::from_secs), }) } diff --git a/libsql-server/src/namespace/fence/mod.rs b/libsql-server/src/namespace/fence/mod.rs index 3ffe2746f8..672b0565e8 100644 --- a/libsql-server/src/namespace/fence/mod.rs +++ b/libsql-server/src/namespace/fence/mod.rs @@ -5,8 +5,9 @@ //! `docs/NAMESPACE_FENCE.md` is the contract and the design. This module holds the parts with //! no I/O: the states and permission matrix ([`state`]), the stable outcome codes and their //! protocol mappings ([`outcome`]), commands and their canonical fingerprint ([`command`]), -//! records, receipts and markers with their strict durable encoding ([`record`]), and the pure -//! transition function ([`transition`]). +//! records, receipts and markers with their strict durable encoding ([`record`]), the pure +//! transition function ([`transition`]), and the metastore tables, compare-and-swap and marker +//! file that persist them ([`store`], driven by `MetaStore::apply_fence_command`). // The persistence, controller and protocol layers that consume these types land in the // following commits of this series; until then most of the module is unused by the rest of @@ -17,6 +18,7 @@ pub mod command; pub mod outcome; pub mod record; pub mod state; +pub mod store; pub mod transition; #[allow(clippy::all)] diff --git a/libsql-server/src/namespace/fence/record.rs b/libsql-server/src/namespace/fence/record.rs index 9599f2a849..a2d9f7f5dd 100644 --- a/libsql-server/src/namespace/fence/record.rs +++ b/libsql-server/src/namespace/fence/record.rs @@ -728,7 +728,7 @@ pub(super) mod tests { } } - fn sample_receipt() -> CommandReceipt { + pub fn sample_receipt() -> CommandReceipt { CommandReceipt { namespace: NamespaceName::from("db1"), operation_id: Uuid::from_u128(1), diff --git a/libsql-server/src/namespace/fence/store.rs b/libsql-server/src/namespace/fence/store.rs new file mode 100644 index 0000000000..2d6f9ca217 --- /dev/null +++ b/libsql-server/src/namespace/fence/store.rs @@ -0,0 +1,951 @@ +//! Durable fence state: the metastore tables, the fence compare-and-swap, and the +//! per-namespace marker file (`docs/NAMESPACE_FENCE.md` sections 5.1 and 5.4 to 5.6). +//! +//! Everything here runs on a metastore connection, inside a transaction the caller opened with +//! `BEGIN IMMEDIATE`, so a fence transition, an ordinary config write and a delete of the same +//! namespace are serialised by SQLite's write lock whichever connection they use. The functions +//! only read and write rows and files: what a command does is decided by +//! [`transition::apply`](super::transition::apply), and when the result is published is the +//! caller's decision, made only after the transaction has committed. +//! +//! Reading is strict. A row with an unknown format version, an undecodable payload, or a +//! revision column that disagrees with its payload, and a marker that says more than the +//! metastore does, all read as [`StoredFence::Unavailable`]: the namespace is +//! `UNKNOWN_UNAVAILABLE` and every gate that consults it stays closed. + +use std::fs::{self, File}; +use std::io::{self, Write as _}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use prost::Message as _; +use rusqlite::{params, OptionalExtension}; +use uuid::Uuid; + +use crate::connection::config::{DatabaseConfig, DurabilityMode}; +use crate::namespace::NamespaceName; +use crate::LIBSQL_PAGE_SIZE; +use libsql_replication::rpc::metadata; + +use super::command::TargetConfig; +use super::outcome::{FenceDetail, FenceError, FenceOutcome}; +use super::record::{ + CommandReceipt, FenceDecodeError, FenceMarker, LegacyBlocks, NamespaceFenceRecord, + FENCE_FORMAT_VERSION, +}; +use super::state::{FenceState, OperationClass}; +use super::transition::CurrentFence; + +/// Name of the per-namespace marker file, inside the namespace's directory. +pub const MARKER_FILE_NAME: &str = ".fence"; + +/// How long receipts of finished operations are kept by default (section 5.5). +pub const DEFAULT_RECEIPT_RETENTION: Duration = Duration::from_secs(30 * 24 * 60 * 60); + +const CREATE_FENCES_TABLE: &str = " + CREATE TABLE IF NOT EXISTS namespace_fences ( + namespace TEXT NOT NULL PRIMARY KEY, + format_version INTEGER NOT NULL, + revision INTEGER NOT NULL, + record BLOB NOT NULL, + FOREIGN KEY (namespace) REFERENCES namespace_configs (namespace) + ON DELETE RESTRICT ON UPDATE RESTRICT + )"; + +const CREATE_RECEIPTS_TABLE: &str = " + CREATE TABLE IF NOT EXISTS namespace_fence_receipts ( + namespace TEXT NOT NULL, + operation_id TEXT NOT NULL, + command_id TEXT NOT NULL, + format_version INTEGER NOT NULL, + revision_after INTEGER NOT NULL, + applied_at INTEGER NOT NULL, + receipt BLOB NOT NULL, + PRIMARY KEY (namespace, operation_id, command_id) + )"; + +/// Errors of the persistence layer. A fence outcome is a result the caller answers with; the +/// others are faults of the metastore or the filesystem. +#[derive(Debug, thiserror::Error)] +pub enum FenceStoreError { + #[error(transparent)] + Fence(#[from] FenceError), + #[error("metastore error: {0}")] + Sqlite(#[from] rusqlite::Error), + #[error("fence marker I/O error: {0}")] + Io(#[from] io::Error), +} + +/// Create the fence tables. Called when the fence is enabled; the tables are additive, and +/// the metastore of a server that never enabled the fence does not have them. +pub fn create_tables(conn: &rusqlite::Connection) -> rusqlite::Result<()> { + conn.execute(CREATE_FENCES_TABLE, ())?; + conn.execute(CREATE_RECEIPTS_TABLE, ())?; + Ok(()) +} + +/// Whether this metastore has ever held fence state. Once it has, fences are loaded and +/// enforced whether or not the fence is enabled (section 13.1). +pub fn tables_exist(conn: &rusqlite::Connection) -> rusqlite::Result { + let count: i64 = conn.query_row( + "SELECT count(*) FROM sqlite_master WHERE type = 'table' + AND name IN ('namespace_fences', 'namespace_fence_receipts')", + (), + |row| row.get(0), + )?; + Ok(count > 0) +} + +/// What the store established about one namespace's fence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StoredFence { + /// No fence record and no marker. `namespace_exists` is whether the namespace has a config + /// row: `UNFENCED` when it does, `ABSENT` when it does not. + None { + namespace_exists: bool, + }, + Record(NamespaceFenceRecord), + /// The control state cannot be established: `UNKNOWN_UNAVAILABLE`. + Unavailable { + detail: FenceDetail, + /// What was wrong, for the operator log and `InspectFence`. + reason: String, + /// The record the marker file holds, when it has a readable one. + marker: Option, + }, +} + +impl StoredFence { + pub fn as_current(&self) -> CurrentFence<'_> { + match self { + StoredFence::None { namespace_exists } => CurrentFence::None { + namespace_exists: *namespace_exists, + }, + StoredFence::Record(r) => CurrentFence::Record(r), + StoredFence::Unavailable { detail, marker, .. } => CurrentFence::Unavailable { + detail: *detail, + marker: marker.as_ref(), + }, + } + } + + pub fn state(&self) -> FenceState { + self.as_current().state() + } + + pub fn revision(&self) -> u64 { + self.as_current().revision() + } + + pub fn record(&self) -> Option<&NamespaceFenceRecord> { + match self { + StoredFence::Record(r) => Some(r), + _ => None, + } + } + + /// The permission-matrix decision for work of `class`, as an error a caller can return. + pub fn permits(&self, class: OperationClass) -> Result<(), FenceError> { + self.state().permits(class).map_err(|outcome| { + let err = FenceError::new(outcome, self.denial_message(class)); + match self { + StoredFence::Unavailable { detail, .. } => err.with_detail(*detail), + _ => err, + } + }) + } + + fn denial_message(&self, class: OperationClass) -> String { + match self { + StoredFence::Record(r) => format!( + "{class:?} is not permitted while the namespace fence is {} (operation {}, revision {})", + r.state, r.operation_id, r.revision + ), + StoredFence::Unavailable { reason, .. } => { + format!("the namespace's fence state cannot be established: {reason}") + } + StoredFence::None { .. } => format!("{class:?} is not permitted"), + } + } +} + +/// Where the marker of `namespace` lives, under the server's `dbs` directory. +pub fn marker_path(dbs_path: &Path, namespace: &NamespaceName) -> PathBuf { + dbs_path.join(namespace.as_str()).join(MARKER_FILE_NAME) +} + +/// Read a namespace's marker. `Ok(None)` when there is none; `Ok(Some(Err(_)))` when there is +/// one that cannot be decoded. +pub fn read_marker( + dbs_path: &Path, + namespace: &NamespaceName, +) -> io::Result>> { + match fs::read(marker_path(dbs_path, namespace)) { + Ok(bytes) => Ok(Some(FenceMarker::decode(&bytes))), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } +} + +/// Durably replace a namespace's marker with a copy of `record`: write a temporary file, fsync +/// it, rename it over the marker and fsync the directory. Creates the namespace directory if +/// it does not exist yet (a target that is being created). +pub fn write_marker(dbs_path: &Path, record: &NamespaceFenceRecord) -> io::Result<()> { + let path = marker_path(dbs_path, &record.namespace); + let dir = path.parent().expect("marker path has a parent"); + fs::create_dir_all(dir)?; + let tmp = dir.join(format!("{MARKER_FILE_NAME}.tmp")); + { + let mut file = File::create(&tmp)?; + file.write_all(&FenceMarker::for_record(record).encode())?; + file.sync_all()?; + } + fs::rename(&tmp, &path)?; + File::open(dir)?.sync_all()?; + Ok(()) +} + +/// Remove a namespace's marker, if it has one, and fsync its directory. +pub fn remove_marker(dbs_path: &Path, namespace: &NamespaceName) -> io::Result<()> { + let path = marker_path(dbs_path, namespace); + match fs::remove_file(&path) { + Ok(()) => File::open(path.parent().expect("marker path has a parent"))?.sync_all(), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +/// Whether the marker agrees with what the metastore says. Returned by [`read_fence`] so a +/// loader can repair a marker that fell behind (a crash between commit and marker write). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MarkerStatus { + /// The marker holds the stored record, or there is neither. + Current, + /// The metastore has a record and the marker is missing or older: rewrite it. + Stale, + /// The marker is what makes the namespace unavailable, or cannot be read. + Conflicting, +} + +struct RawFenceRow { + format_version: i64, + revision: i64, + record: Vec, +} + +fn read_raw_row( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result> { + conn.query_row( + "SELECT format_version, revision, record FROM namespace_fences WHERE namespace = ?1", + [namespace.as_str()], + |row| { + Ok(RawFenceRow { + format_version: row.get(0)?, + revision: row.get(1)?, + record: row.get(2)?, + }) + }, + ) + .optional() +} + +/// The stored revision column of `namespace`'s fence row, whatever its payload says. +pub fn stored_revision( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result> { + conn.query_row( + "SELECT revision FROM namespace_fences WHERE namespace = ?1", + [namespace.as_str()], + |row| row.get(0), + ) + .optional() +} + +fn decode_row(row: &RawFenceRow) -> Result { + let format_version = u32::try_from(row.format_version) + .map_err(|_| FenceDecodeError::UnsupportedFormatVersion(u32::MAX))?; + let revision = u64::try_from(row.revision) + .map_err(|_| FenceDecodeError::Invalid("negative revision column"))?; + NamespaceFenceRecord::decode(format_version, revision, &row.record) +} + +fn decode_detail(e: &FenceDecodeError) -> FenceDetail { + match e { + FenceDecodeError::UnsupportedFormatVersion(_) => FenceDetail::UnsupportedFormatVersion, + FenceDecodeError::Undecodable(_) | FenceDecodeError::Invalid(_) => { + FenceDetail::CorruptRecord + } + } +} + +/// Read the config row of `namespace`, if it has one. +pub fn read_config_row( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> Result, FenceStoreError> { + let bytes: Option> = conn + .query_row( + "SELECT config FROM namespace_configs WHERE namespace = ?1", + [namespace.as_str()], + |row| row.get(0), + ) + .optional()?; + match bytes { + None => Ok(None), + Some(bytes) => match metadata::DatabaseConfig::decode(&bytes[..]) { + Ok(c) => Ok(Some(DatabaseConfig::from(&c))), + Err(e) => Err(FenceError::new( + FenceOutcome::FenceStateUnavailable, + format!("the namespace's config row cannot be decoded: {e}"), + ) + .with_detail(FenceDetail::CorruptRecord) + .into()), + }, + } +} + +fn config_row_exists( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result { + conn.query_row( + "SELECT count(*) FROM namespace_configs WHERE namespace = ?1", + [namespace.as_str()], + |row| row.get::<_, i64>(0), + ) + .map(|n| n > 0) +} + +/// Establish the fence of `namespace` from its row and its marker (section 5.6). +/// +/// Only a metastore that has the fence tables can hold a record; `dbs_path` is where the +/// namespace directories, and so the markers, are. +pub fn read_fence( + conn: &rusqlite::Connection, + dbs_path: &Path, + namespace: &NamespaceName, +) -> Result<(StoredFence, MarkerStatus), FenceStoreError> { + let row = read_raw_row(conn, namespace)?; + let marker = read_marker(dbs_path, namespace)?; + + let marker_record = match &marker { + Some(Ok(m)) => Some(m.record.clone()), + _ => None, + }; + + let Some(row) = row else { + return Ok(match marker { + None => ( + StoredFence::None { + namespace_exists: config_row_exists(conn, namespace)?, + }, + MarkerStatus::Current, + ), + Some(Err(e)) => ( + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + reason: format!("the fence marker cannot be decoded: {e}"), + marker: None, + }, + MarkerStatus::Conflicting, + ), + Some(Ok(m)) => { + let incomplete_creation = m.record.state == FenceState::TargetQuarantined + && m.record.revision == 1 + && !config_row_exists(conn, namespace)?; + let (detail, reason) = if incomplete_creation { + ( + FenceDetail::IncompleteTargetCreation, + "a target creation was interrupted before its metastore commit".to_string(), + ) + } else { + ( + FenceDetail::MetastoreBehindMarker, + format!( + "the marker records revision {} but the metastore has no fence record", + m.record.revision + ), + ) + }; + ( + StoredFence::Unavailable { + detail, + reason, + marker: Some(m.record), + }, + MarkerStatus::Conflicting, + ) + } + }); + }; + + let record = match decode_row(&row) { + Ok(record) => record, + Err(e) => { + return Ok(( + StoredFence::Unavailable { + detail: decode_detail(&e), + reason: format!("the fence record cannot be read: {e}"), + marker: marker_record, + }, + MarkerStatus::Conflicting, + )) + } + }; + + if record.namespace != *namespace { + return Ok(( + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + reason: format!("the fence record names namespace `{}`", record.namespace), + marker: marker_record, + }, + MarkerStatus::Conflicting, + )); + } + + Ok(match marker { + Some(Ok(m)) if m.record.revision > record.revision => ( + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + reason: format!( + "the marker records revision {} but the metastore has revision {}", + m.record.revision, record.revision + ), + marker: Some(m.record), + }, + MarkerStatus::Conflicting, + ), + Some(Ok(m)) if m.record.revision == record.revision && m.record != record => ( + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + reason: format!( + "the marker and the metastore disagree at revision {}", + record.revision + ), + marker: Some(m.record), + }, + MarkerStatus::Conflicting, + ), + Some(Ok(m)) if m.record == record => (StoredFence::Record(record), MarkerStatus::Current), + // Missing, older, or unreadable while the metastore has a well-formed record: the + // metastore is authoritative and the marker is rewritten. + _ => (StoredFence::Record(record), MarkerStatus::Stale), + }) +} + +/// Look up the receipt for `(namespace, operation_id, command_id)`. +pub fn read_receipt( + conn: &rusqlite::Connection, + namespace: &NamespaceName, + operation_id: Uuid, + command_id: Uuid, +) -> Result>, rusqlite::Error> { + let row: Option<(i64, Vec)> = conn + .query_row( + "SELECT format_version, receipt FROM namespace_fence_receipts + WHERE namespace = ?1 AND operation_id = ?2 AND command_id = ?3", + params![ + namespace.as_str(), + operation_id.to_string(), + command_id.to_string() + ], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + Ok(row.map(|(v, bytes)| decode_receipt(v, &bytes, namespace, operation_id, command_id))) +} + +fn decode_receipt( + format_version: i64, + bytes: &[u8], + namespace: &NamespaceName, + operation_id: Uuid, + command_id: Uuid, +) -> Result { + let format_version = u32::try_from(format_version) + .map_err(|_| FenceDecodeError::UnsupportedFormatVersion(u32::MAX))?; + let receipt = CommandReceipt::decode(format_version, bytes)?; + if receipt.namespace != *namespace + || receipt.operation_id != operation_id + || receipt.command_id != command_id + { + return Err(FenceDecodeError::Invalid("receipt disagrees with its key")); + } + Ok(receipt) +} + +/// One stored receipt, as read for inspection. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoredReceipt { + pub operation_id: String, + pub command_id: String, + pub revision_after: i64, + pub applied_at_ms: i64, + pub receipt: Result, +} + +/// Every receipt of `namespace`, oldest first. +pub fn read_receipts( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result> { + let mut stmt = conn.prepare( + "SELECT operation_id, command_id, format_version, revision_after, applied_at, receipt + FROM namespace_fence_receipts WHERE namespace = ?1 + ORDER BY applied_at, revision_after, operation_id, command_id", + )?; + let rows = stmt.query_map([namespace.as_str()], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, i64>(4)?, + row.get::<_, Vec>(5)?, + )) + })?; + let mut out = Vec::new(); + for row in rows { + let (op, cmd, version, revision_after, applied_at, bytes) = row?; + let receipt = match (Uuid::parse_str(&op), Uuid::parse_str(&cmd)) { + (Ok(op_id), Ok(cmd_id)) => decode_receipt(version, &bytes, namespace, op_id, cmd_id), + _ => Err(FenceDecodeError::Invalid("receipt key is not a uuid")), + }; + out.push(StoredReceipt { + operation_id: op, + command_id: cmd, + revision_after, + applied_at_ms: applied_at, + receipt, + }); + } + Ok(out) +} + +/// Compare-and-swap the fence row of `record.namespace`: it must currently have revision +/// `previous` (`None`: no row). The caller holds the write lock, so this only fails if the +/// caller read something other than what is stored, which is a bug; it is still checked. +pub fn write_record( + conn: &rusqlite::Connection, + record: &NamespaceFenceRecord, + previous: Option, +) -> Result<(), FenceStoreError> { + let revision = i64::try_from(record.revision).expect("revision fits in i64"); + let bytes = record.encode(); + let changed = match previous { + None => conn.execute( + "INSERT INTO namespace_fences (namespace, format_version, revision, record) + VALUES (?1, ?2, ?3, ?4)", + params![ + record.namespace.as_str(), + FENCE_FORMAT_VERSION, + revision, + bytes + ], + )?, + Some(previous) => conn.execute( + "UPDATE namespace_fences SET format_version = ?2, revision = ?3, record = ?4 + WHERE namespace = ?1 AND revision = ?5", + params![ + record.namespace.as_str(), + FENCE_FORMAT_VERSION, + revision, + bytes, + previous + ], + )?, + }; + if changed != 1 { + return Err(FenceError::new( + FenceOutcome::FenceRevisionMismatch, + "the stored fence record changed under the transition", + ) + .into()); + } + Ok(()) +} + +/// Store `receipt`, replacing any receipt with the same key (a finished drain replaces its +/// `DRAINING` receipt). +pub fn write_receipt( + conn: &rusqlite::Connection, + receipt: &CommandReceipt, +) -> rusqlite::Result<()> { + conn.execute( + "INSERT OR REPLACE INTO namespace_fence_receipts + (namespace, operation_id, command_id, format_version, revision_after, applied_at, receipt) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + receipt.namespace.as_str(), + receipt.operation_id.to_string(), + receipt.command_id.to_string(), + FENCE_FORMAT_VERSION, + i64::try_from(receipt.revision_after).expect("revision fits in i64"), + receipt.applied_at_ms, + receipt.encode(), + ], + )?; + Ok(()) +} + +/// Prune receipts of operations other than `owner` that are older than `retention` at +/// `now_ms` (section 5.5). The owner's receipts are never pruned. +pub fn prune_receipts( + conn: &rusqlite::Connection, + namespace: &NamespaceName, + owner: Uuid, + now_ms: i64, + retention: Duration, +) -> rusqlite::Result { + let cutoff = now_ms.saturating_sub(i64::try_from(retention.as_millis()).unwrap_or(i64::MAX)); + conn.execute( + "DELETE FROM namespace_fence_receipts + WHERE namespace = ?1 AND operation_id != ?2 AND applied_at < ?3", + params![namespace.as_str(), owner.to_string(), cutoff], + ) +} + +/// Remove every trace of `namespace`'s fence, for a delete of a namespace whose fence permits +/// it. Returns the number of receipts removed. +pub fn delete_fence( + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> rusqlite::Result { + conn.execute( + "DELETE FROM namespace_fences WHERE namespace = ?1", + [namespace.as_str()], + )?; + conn.execute( + "DELETE FROM namespace_fence_receipts WHERE namespace = ?1", + [namespace.as_str()], + ) +} + +/// The config as it is stored while `blocks` is the legacy mirror: `config` with its +/// `block_*` fields replaced (section 13.2). +pub fn with_legacy_blocks(config: &DatabaseConfig, blocks: &LegacyBlocks) -> DatabaseConfig { + DatabaseConfig { + block_reads: blocks.block_reads, + block_writes: blocks.block_writes, + block_reason: blocks.block_reason.clone(), + ..config.clone() + } +} + +/// The `block_*` fields of `config`. +pub fn legacy_blocks_of(config: &DatabaseConfig) -> LegacyBlocks { + LegacyBlocks { + block_reads: config.block_reads, + block_writes: config.block_writes, + block_reason: config.block_reason.clone(), + } +} + +/// Upsert the config row of `namespace`. +pub fn write_config_row( + conn: &rusqlite::Connection, + namespace: &NamespaceName, + config: &DatabaseConfig, +) -> rusqlite::Result<()> { + let encoded = metadata::DatabaseConfig::from(config).encode_to_vec(); + conn.execute( + "INSERT INTO namespace_configs (namespace, config) VALUES (?1, ?2) + ON CONFLICT(namespace) DO UPDATE SET config = excluded.config", + params![namespace.as_str(), encoded], + )?; + Ok(()) +} + +/// The namespace config a target is created with, before the legacy mirror is applied. +pub fn target_database_config(config: &TargetConfig) -> Result { + let invalid = |message: String| { + FenceError::new(FenceOutcome::FencePreconditionFailed, message) + .with_detail(FenceDetail::InvalidArgument) + }; + let mut out = DatabaseConfig::default(); + if let Some(bytes) = config.max_db_size { + out.max_db_pages = bytes / LIBSQL_PAGE_SIZE; + } + out.jwt_key = config.jwt_key.clone(); + if let Some(s) = config.txn_timeout_s { + out.txn_timeout = Some(Duration::from_secs(s)); + } + out.allow_attach = config.allow_attach; + if let Some(mode) = &config.durability_mode { + out.durability_mode = mode + .parse::() + .map_err(|()| invalid(format!("unknown durability mode `{mode}`")))?; + } + out.bottomless_db_id = config.bottomless_db_id.clone(); + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::namespace::fence::record::tests as samples; + + fn conn() -> rusqlite::Connection { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + conn.execute("PRAGMA foreign_keys=ON", ()).unwrap(); + conn.execute( + "CREATE TABLE namespace_configs (namespace TEXT NOT NULL PRIMARY KEY, config BLOB NOT NULL)", + (), + ) + .unwrap(); + create_tables(&conn).unwrap(); + conn + } + + fn sample() -> NamespaceFenceRecord { + samples::sample_record() + } + + #[test] + fn tables_are_additive_and_detectable() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + assert!(!tables_exist(&conn).unwrap()); + create_tables(&conn).unwrap(); + assert!(tables_exist(&conn).unwrap()); + // Idempotent. + create_tables(&conn).unwrap(); + } + + #[test] + fn record_round_trips_and_cas_checks_revision() { + let dir = tempfile::tempdir().unwrap(); + let conn = conn(); + let record = sample(); + write_config_row(&conn, &record.namespace, &DatabaseConfig::default()).unwrap(); + write_record(&conn, &record, None).unwrap(); + // A second insert, or an update from the wrong revision, is refused. + assert!(write_record(&conn, &record, None).is_err()); + let mut next = record.clone(); + next.revision += 1; + let err = write_record(&conn, &next, Some(1)).unwrap_err(); + assert!( + matches!(err, FenceStoreError::Fence(e) if e.outcome() == FenceOutcome::FenceRevisionMismatch) + ); + write_record(&conn, &next, Some(2)).unwrap(); + + let (stored, marker) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert_eq!(stored, StoredFence::Record(next)); + assert_eq!(marker, MarkerStatus::Stale); + } + + #[test] + fn fence_row_needs_a_config_row() { + let conn = conn(); + let err = write_record(&conn, &sample(), None).unwrap_err(); + assert!(matches!(err, FenceStoreError::Sqlite(_)), "{err:?}"); + } + + #[test] + fn delete_of_config_row_is_restricted_by_the_fence_row() { + let conn = conn(); + let record = sample(); + write_config_row(&conn, &record.namespace, &DatabaseConfig::default()).unwrap(); + write_record(&conn, &record, None).unwrap(); + assert!(conn + .execute( + "DELETE FROM namespace_configs WHERE namespace = ?1", + [record.namespace.as_str()] + ) + .is_err()); + delete_fence(&conn, &record.namespace).unwrap(); + conn.execute( + "DELETE FROM namespace_configs WHERE namespace = ?1", + [record.namespace.as_str()], + ) + .unwrap(); + } + + #[test] + fn marker_round_trips_and_is_compared() { + let dir = tempfile::tempdir().unwrap(); + let conn = conn(); + let record = sample(); + write_config_row(&conn, &record.namespace, &DatabaseConfig::default()).unwrap(); + write_record(&conn, &record, None).unwrap(); + write_marker(dir.path(), &record).unwrap(); + assert!(!dir + .path() + .join(record.namespace.as_str()) + .join(".fence.tmp") + .exists()); + let (stored, status) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert_eq!(stored, StoredFence::Record(record.clone())); + assert_eq!(status, MarkerStatus::Current); + + // A marker ahead of the metastore: the metastore went backwards. + let mut ahead = record.clone(); + ahead.revision += 1; + write_marker(dir.path(), &ahead).unwrap(); + let (stored, status) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert!(matches!( + stored, + StoredFence::Unavailable { detail: FenceDetail::MetastoreBehindMarker, marker: Some(ref m), .. } if *m == ahead + )); + assert_eq!(status, MarkerStatus::Conflicting); + + // Same revision, different contents. + let mut forked = record.clone(); + forked.operation_id = Uuid::from_u128(77); + write_marker(dir.path(), &forked).unwrap(); + let (stored, _) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert_eq!(stored.state(), FenceState::UnknownUnavailable); + + // An undecodable marker beside a good record: the record wins and the marker is stale. + fs::write(marker_path(dir.path(), &record.namespace), b"garbage").unwrap(); + let (stored, status) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert_eq!(stored, StoredFence::Record(record.clone())); + assert_eq!(status, MarkerStatus::Stale); + + // Marker without a row. + delete_fence(&conn, &record.namespace).unwrap(); + write_marker(dir.path(), &record).unwrap(); + let (stored, _) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert!(matches!( + stored, + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + .. + } + )); + fs::write(marker_path(dir.path(), &record.namespace), b"garbage").unwrap(); + let (stored, _) = read_fence(&conn, dir.path(), &record.namespace).unwrap(); + assert!(matches!( + stored, + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + marker: None, + .. + } + )); + } + + #[test] + fn corrupt_rows_are_unavailable() { + let dir = tempfile::tempdir().unwrap(); + let conn = conn(); + let record = sample(); + let ns = record.namespace.clone(); + write_config_row(&conn, &ns, &DatabaseConfig::default()).unwrap(); + write_record(&conn, &record, None).unwrap(); + + let set = |sql: &str| { + conn.execute(sql, [ns.as_str()]).unwrap(); + }; + let detail = || match read_fence(&conn, dir.path(), &ns).unwrap().0 { + StoredFence::Unavailable { detail, .. } => detail, + other => panic!("expected unavailable, got {other:?}"), + }; + + set("UPDATE namespace_fences SET format_version = 2 WHERE namespace = ?1"); + assert_eq!(detail(), FenceDetail::UnsupportedFormatVersion); + set("UPDATE namespace_fences SET format_version = 1, revision = 3 WHERE namespace = ?1"); + assert_eq!(detail(), FenceDetail::CorruptRecord); + set("UPDATE namespace_fences SET revision = 2, record = x'ffff' WHERE namespace = ?1"); + assert_eq!(detail(), FenceDetail::CorruptRecord); + set("UPDATE namespace_fences SET revision = -1 WHERE namespace = ?1"); + assert_eq!(detail(), FenceDetail::CorruptRecord); + + let stored = read_fence(&conn, dir.path(), &ns).unwrap().0; + for class in OperationClass::ALL { + let r = stored.permits(class); + match class { + OperationClass::Maintenance | OperationClass::Observability => assert!(r.is_ok()), + _ => { + let e = r.unwrap_err(); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + assert_eq!(e.detail(), Some(FenceDetail::CorruptRecord)); + } + } + } + } + + #[test] + fn receipts_round_trip_and_prune() { + let conn = conn(); + let record = sample(); + let ns = record.namespace.clone(); + let base = samples::sample_receipt(); + let owner = base.operation_id; + let other = Uuid::from_u128(0xbeef); + + let mut r_owner_old = base.clone(); + r_owner_old.applied_at_ms = 10; + let mut r_other_old = base.clone(); + r_other_old.operation_id = other; + r_other_old.applied_at_ms = 10; + let mut r_other_new = base.clone(); + r_other_new.operation_id = other; + r_other_new.command_id = Uuid::from_u128(0xc0de); + r_other_new.applied_at_ms = 5_000; + for r in [&r_owner_old, &r_other_old, &r_other_new] { + write_receipt(&conn, r).unwrap(); + } + assert_eq!( + read_receipt(&conn, &ns, owner, base.command_id) + .unwrap() + .unwrap() + .unwrap(), + r_owner_old + ); + assert!(read_receipt(&conn, &ns, owner, Uuid::from_u128(1234)) + .unwrap() + .is_none()); + + // Retention 1s at t=6s: only the other operation's old receipt goes. + let pruned = prune_receipts(&conn, &ns, owner, 6_000, Duration::from_secs(1)).unwrap(); + assert_eq!(pruned, 1); + let left: Vec<_> = read_receipts(&conn, &ns) + .unwrap() + .into_iter() + .map(|r| r.receipt.unwrap()) + .collect(); + assert_eq!(left, vec![r_owner_old.clone(), r_other_new]); + + // A receipt stored under the wrong key reads as corrupt. + conn.execute( + "UPDATE namespace_fence_receipts SET command_id = ?1 WHERE operation_id = ?2", + params![Uuid::from_u128(4321).to_string(), owner.to_string()], + ) + .unwrap(); + assert!(read_receipt(&conn, &ns, owner, Uuid::from_u128(4321)) + .unwrap() + .unwrap() + .is_err()); + } + + #[test] + fn target_config_conversion() { + let c = target_database_config(&TargetConfig { + max_db_size: Some(4096 * 10), + jwt_key: Some("k".into()), + txn_timeout_s: Some(7), + allow_attach: true, + durability_mode: Some("strong".into()), + bottomless_db_id: Some("b".into()), + }) + .unwrap(); + assert_eq!(c.max_db_pages, 10); + assert_eq!(c.jwt_key.as_deref(), Some("k")); + assert_eq!(c.txn_timeout, Some(Duration::from_secs(7))); + assert!(c.allow_attach); + assert_eq!(c.durability_mode, DurabilityMode::Strong); + assert_eq!(c.bottomless_db_id.as_deref(), Some("b")); + assert!(!c.block_reads && !c.block_writes); + + let e = target_database_config(&TargetConfig { + durability_mode: Some("nope".into()), + ..Default::default() + }) + .unwrap_err(); + assert_eq!(e.detail(), Some(FenceDetail::InvalidArgument)); + } +} diff --git a/libsql-server/src/namespace/meta_store.rs b/libsql-server/src/namespace/meta_store.rs index 70b419ebe9..098102e3f7 100644 --- a/libsql-server/src/namespace/meta_store.rs +++ b/libsql-server/src/namespace/meta_store.rs @@ -1,6 +1,7 @@ #![allow(clippy::mutable_key_type)] -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; use std::{collections::HashMap, fs::read_dir}; use bottomless::bottomless_wal::BottomlessWalWrapper; @@ -14,11 +15,13 @@ use libsql_sys::wal::{ }; use parking_lot::Mutex; use prost::Message; +use rusqlite::TransactionBehavior; use tokio::sync::oneshot; use tokio::sync::{ mpsc, watch::{self, Receiver, Sender}, }; +use uuid::Uuid; use crate::config::BottomlessConfig; use crate::connection::config::DatabaseConfig; @@ -28,6 +31,16 @@ use crate::{ config::MetaStoreConfig, connection::legacy::open_conn_active_checkpoint, error::Error, Result, }; +use super::fence::command::{FenceCommand, FenceRequest}; +use super::fence::outcome::{FenceDetail, FenceError, FenceOutcome}; +use super::fence::record::{ + CommandReceipt, NamespaceFenceRecord, ServerIdentity, ValidationSnapshot, +}; +use super::fence::state::OperationClass; +use super::fence::store::{ + self as fence_store, FenceStoreError, MarkerStatus, StoredFence, StoredReceipt, +}; +use super::fence::transition::{self, ApplyEnv, Decision, DrainCompletion}; use super::NamespaceName; type ChangeMsg = ( @@ -75,6 +88,19 @@ struct MetaStoreInner { conn: tokio::sync::Mutex, wal_manager: MetaStoreWalManager, db_kind: DatabaseKind, + /// `/dbs`, where namespace directories and their fence markers are. + dbs_path: PathBuf, + fence: FenceSettings, +} + +/// How this metastore treats namespace fences (`docs/NAMESPACE_FENCE.md` section 13.1). +#[derive(Debug, Clone, Copy)] +struct FenceSettings { + /// The fence may be used: its tables exist and commands are accepted. + enabled: bool, + /// The fence tables exist, so fence state is loaded and enforced. + tables: bool, + receipt_retention: Duration, } fn setup_connection(conn: &rusqlite::Connection) -> Result<()> { @@ -187,12 +213,24 @@ impl MetaStoreInner { db_kind: DatabaseKind, ) -> Result { setup_connection(&conn)?; + if config.namespace_fence { + fence_store::create_tables(&conn)?; + } + let fence = FenceSettings { + enabled: config.namespace_fence, + tables: fence_store::tables_exist(&conn)?, + receipt_retention: config + .namespace_fence_receipt_retention + .unwrap_or(fence_store::DEFAULT_RECEIPT_RETENTION), + }; let mut this = MetaStoreInner { configs: Default::default(), conn: conn.into(), wal_manager, db_kind, + dbs_path: base_path.join("dbs"), + fence, }; if config.allow_recover_from_fs { @@ -200,6 +238,9 @@ impl MetaStoreInner { } this.restore()?; + if this.fence.tables { + this.restore_fences()?; + } Ok(this) } @@ -301,6 +342,53 @@ impl MetaStoreInner { Ok(()) } + + /// Load every namespace's fence after the configs (section 5.6). The stored config row of + /// a fenced namespace carries the legacy mirror of the fence in its `block_*` fields + /// (section 13.2); the in-memory config is the namespace's own configuration, so those + /// fields are put back to the values the record saved. A marker that fell behind its + /// record is rewritten. A namespace whose fence cannot be established is logged and keeps + /// its stored config, mirror included. + fn restore_fences(&mut self) -> Result<()> { + let namespaces: Vec = self.configs.get_mut().keys().cloned().collect(); + let conn = self.conn.get_mut(); + let mut fenced = 0usize; + for ns in namespaces { + let (stored, marker) = match fence_store::read_fence(conn, &self.dbs_path, &ns) { + Ok(r) => r, + Err(FenceStoreError::Sqlite(e)) => return Err(e.into()), + Err(e) => { + tracing::error!(namespace = %ns, "cannot establish namespace fence: {e}"); + continue; + } + }; + match &stored { + StoredFence::None { .. } => continue, + StoredFence::Record(record) => { + fenced += 1; + if marker == MarkerStatus::Stale { + if let Err(e) = fence_store::write_marker(&self.dbs_path, record) { + tracing::error!(namespace = %ns, "failed to rewrite fence marker: {e}"); + } + } + let sender = self.configs.get_mut().get_mut(&ns).expect("listed above"); + let config = sender.borrow().config.clone(); + let config = fence_store::with_legacy_blocks(&config, &record.legacy_blocks); + sender.send_modify(|c| c.config = Arc::new(config)); + } + StoredFence::Unavailable { detail, reason, .. } => { + fenced += 1; + tracing::error!( + namespace = %ns, + %detail, + "namespace fence state is UNKNOWN_UNAVAILABLE: {reason}" + ); + } + } + } + tracing::info!("loaded {fenced} namespace fence(s)"); + Ok(()) + } } /// Handles config change updates by inserting them into the database and in-memory @@ -313,6 +401,11 @@ fn process(msg: ChangeMsg, inner: Arc) { } else { Ok(()) }; + // A config that was not persisted is not published. + if ret.is_err() { + let _ = ret_chan.send(ret); + return; + } let mut configs = inner.configs.blocking_lock(); if let Some(config_watch) = configs.get_mut(&namespace) { let new_version = config_watch.borrow().version.wrapping_add(1); @@ -349,22 +442,22 @@ fn try_process( namespace: &NamespaceName, config: &DatabaseConfig, ) -> Result<()> { - let config_encoded = metadata::DatabaseConfig::from(&*config).encode_to_vec(); - let mut conn = inner.conn.blocking_lock(); + // `BEGIN IMMEDIATE`: the write lock is what serialises this write with fence transitions + // (docs/NAMESPACE_FENCE.md section 5.4), including those of other metastore connections. + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + if inner.fence.tables { + let (stored, _) = + fence_store::read_fence(&tx, &inner.dbs_path, namespace).map_err(fence_store_error)?; + stored.permits(OperationClass::Lifecycle)?; + } if let Some(schema) = config.shared_schema_name.as_ref() { - let tx = conn.transaction()?; if inner.db_kind.is_primary() { - if let Some(ref schema) = config.shared_schema_name { - if crate::schema::db::has_pending_migration_jobs(&tx, schema)? { - return Err(crate::Error::PendingMigrationOnSchema(schema.clone())); - } + if crate::schema::db::has_pending_migration_jobs(&tx, schema)? { + return Err(crate::Error::PendingMigrationOnSchema(schema.clone())); } } - tx.execute( - "INSERT INTO namespace_configs (namespace, config) VALUES (?1, ?2) ON CONFLICT(namespace) DO UPDATE SET config=excluded.config", - rusqlite::params![namespace.as_str(), config_encoded], - )?; + fence_store::write_config_row(&tx, namespace, config)?; tx.execute( "DELETE FROM shared_schema_links WHERE namespace = ?", rusqlite::params![namespace.as_str()], @@ -373,13 +466,10 @@ fn try_process( "INSERT OR REPLACE INTO shared_schema_links (shared_schema_name, namespace) VALUES (?1, ?2)", rusqlite::params![schema.as_str(), namespace.as_str()], )?; - tx.commit()?; } else { - conn.execute( - "INSERT INTO namespace_configs (namespace, config) VALUES (?1, ?2) ON CONFLICT(namespace) DO UPDATE SET config=excluded.config", - rusqlite::params![namespace.as_str(), config_encoded], - )?; + fence_store::write_config_row(&tx, namespace, config)?; } + tx.commit()?; if let Err(e) = checkpoint(&conn) { tracing::warn!("failed to checkpoint metastore: {e}"); @@ -388,11 +478,335 @@ fn try_process( Ok(()) } +fn fence_store_error(e: FenceStoreError) -> Error { + match e { + FenceStoreError::Fence(e) => Error::NamespaceFence(e), + FenceStoreError::Sqlite(e) => Error::RusqliteError(e), + FenceStoreError::Io(e) => Error::IOError(e), + } +} + fn checkpoint(conn: &rusqlite::Connection) -> Result<()> { conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", (), |_| Ok(()))?; Ok(()) } +/// Facts about the server and the live namespace that a fence command needs and the metastore +/// does not hold (see [`ApplyEnv`]). The store adds what it reads inside the transaction. +#[derive(Debug, Clone)] +pub struct FenceContext { + pub server: ServerIdentity, + /// Wall-clock time in milliseconds since the Unix epoch. + pub now_ms: i64, + /// The namespace's current replication log id, if it exists and has one. + pub namespace_log_id: Option, + /// A fresh id for `CreateTargetQuarantined`. + pub new_incarnation_id: Uuid, + /// Whether the request carried the configured adoption key. + pub adoption_authorised: bool, + /// What the server observed of a target, for `RecordTargetValidation`. + pub validation_snapshot: Option, +} + +impl FenceContext { + /// A context for `server` at the current time with a fresh incarnation id. + pub fn now(server: ServerIdentity, namespace_log_id: Option) -> Self { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX)); + Self { + server, + now_ms, + namespace_log_id, + new_incarnation_id: Uuid::new_v4(), + adoption_authorised: false, + validation_snapshot: None, + } + } +} + +/// How a fence command was answered. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FenceCommitKind { + /// The command had been applied before; nothing was written. + Replayed, + /// The command started a drain that is still to be completed; nothing was written and the + /// controller resumes the drain. + Resumed, + /// The command's receipt (and, where it changed, the record) was committed. + Committed, +} + +/// The committed result of a fence command. +#[derive(Debug, Clone)] +pub struct FenceCommit { + pub kind: FenceCommitKind, + pub receipt: CommandReceipt, + /// The namespace's fence record after the command (for a replay, as it is now). + pub record: Option, + /// For a `CreateTargetQuarantined` that was committed, the namespace config it created. + /// It is not in the in-memory config map yet: the caller installs the target's gate first + /// and then publishes it. + pub created_config: Option>, +} + +/// A namespace's fence as read by `inspect_fence`. +#[derive(Debug, Clone)] +pub struct FenceInspection { + pub fence: StoredFence, + pub receipts: Vec, +} + +fn fence_disabled() -> FenceError { + FenceError::new( + FenceOutcome::FencePreconditionFailed, + "namespace fences are not enabled on this server", + ) + .with_detail(FenceDetail::FenceDisabled) +} + +fn not_primary() -> FenceError { + FenceError::new( + FenceOutcome::FencePreconditionFailed, + "namespace fences are only changed on a primary", + ) + .with_detail(FenceDetail::NotPrimary) +} + +fn unavailable_receipt(e: impl std::fmt::Display) -> FenceError { + FenceError::new( + FenceOutcome::FenceStateUnavailable, + format!("the stored receipt for this command cannot be read: {e}"), + ) + .with_detail(FenceDetail::CorruptRecord) +} + +/// The `ApplyEnv` for a command: the caller's context plus what the transaction read. +fn apply_env( + ctx: &FenceContext, + stored: &StoredFence, + config: Option<&DatabaseConfig>, +) -> ApplyEnv { + ApplyEnv { + now_ms: ctx.now_ms, + server: ctx.server.clone(), + namespace_log_id: ctx.namespace_log_id, + shared_schema: config.is_some_and(|c| c.is_shared_schema || c.shared_schema_name.is_some()), + // The namespace's own values: a stored record's saved values while it is in force, + // otherwise the config row as stored. + legacy_blocks: match stored { + StoredFence::Record(r) if !r.state.is_operation_finished() => r.legacy_blocks.clone(), + _ => config + .map(fence_store::legacy_blocks_of) + .unwrap_or_default(), + }, + new_incarnation_id: ctx.new_incarnation_id, + adoption_authorised: ctx.adoption_authorised, + validation_snapshot: ctx.validation_snapshot, + } +} + +fn apply_fence_command( + inner: &MetaStoreInner, + request: &FenceRequest, + ctx: &FenceContext, +) -> std::result::Result { + if !inner.fence.enabled || !inner.fence.tables { + return Err(fence_disabled().into()); + } + if !inner.db_kind.is_primary() { + return Err(not_primary().into()); + } + let ns = &request.namespace; + let mut conn = inner.conn.blocking_lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + + let (stored, marker) = fence_store::read_fence(&tx, &inner.dbs_path, ns)?; + let existing = + match fence_store::read_receipt(&tx, ns, request.operation_id, request.command_id)? { + None => None, + Some(Ok(r)) => Some(r), + Some(Err(e)) => return Err(unavailable_receipt(e).into()), + }; + let config = fence_store::read_config_row(&tx, ns)?; + let env = apply_env(ctx, &stored, config.as_ref()); + + let decision = transition::apply(stored.as_current(), existing.as_ref(), request, &env)?; + let (record, receipt) = match decision { + Decision::Replay(receipt) | Decision::Resume(receipt) => { + let kind = if receipt.is_final() { + FenceCommitKind::Replayed + } else { + FenceCommitKind::Resumed + }; + return Ok(FenceCommit { + kind, + receipt, + record: stored.record().cloned(), + created_config: None, + }); + } + Decision::Apply { record, receipt } => (record, receipt), + }; + + let mut created_config = None; + if let Some(next) = &record { + let previous = fence_store::stored_revision(&tx, ns)?; + if let FenceCommand::CreateTargetQuarantined { config: target } = &request.command { + // Section 10.1: the marker first, then the config row, the record and the receipt + // in one transaction. A crash in between leaves a marker without rows, which only a + // replay of this command completes. + let logical = fence_store::target_database_config(target)?; + fence_store::write_marker(&inner.dbs_path, next)?; + fence_store::write_config_row( + &tx, + ns, + &fence_store::with_legacy_blocks(&logical, &next.legacy_mirror()), + )?; + created_config = Some(Arc::new(logical)); + } else { + let Some(config) = &config else { + return Err(FenceError::new( + FenceOutcome::FenceStateUnavailable, + "the fenced namespace has no config row", + ) + .with_detail(FenceDetail::CorruptRecord) + .into()); + }; + fence_store::write_config_row( + &tx, + ns, + &fence_store::with_legacy_blocks(config, &next.legacy_mirror()), + )?; + } + fence_store::write_record(&tx, next, previous)?; + } + fence_store::write_receipt(&tx, &receipt)?; + let owner = record + .as_ref() + .or(stored.record()) + .map_or(request.operation_id, |r| r.operation_id); + fence_store::prune_receipts(&tx, ns, owner, ctx.now_ms, inner.fence.receipt_retention)?; + tx.commit()?; + + let current = record.or_else(|| stored.record().cloned()); + after_fence_commit( + inner, + &conn, + current.as_ref(), + record_changed(¤t, &stored, marker), + ); + + Ok(FenceCommit { + kind: FenceCommitKind::Committed, + receipt, + record: current, + created_config, + }) +} + +/// Whether the marker has to be written after a commit: the record changed, or it had fallen +/// behind. +fn record_changed( + current: &Option, + stored: &StoredFence, + marker: MarkerStatus, +) -> bool { + current.as_ref() != stored.record() || marker == MarkerStatus::Stale +} + +fn after_fence_commit( + inner: &MetaStoreInner, + conn: &rusqlite::Connection, + record: Option<&NamespaceFenceRecord>, + write_marker: bool, +) { + if let (Some(record), true) = (record, write_marker) { + // The metastore is authoritative; a marker that is missing or behind is repaired on + // the next load (section 5.6). + if let Err(e) = fence_store::write_marker(&inner.dbs_path, record) { + tracing::error!(namespace = %record.namespace, "failed to write fence marker: {e}"); + } + } + if let Err(e) = checkpoint(conn) { + tracing::warn!("failed to checkpoint metastore: {e}"); + } +} + +fn complete_fence_drain( + inner: &MetaStoreInner, + ns: &NamespaceName, + operation_id: Uuid, + command_id: Uuid, + completion: DrainCompletion, + ctx: &FenceContext, +) -> std::result::Result { + if !inner.fence.tables { + return Err(fence_disabled().into()); + } + let mut conn = inner.conn.blocking_lock(); + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + + let (stored, _) = fence_store::read_fence(&tx, &inner.dbs_path, ns)?; + let record = match &stored { + StoredFence::Record(r) => r.clone(), + other => { + return Err(other + .permits(OperationClass::NormalWrite) + .err() + .unwrap_or_else(|| { + FenceError::new( + FenceOutcome::InvalidFenceTransition, + "the namespace has no fence record", + ) + }) + .into()) + } + }; + let receipt = match fence_store::read_receipt(&tx, ns, operation_id, command_id)? { + Some(Ok(r)) => r, + Some(Err(e)) => return Err(unavailable_receipt(e).into()), + None => { + return Err(FenceError::new( + FenceOutcome::InvalidFenceTransition, + format!("command {command_id} of operation {operation_id} has no receipt"), + ) + .into()) + } + }; + if receipt.is_final() { + return Ok(FenceCommit { + kind: FenceCommitKind::Replayed, + receipt, + record: Some(record), + created_config: None, + }); + } + + let config = fence_store::read_config_row(&tx, ns)?; + let env = apply_env(ctx, &stored, config.as_ref()); + let (next, final_receipt) = transition::complete_drain(&record, &receipt, completion, &env)?; + if let Some(config) = &config { + fence_store::write_config_row( + &tx, + ns, + &fence_store::with_legacy_blocks(config, &next.legacy_mirror()), + )?; + } + fence_store::write_record(&tx, &next, fence_store::stored_revision(&tx, ns)?)?; + fence_store::write_receipt(&tx, &final_receipt)?; + tx.commit()?; + + after_fence_commit(inner, &conn, Some(&next), true); + + Ok(FenceCommit { + kind: FenceCommitKind::Committed, + receipt: final_receipt, + record: Some(next), + created_config: None, + }) +} + impl MetaStore { #[tracing::instrument(skip(config, base_path, conn, wal_manager))] pub async fn new( @@ -523,7 +937,26 @@ impl MetaStore { let r = if let Some(sender) = configs.get(&namespace) { tracing::debug!("removed namespace `{}` from meta store", namespace); let config = sender.borrow().clone(); - let tx = conn.transaction()?; + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + if self.inner.fence.tables { + let (stored, _) = fence_store::read_fence(&tx, &self.inner.dbs_path, &namespace) + .map_err(fence_store_error)?; + stored.permits(OperationClass::Lifecycle)?; + if !matches!(stored, StoredFence::None { .. }) { + // The marker goes before the commit: a crash in between leaves a record + // without a marker, which is repaired on load, rather than a marker + // without a record, which would make the name unavailable. + fence_store::remove_marker(&self.inner.dbs_path, &namespace)?; + let receipts = fence_store::delete_fence(&tx, &namespace)?; + tracing::info!( + namespace = %namespace, + state = %stored.state(), + revision = stored.revision(), + receipts, + "removing namespace fence with its namespace" + ); + } + } if config.config.is_shared_schema { if crate::schema::db::schema_has_linked_dbs(&tx, &namespace)? { return Err(crate::Error::HasLinkedDbs(namespace.clone())); @@ -562,6 +995,115 @@ impl MetaStore { self.inner.configs.lock().await.contains_key(namespace) } + /// Whether namespace fences may be used on this server. + pub fn fence_enabled(&self) -> bool { + self.inner.fence.enabled + } + + /// Whether this metastore holds fence state, so fences are loaded and enforced. + pub fn fence_enforced(&self) -> bool { + self.inner.fence.tables + } + + /// Run one fence command as a compare-and-swap in a single metastore transaction + /// (`docs/NAMESPACE_FENCE.md` sections 5.3 and 5.4). Nothing is published here: the + /// caller publishes the result only after this returns, which is after the commit. + /// + /// A fence outcome that is an error (`FENCE_REVISION_MISMATCH`, …) is returned as + /// [`Error::NamespaceFence`] and nothing is written. + pub async fn apply_fence_command( + &self, + request: FenceRequest, + ctx: FenceContext, + ) -> Result { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || apply_fence_command(&inner, &request, &ctx)) + .await? + .map_err(fence_store_error) + } + + /// Finish the drain that the owning operation's `DRAINING` receipt + /// `(operation_id, command_id)` started, once the controller has proven `completion`. The + /// final receipt replaces the `DRAINING` one. If the drain was already completed, the + /// final receipt is returned as a replay. + pub async fn complete_fence_drain( + &self, + namespace: NamespaceName, + operation_id: Uuid, + command_id: Uuid, + completion: DrainCompletion, + ctx: FenceContext, + ) -> Result { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || { + complete_fence_drain( + &inner, + &namespace, + operation_id, + command_id, + completion, + &ctx, + ) + }) + .await? + .map_err(fence_store_error) + } + + /// Read a namespace's fence and all of its receipts (`InspectFence`). Never writes. + pub async fn inspect_fence(&self, namespace: NamespaceName) -> Result { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || -> std::result::Result<_, FenceStoreError> { + let mut conn = inner.conn.blocking_lock(); + if !inner.fence.tables { + let tx = conn.transaction()?; + let exists = fence_store::read_config_row(&tx, &namespace)?.is_some(); + return Ok(FenceInspection { + fence: StoredFence::None { + namespace_exists: exists, + }, + receipts: Vec::new(), + }); + } + let tx = conn.transaction()?; + let (fence, _) = fence_store::read_fence(&tx, &inner.dbs_path, &namespace)?; + let receipts = fence_store::read_receipts(&tx, &namespace)?; + Ok(FenceInspection { fence, receipts }) + }) + .await? + .map_err(fence_store_error) + } + + /// Every namespace with fence state, and that state. Namespaces without a record or a + /// marker are left out. + pub async fn load_fences(&self) -> Result> { + let inner = self.inner.clone(); + tokio::task::spawn_blocking(move || -> std::result::Result<_, FenceStoreError> { + if !inner.fence.tables { + return Ok(Vec::new()); + } + let mut conn = inner.conn.blocking_lock(); + let tx = conn.transaction()?; + let names: Vec = { + let mut stmt = tx.prepare("SELECT namespace FROM namespace_configs")?; + let rows = stmt.query_map((), |r| r.get::<_, String>(0))?; + rows.collect::>()? + }; + let mut out = Vec::new(); + for name in names { + let Ok(ns) = NamespaceName::from_string(name) else { + continue; + }; + let (fence, _) = fence_store::read_fence(&tx, &inner.dbs_path, &ns)?; + if !matches!(fence, StoredFence::None { .. }) { + out.push((ns, fence)); + } + } + Ok(out) + }) + .await? + .map_err(fence_store_error) + } + pub(crate) async fn shutdown(&self) -> crate::Result<()> { let replicator = self.inner.wal_manager.wrapper().as_ref(); @@ -729,3 +1271,806 @@ impl MetaStoreHandle { &self.namespace } } + +#[cfg(test)] +mod fence_tests { + use std::path::Path; + + use tempfile::tempdir; + + use super::*; + use crate::namespace::fence::command::TargetConfig; + use crate::namespace::fence::record::{FenceMarker, FrozenBoundary}; + use crate::namespace::fence::state::FenceState; + + const LOG: Uuid = Uuid::from_u128(0x10); + const INCARNATION: Uuid = Uuid::from_u128(0x20); + const OP: Uuid = Uuid::from_u128(0xa); + const OTHER_OP: Uuid = Uuid::from_u128(0xb); + + async fn open_with(dir: &Path, config: MetaStoreConfig) -> MetaStore { + let (maker, manager) = metastore_connection_maker(None, dir).await.unwrap(); + let conn = maker().unwrap(); + MetaStore::new(config, dir, conn, manager, DatabaseKind::Primary) + .await + .unwrap() + } + + async fn open(dir: &Path, fence: bool) -> MetaStore { + open_with( + dir, + MetaStoreConfig { + namespace_fence: fence, + ..Default::default() + }, + ) + .await + } + + /// A second, independent connection to the same metastore database (like the schema + /// scheduler's). + async fn raw(dir: &Path) -> MetaStoreConnection { + let (maker, _) = metastore_connection_maker(None, dir).await.unwrap(); + maker().unwrap() + } + + fn raw_config(conn: &rusqlite::Connection, ns: &str) -> DatabaseConfig { + fence_store::read_config_row(conn, &NamespaceName::from(ns.to_string().leak() as &str)) + .unwrap() + .unwrap() + } + + fn ctx(now_ms: i64) -> FenceContext { + FenceContext { + server: ServerIdentity { + build: "test".into(), + instance_id: Uuid::from_u128(0x99), + }, + now_ms, + namespace_log_id: Some(LOG), + new_incarnation_id: INCARNATION, + adoption_authorised: false, + validation_snapshot: None, + } + } + + fn request( + ns: &'static str, + op: Uuid, + command_id: u128, + expected_state: FenceState, + expected_revision: u64, + command: FenceCommand, + ) -> FenceRequest { + FenceRequest { + namespace: ns.into(), + operation_id: op, + command_id: Uuid::from_u128(command_id), + expected_state, + expected_revision, + command, + } + } + + fn acquire(ns: &'static str, op: Uuid, command_id: u128) -> FenceRequest { + request( + ns, + op, + command_id, + FenceState::Unfenced, + 0, + FenceCommand::AcquireSourceWriteFence { + expected_log_id: LOG, + drain_policy: None, + }, + ) + } + + fn outcome_of(r: Result) -> FenceOutcome { + match r { + Ok(c) => c.receipt.outcome, + Err(Error::NamespaceFence(e)) => e.outcome(), + Err(e) => panic!("unexpected error: {e}"), + } + } + + fn fence_error(r: Result) -> FenceError { + match r { + Err(Error::NamespaceFence(e)) => e, + other => panic!("expected a fence error, got {other:?}"), + } + } + + async fn create_namespace(store: &MetaStore, ns: &'static str) -> MetaStoreHandle { + let handle = store.handle(ns.into()).await; + handle + .store(DatabaseConfig { + max_db_pages: 1234, + block_reason: Some("pre-fence".into()), + ..Default::default() + }) + .await + .unwrap(); + handle + } + + fn remove_blocking(store: &MetaStore, ns: &'static str) -> Result>> { + let store = store.clone(); + std::thread::spawn(move || store.remove(ns.into())) + .join() + .unwrap() + } + + #[tokio::test] + async fn fence_cas_persists_across_restart() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + + let commit = store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + assert_eq!(commit.kind, FenceCommitKind::Committed); + assert_eq!(commit.receipt.outcome, FenceOutcome::Draining); + let record = commit.record.unwrap(); + assert_eq!( + (record.state, record.revision), + (FenceState::SourceDraining, 1) + ); + + // The stored row carries the legacy mirror; the in-memory config is untouched. + let conn = raw(dir.path()).await; + let row = raw_config(&conn, "db"); + assert!(row.block_writes && !row.block_reads); + assert!(row + .block_reason + .unwrap() + .starts_with("namespace fence: SOURCE_DRAINING")); + assert!(!handle.get().block_writes); + + let boundary = FrozenBoundary { + log_id: LOG, + frame_no: 42, + }; + let commit = store + .complete_fence_drain( + "db".into(), + OP, + Uuid::from_u128(1), + DrainCompletion::SourceWrites { boundary }, + ctx(2_000), + ) + .await + .unwrap(); + assert_eq!(commit.receipt.outcome, FenceOutcome::Applied); + let record = commit.record.unwrap(); + assert_eq!( + (record.state, record.revision), + (FenceState::SourceWriteFenced, 2) + ); + // Completing again is a replay of the final answer. + let again = store + .complete_fence_drain( + "db".into(), + OP, + Uuid::from_u128(1), + DrainCompletion::SourceWrites { boundary }, + ctx(2_500), + ) + .await + .unwrap(); + assert_eq!(again.kind, FenceCommitKind::Replayed); + + let marker = fence_store::read_marker(&dir.path().join("dbs"), &"db".into()) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(marker, FenceMarker::for_record(&record)); + + drop(handle); + drop(store); + + // Restart. + let store = open(dir.path(), true).await; + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!(inspection.fence, StoredFence::Record(record.clone())); + assert_eq!(inspection.fence.revision(), 2); + assert_eq!(record.frozen_boundary, Some(boundary)); + assert_eq!(inspection.receipts.len(), 1); + let receipt = inspection.receipts[0].receipt.clone().unwrap(); + assert_eq!( + (receipt.outcome, receipt.revision_after), + (FenceOutcome::Applied, 2) + ); + + // The in-memory config is the namespace's own, not the mirror. + let handle = store.handle("db".into()).await; + let config = handle.get(); + assert!(!config.block_writes && !config.block_reads); + assert_eq!(config.block_reason.as_deref(), Some("pre-fence")); + assert_eq!(config.max_db_pages, 1234); + + // The lost response of the first command is answered from its receipt, even though + // the revision has advanced. + let replay = store + .apply_fence_command(acquire("db", OP, 1), ctx(3_000)) + .await + .unwrap(); + assert_eq!(replay.kind, FenceCommitKind::Replayed); + assert_eq!(replay.receipt, receipt); + + let fences = store.load_fences().await.unwrap(); + assert_eq!( + fences, + vec![("db".into(), StoredFence::Record(record.clone()))] + ); + + // Release restores the legacy fields as they were before the fence. + let release = request( + "db", + OP, + 2, + FenceState::SourceWriteFenced, + 2, + FenceCommand::ReleaseSourceWriteFence, + ); + let commit = store + .apply_fence_command(release, ctx(4_000)) + .await + .unwrap(); + assert_eq!(commit.record.unwrap().revision, 3); + let row = raw_config(&conn, "db"); + assert!(!row.block_writes && !row.block_reads); + assert_eq!(row.block_reason.as_deref(), Some("pre-fence")); + assert_eq!(row.max_db_pages, 1234); + } + + #[tokio::test] + async fn flag_off_still_enforces_existing_fences() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + drop(handle); + drop(store); + + let store = open(dir.path(), false).await; + assert!(!store.fence_enabled()); + assert!(store.fence_enforced()); + let e = fence_error( + store + .apply_fence_command(acquire("db", OTHER_OP, 2), ctx(2_000)) + .await, + ); + assert_eq!(e.detail(), Some(FenceDetail::FenceDisabled)); + let handle = store.handle("db".into()).await; + let e = fence_error(handle.store(DatabaseConfig::default()).await); + assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); + assert_eq!( + store + .inspect_fence("db".into()) + .await + .unwrap() + .fence + .state(), + FenceState::SourceDraining + ); + } + + #[tokio::test] + async fn disabled_fence_creates_nothing() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), false).await; + let handle = create_namespace(&store, "db").await; + assert!(!store.fence_enforced()); + let e = fence_error( + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await, + ); + assert_eq!(e.outcome(), FenceOutcome::FencePreconditionFailed); + assert_eq!(e.detail(), Some(FenceDetail::FenceDisabled)); + let conn = raw(dir.path()).await; + assert!(!fence_store::tables_exist(&conn).unwrap()); + handle + .store(DatabaseConfig { + max_db_pages: 7, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(raw_config(&conn, "db").max_db_pages, 7); + assert_eq!( + store.inspect_fence("db".into()).await.unwrap().fence, + StoredFence::None { + namespace_exists: true + } + ); + assert!(store.load_fences().await.unwrap().is_empty()); + } + + #[tokio::test] + async fn concurrent_cas_has_exactly_one_winner() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let _handle = create_namespace(&store, "db").await; + + let attempts = (0..16u128).map(|i| { + let store = store.clone(); + async move { + store + .apply_fence_command(acquire("db", Uuid::from_u128(0x100 + i), 1), ctx(1_000)) + .await + } + }); + let outcomes: Vec<_> = futures::future::join_all(attempts) + .await + .into_iter() + .map(outcome_of) + .collect(); + assert_eq!( + outcomes + .iter() + .filter(|o| **o == FenceOutcome::Draining) + .count(), + 1, + "{outcomes:?}" + ); + assert!(outcomes.iter().all(|o| matches!( + o, + FenceOutcome::Draining | FenceOutcome::FenceOwnedByAnotherOperation + ))); + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!(inspection.fence.revision(), 1); + assert_eq!(inspection.receipts.len(), 1); + } + + #[tokio::test] + async fn fence_cas_and_config_writes_serialise() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + + // Another metastore connection holds the write lock with an uncommitted config + // change. The fence transition cannot interleave with it: it gives up on the lock + // and writes nothing. + let mut other = raw(dir.path()).await; + let tx = other + .transaction_with_behavior(TransactionBehavior::Immediate) + .unwrap(); + let mut changed = raw_config(&tx, "db"); + changed.max_db_pages = 42; + fence_store::write_config_row(&tx, &"db".into(), &changed).unwrap(); + let r = store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await; + assert!( + matches!(r, Err(Error::RusqliteError(_))), + "expected the write lock to be busy, got {r:?}" + ); + tx.commit().unwrap(); + assert_eq!( + store + .inspect_fence("db".into()) + .await + .unwrap() + .fence + .state(), + FenceState::Unfenced + ); + + // After the commit the transition reads the row as committed, so the other writer's + // change survives underneath the mirror, although the in-memory config never saw it. + assert_eq!(handle.get().max_db_pages, 1234); + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + let conn = raw(dir.path()).await; + let row = raw_config(&conn, "db"); + assert_eq!(row.max_db_pages, 42); + assert!(row.block_writes); + + // An ordinary config write is refused inside its transaction while the fence denies + // lifecycle operations, and a refused write is not published. + let e = fence_error( + handle + .store(DatabaseConfig { + max_db_pages: 9, + ..Default::default() + }) + .await, + ); + assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); + assert_eq!(handle.get().max_db_pages, 1234); + assert_eq!(raw_config(&conn, "db").max_db_pages, 42); + let e = fence_error(handle.flush().await); + assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); + + // So is a delete, and the fence row would stop an older binary's delete too. + let e = fence_error(remove_blocking(&store, "db")); + assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); + assert!(conn + .execute("DELETE FROM namespace_configs WHERE namespace = 'db'", ()) + .is_err()); + + // Once released, config writes and delete work again; delete takes the fence with it. + let release = request( + "db", + OP, + 2, + FenceState::SourceDraining, + 1, + FenceCommand::ReleaseSourceWriteFence, + ); + store + .apply_fence_command(release, ctx(2_000)) + .await + .unwrap(); + handle + .store(DatabaseConfig { + max_db_pages: 9, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(handle.get().max_db_pages, 9); + drop(handle); + assert!(remove_blocking(&store, "db").unwrap().is_some()); + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!( + inspection.fence, + StoredFence::None { + namespace_exists: false + } + ); + assert!(inspection.receipts.is_empty()); + } + + #[tokio::test] + async fn corrupt_fence_row_fails_closed() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + drop(handle); + drop(store); + + let conn = raw(dir.path()).await; + conn.execute( + "UPDATE namespace_fences SET record = x'00ff00' WHERE namespace = 'db'", + (), + ) + .unwrap(); + + // Startup does not fail, and does not guess. + let store = open(dir.path(), true).await; + let fence = store.inspect_fence("db".into()).await.unwrap().fence; + assert!(matches!( + fence, + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + .. + } + )); + // The config keeps the stored mirror, so statement-level checks stay closed too. + let handle = store.handle("db".into()).await; + assert!(handle.get().block_writes); + + let release = request( + "db", + OP, + 2, + FenceState::SourceDraining, + 1, + FenceCommand::ReleaseSourceWriteFence, + ); + let e = fence_error(store.apply_fence_command(release, ctx(2_000)).await); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + assert_eq!(e.detail(), Some(FenceDetail::CorruptRecord)); + let e = fence_error(handle.store(DatabaseConfig::default()).await); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + let e = fence_error( + store + .complete_fence_drain( + "db".into(), + OP, + Uuid::from_u128(1), + DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 1, + }, + }, + ctx(2_000), + ) + .await, + ); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + + // An unknown format version is its own reason. + conn.execute( + "UPDATE namespace_fences SET format_version = 99 WHERE namespace = 'db'", + (), + ) + .unwrap(); + let fence = store.inspect_fence("db".into()).await.unwrap().fence; + assert!(matches!( + fence, + StoredFence::Unavailable { + detail: FenceDetail::UnsupportedFormatVersion, + .. + } + )); + } + + #[tokio::test] + async fn marker_tracks_the_metastore() { + let dir = tempdir().unwrap(); + let dbs = dir.path().join("dbs"); + let store = open(dir.path(), true).await; + let handle = create_namespace(&store, "db").await; + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap(); + let conn = raw(dir.path()).await; + let (v1, r1, b1): (i64, i64, Vec) = conn + .query_row( + "SELECT format_version, revision, record FROM namespace_fences", + (), + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .unwrap(); + let commit = store + .complete_fence_drain( + "db".into(), + OP, + Uuid::from_u128(1), + DrainCompletion::SourceWrites { + boundary: FrozenBoundary { + log_id: LOG, + frame_no: 7, + }, + }, + ctx(2_000), + ) + .await + .unwrap(); + let record = commit.record.unwrap(); + drop(handle); + drop(store); + + // A marker lost after the commit is rewritten from the metastore on load. + std::fs::remove_file(fence_store::marker_path(&dbs, &"db".into())).unwrap(); + let store = open(dir.path(), true).await; + let marker = fence_store::read_marker(&dbs, &"db".into()) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(marker.record, record); + drop(store); + + // A metastore that went backwards (restored to revision 1) is not trusted over the + // newer marker, and loading it does not overwrite the marker. + conn.execute( + "UPDATE namespace_fences SET format_version = ?1, revision = ?2, record = ?3", + rusqlite::params![v1, r1, b1], + ) + .unwrap(); + let store = open(dir.path(), true).await; + let fence = store.inspect_fence("db".into()).await.unwrap().fence; + match fence { + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + marker: Some(m), + .. + } => assert_eq!(m, record), + other => panic!("expected metastore_behind_marker, got {other:?}"), + } + let marker = fence_store::read_marker(&dbs, &"db".into()) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(marker.record, record); + let e = fence_error( + store + .apply_fence_command( + request( + "db", + OP, + 3, + FenceState::SourceWriteFenced, + 2, + FenceCommand::ReleaseSourceWriteFence, + ), + ctx(3_000), + ) + .await, + ); + assert_eq!(e.detail(), Some(FenceDetail::MetastoreBehindMarker)); + } + + fn create_target(ns: &'static str, command_id: u128) -> FenceRequest { + request( + ns, + OP, + command_id, + FenceState::Absent, + 0, + FenceCommand::CreateTargetQuarantined { + config: TargetConfig { + max_db_size: Some(4096 * 100), + ..Default::default() + }, + }, + ) + } + + #[tokio::test] + async fn target_creation_is_atomic_and_replayable() { + let dir = tempdir().unwrap(); + let dbs = dir.path().join("dbs"); + let store = open(dir.path(), true).await; + + let commit = store + .apply_fence_command(create_target("tgt", 1), ctx(1_000)) + .await + .unwrap(); + assert_eq!(commit.receipt.outcome, FenceOutcome::Applied); + let record = commit.record.clone().unwrap(); + assert_eq!( + (record.state, record.revision), + (FenceState::TargetQuarantined, 1) + ); + assert_eq!(record.identity.target_incarnation_id, Some(INCARNATION)); + let created = commit.created_config.unwrap(); + assert_eq!(created.max_db_pages, 100); + assert!(!created.block_reads && !created.block_writes); + // Not published: the caller installs the gate first. + assert!(!store.exists(&"tgt".into()).await); + // Stored with the legacy mirror, and with its marker. + let conn = raw(dir.path()).await; + let row = raw_config(&conn, "tgt"); + assert!(row.block_reads && row.block_writes); + assert_eq!( + fence_store::read_marker(&dbs, &"tgt".into()) + .unwrap() + .unwrap() + .unwrap() + .record, + record + ); + let replay = store + .apply_fence_command(create_target("tgt", 1), ctx(2_000)) + .await + .unwrap(); + assert_eq!(replay.kind, FenceCommitKind::Replayed); + // A new command of the owner asking for the same thing is ALREADY_APPLIED; another + // operation cannot take the name. + let again = store + .apply_fence_command(create_target("tgt", 2), ctx(2_000)) + .await + .unwrap(); + assert_eq!(again.receipt.outcome, FenceOutcome::AlreadyApplied); + assert_eq!(again.record.unwrap().revision, 1); + let mut other = create_target("tgt", 5); + other.operation_id = OTHER_OP; + let e = fence_error(store.apply_fence_command(other, ctx(2_000)).await); + assert_eq!(e.outcome(), FenceOutcome::FenceOwnedByAnotherOperation); + + // A crash between the marker and the commit: the marker is all that is left. + store + .apply_fence_command(create_target("tgt2", 3), ctx(3_000)) + .await + .unwrap(); + let marker = fence_store::read_marker(&dbs, &"tgt2".into()) + .unwrap() + .unwrap() + .unwrap(); + for sql in [ + "DELETE FROM namespace_fence_receipts WHERE namespace = 'tgt2'", + "DELETE FROM namespace_fences WHERE namespace = 'tgt2'", + "DELETE FROM namespace_configs WHERE namespace = 'tgt2'", + ] { + conn.execute(sql, ()).unwrap(); + } + let fence = store.inspect_fence("tgt2".into()).await.unwrap().fence; + assert!(matches!( + fence, + StoredFence::Unavailable { + detail: FenceDetail::IncompleteTargetCreation, + .. + } + )); + // Only the same command completes it, keeping the incarnation id it announced. + let e = fence_error( + store + .apply_fence_command(create_target("tgt2", 4), ctx(4_000)) + .await, + ); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable); + let mut later = ctx(5_000); + later.new_incarnation_id = Uuid::from_u128(0x21); + let commit = store + .apply_fence_command(create_target("tgt2", 3), later) + .await + .unwrap(); + assert_eq!(commit.kind, FenceCommitKind::Committed); + let record = commit.record.unwrap(); + assert_eq!( + record.identity.target_incarnation_id, + marker.record.identity.target_incarnation_id + ); + assert_eq!( + store.inspect_fence("tgt2".into()).await.unwrap().fence, + StoredFence::Record(record) + ); + } + + async fn run_source_operation(store: &MetaStore, op: Uuid, base: u128, rev: u64, now: i64) { + let expected = if rev == 0 { + FenceState::Unfenced + } else { + FenceState::Released + }; + let mut acq = acquire("db", op, base); + acq.expected_state = expected; + acq.expected_revision = rev; + store.apply_fence_command(acq, ctx(now)).await.unwrap(); + let release = request( + "db", + op, + base + 1, + FenceState::SourceDraining, + rev + 1, + FenceCommand::ReleaseSourceWriteFence, + ); + store + .apply_fence_command(release, ctx(now + 1)) + .await + .unwrap(); + } + + #[tokio::test] + async fn receipts_of_finished_operations_are_pruned_after_retention() { + let dir = tempdir().unwrap(); + let store = open_with( + dir.path(), + MetaStoreConfig { + namespace_fence: true, + namespace_fence_receipt_retention: Some(Duration::from_secs(1)), + ..Default::default() + }, + ) + .await; + let _handle = create_namespace(&store, "db").await; + + run_source_operation(&store, OP, 1, 0, 1_000).await; + // Within the retention period, the finished operation's receipts are kept. + run_source_operation(&store, OTHER_OP, 10, 2, 1_500).await; + let ops = |receipts: &[StoredReceipt]| { + receipts + .iter() + .map(|r| r.receipt.clone().unwrap().operation_id) + .collect::>() + }; + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!(ops(&inspection.receipts), vec![OP, OP, OTHER_OP, OTHER_OP]); + // Later, a transition prunes other operations' old receipts, never the owner's. + run_source_operation(&store, Uuid::from_u128(0xc), 20, 4, 10_000).await; + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_eq!( + ops(&inspection.receipts), + vec![Uuid::from_u128(0xc), Uuid::from_u128(0xc)] + ); + assert_eq!(inspection.fence.revision(), 6); + } +} From 872576befc06f6564aa91d8d3405231c23fafab5 Mon Sep 17 00:00:00 2001 From: River Date: Tue, 29 Sep 2026 14:51:12 +0000 Subject: [PATCH 3/3] libsql-server: fail closed on ambiguous metastore recovery When namespace fences are in use (the flag is on, the fence tables exist, or a namespace directory holds a fence marker), a namespace whose state startup cannot establish is registered UNKNOWN_UNAVAILABLE instead of being skipped or given a default config: - an undecodable config row, fence row or marker; - a directory with a marker that the metastore has no trustworthy record for: filesystem recovery, a metastore rebuilt by destroy_on_error, a metastore restored from an older backup or without fence tables, and a target whose creation was interrupted. Such a namespace is refused by lookups, config writes and deletes with FENCE_STATE_UNAVAILABLE, is reported by InspectFence, and is settled only by a fence command that commits for it. MetaStore::handle() no longer default-creates on read paths: the new non-creating MetaStore::lookup serves NamespaceStore::with, fork's source and ATTACH authorisation. handle(), used only by creating paths, refuses unavailable names and names without a config whose directory holds a marker. Filesystem recovery skips marked directories, destroy_on_error moves the broken metastore aside instead of deleting it, and a fence row for a namespace name that cannot be decoded stops startup. Servers that never used fences recover as before. Co-authored-by: Tomasz Szymczyszyn --- libsql-server/src/connection/program.rs | 8 +- libsql-server/src/namespace/fence/store.rs | 39 + libsql-server/src/namespace/meta_store.rs | 802 +++++++++++++++++++-- libsql-server/src/namespace/store.rs | 27 +- libsql-server/src/schema/db.rs | 3 + 5 files changed, 825 insertions(+), 54 deletions(-) diff --git a/libsql-server/src/connection/program.rs b/libsql-server/src/connection/program.rs index 08dd9526f3..4d5ada51ff 100644 --- a/libsql-server/src/connection/program.rs +++ b/libsql-server/src/connection/program.rs @@ -370,7 +370,13 @@ pub async fn check_program_auth( } StmtKind::Attach(ref ns) => { ctx.auth.has_right(ns, Permission::AttachRead)?; - if !ctx.meta_store.handle(ns.clone()).await.get().allow_attach { + // A non-creating lookup: a missing namespace does not allow attach, and one + // whose fence state is not established is refused with its fence error. + let allow_attach = match ctx.meta_store.lookup(ns).await? { + Some(handle) => handle.get().allow_attach, + None => false, + }; + if !allow_attach { return Err(Error::Forbidden(format!( "Namespace `{ns}` doesn't allow attach" ))); diff --git a/libsql-server/src/namespace/fence/store.rs b/libsql-server/src/namespace/fence/store.rs index 2d6f9ca217..bb7a92a20f 100644 --- a/libsql-server/src/namespace/fence/store.rs +++ b/libsql-server/src/namespace/fence/store.rs @@ -215,6 +215,32 @@ pub fn remove_marker(dbs_path: &Path, namespace: &NamespaceName) -> io::Result<( } } +/// The namespace directories under `dbs_path` that hold a marker, in no particular order. A +/// directory whose name is not a valid namespace name is returned as its raw name, so the +/// caller can refuse to start rather than ignore it. +pub fn scan_markers(dbs_path: &Path) -> io::Result>> { + let entries = match fs::read_dir(dbs_path) { + Ok(entries) => entries, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e), + }; + let mut out = Vec::new(); + for entry in entries { + let entry = entry?; + if !entry.file_type()?.is_dir() || !entry.path().join(MARKER_FILE_NAME).try_exists()? { + continue; + } + let raw = entry.file_name(); + out.push(match raw.to_str() { + Some(name) => { + NamespaceName::from_string(name.to_string()).map_err(|_| name.to_string()) + } + None => Err(raw.to_string_lossy().into_owned()), + }); + } + Ok(out) +} + /// Whether the marker agrees with what the metastore says. Returned by [`read_fence`] so a /// loader can repair a marker that fell behind (a crash between commit and marker write). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -264,6 +290,19 @@ pub fn stored_revision( .optional() } +/// Like [`stored_revision`], for a namespace name that is not a valid [`NamespaceName`]. +pub fn stored_revision_raw( + conn: &rusqlite::Connection, + namespace: &str, +) -> rusqlite::Result> { + conn.query_row( + "SELECT revision FROM namespace_fences WHERE namespace = ?1", + [namespace], + |row| row.get(0), + ) + .optional() +} + fn decode_row(row: &RawFenceRow) -> Result { let format_version = u32::try_from(row.format_version) .map_err(|_| FenceDecodeError::UnsupportedFormatVersion(u32::MAX))?; diff --git a/libsql-server/src/namespace/meta_store.rs b/libsql-server/src/namespace/meta_store.rs index 098102e3f7..175fb24b55 100644 --- a/libsql-server/src/namespace/meta_store.rs +++ b/libsql-server/src/namespace/meta_store.rs @@ -91,6 +91,12 @@ struct MetaStoreInner { /// `/dbs`, where namespace directories and their fence markers are. dbs_path: PathBuf, fence: FenceSettings, + /// Namespaces whose state could not be recovered at startup and that are therefore + /// `UNKNOWN_UNAVAILABLE` (`docs/NAMESPACE_FENCE.md` section 13.3): an undecodable config + /// row, a fence that cannot be established, or a marker the metastore has no trustworthy + /// record for. They are refused by lookups and by every config or lifecycle change, and + /// never default-created. A fence command that commits for the name takes it out. + recovered: Mutex>, } /// How this metastore treats namespace fences (`docs/NAMESPACE_FENCE.md` section 13.1). @@ -100,6 +106,9 @@ struct FenceSettings { enabled: bool, /// The fence tables exist, so fence state is loaded and enforced. tables: bool, + /// Recovery fails closed (section 13.3): the flag is on, the fence tables exist, or a + /// namespace directory holds a marker. + fail_closed: bool, receipt_retention: Duration, } @@ -216,9 +225,13 @@ impl MetaStoreInner { if config.namespace_fence { fence_store::create_tables(&conn)?; } + let tables = fence_store::tables_exist(&conn)?; + let dbs_path = base_path.join("dbs"); + let marked = marked_namespaces(&dbs_path)?; let fence = FenceSettings { enabled: config.namespace_fence, - tables: fence_store::tables_exist(&conn)?, + tables, + fail_closed: config.namespace_fence || tables || !marked.is_empty(), receipt_retention: config .namespace_fence_receipt_retention .unwrap_or(fence_store::DEFAULT_RECEIPT_RETENTION), @@ -229,8 +242,9 @@ impl MetaStoreInner { conn: conn.into(), wal_manager, db_kind, - dbs_path: base_path.join("dbs"), + dbs_path, fence, + recovered: Default::default(), }; if config.allow_recover_from_fs { @@ -241,10 +255,72 @@ impl MetaStoreInner { if this.fence.tables { this.restore_fences()?; } + this.register_marked(&marked)?; Ok(this) } + /// Register every namespace directory with a marker that the metastore has no trustworthy + /// fence record for as `UNKNOWN_UNAVAILABLE` (section 13.3): a metastore that was rebuilt + /// (`destroy_on_error`), recovered from the filesystem, restored from an older backup, or + /// that lost its fence tables, and a target whose creation was interrupted. + fn register_marked(&mut self, marked: &[NamespaceName]) -> Result<()> { + for ns in marked { + if self.recovered.get_mut().contains_key(ns) { + continue; + } + let known = self.configs.get_mut().contains_key(ns); + if self.fence.tables { + if known { + // `restore_fences` compared the marker with the record. + continue; + } + let stored = match fence_store::read_fence(self.conn.get_mut(), &self.dbs_path, ns) + { + Ok((stored, _)) => stored, + Err(FenceStoreError::Sqlite(e)) => return Err(e.into()), + Err(e) => StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + reason: format!("the fence marker cannot be read: {e}"), + marker: None, + }, + }; + let (detail, reason, marker) = match stored { + StoredFence::Unavailable { + detail, + reason, + marker, + } => (detail, reason, marker), + other => ( + FenceDetail::CorruptRecord, + format!( + "the namespace has fence state {} but no usable config row", + other.state() + ), + other.record().cloned(), + ), + }; + mark_unavailable(self.recovered.get_mut(), ns.clone(), detail, reason, marker); + } else { + let (detail, marker) = match fence_store::read_marker(&self.dbs_path, ns)? { + None => continue, + Some(Ok(m)) => (FenceDetail::MetastoreBehindMarker, Some(m.record)), + Some(Err(_)) => (FenceDetail::CorruptRecord, None), + }; + let reason = "the namespace directory holds a fence marker but the metastore has \ + no fence tables (it was rebuilt, recovered or restored without them)" + .to_string(); + mark_unavailable(self.recovered.get_mut(), ns.clone(), detail, reason, marker); + } + } + Ok(()) + } + + /// The fence error for a namespace registered as unavailable at startup. + fn recovery_denial(&self, namespace: &NamespaceName) -> Option { + self.recovered.lock().get(namespace).map(unavailable_error) + } + fn maybe_recover_from_fs(&mut self, base_path: &Path) -> Result<()> { let count = self.conn @@ -267,6 +343,16 @@ impl MetaStoreInner { let config_path = entry.path().join("config.json"); let name = NamespaceName::from_string(entry.file_name().to_str().unwrap().to_string())?; + if entry + .path() + .join(fence_store::MARKER_FILE_NAME) + .try_exists()? + { + // A fenced namespace is never recovered with a guessed config; it is + // registered as unavailable below (section 13.3). + tracing::warn!("not recovering fenced namespace `{name}` from the filesystem"); + continue; + } let config = if config_path.try_exists()? { let config_bytes = std::fs::read(&config_path)?; serde_json::from_slice(&config_bytes)? @@ -291,10 +377,10 @@ impl MetaStoreInner { fn restore(&mut self) -> Result<()> { tracing::info!("restoring meta store"); - let mut stmt = self - .conn - .get_mut() - .prepare("SELECT namespace, config FROM namespace_configs")?; + let fence = self.fence; + let conn: &rusqlite::Connection = self.conn.get_mut(); + let mut unavailable = Vec::new(); + let mut stmt = conn.prepare("SELECT namespace, config FROM namespace_configs")?; let rows = stmt.query(())?.mapped(|r| { let ns = r.get::<_, String>(0)?; @@ -306,9 +392,19 @@ impl MetaStoreInner { for row in rows { match row { Ok((k, v)) => { - let ns = match NamespaceName::from_string(k) { + let ns = match NamespaceName::from_string(k.clone()) { Ok(ns) => ns, Err(e) => { + // A name nothing can address cannot be served or default-created, + // so a legacy row is skipped as before. A fenced one is an operator + // problem: its fence could not be enforced or inspected. + if fence.tables && fence_store::stored_revision_raw(conn, &k)?.is_some() + { + return Err(Error::Internal(format!( + "the metastore holds a namespace fence for `{k}`, which is not \ + a valid namespace name; refusing to start" + ))); + } tracing::warn!("unable to convert namespace name: {}", e); continue; } @@ -316,6 +412,11 @@ impl MetaStoreInner { let config = match metadata::DatabaseConfig::decode(&v[..]) { Ok(c) => Arc::new(DatabaseConfig::from(&c)), + Err(e) if fence.fail_closed => { + unavailable + .push((ns, format!("the config row cannot be decoded: {e}"))); + continue; + } Err(e) => { tracing::warn!("unable to convert config: {}", e); continue; @@ -338,6 +439,17 @@ impl MetaStoreInner { } } + drop(stmt); + for (ns, reason) in unavailable { + mark_unavailable( + self.recovered.get_mut(), + ns, + FenceDetail::CorruptRecord, + reason, + None, + ); + } + tracing::info!("meta store restore completed"); Ok(()) @@ -358,7 +470,14 @@ impl MetaStoreInner { Ok(r) => r, Err(FenceStoreError::Sqlite(e)) => return Err(e.into()), Err(e) => { - tracing::error!(namespace = %ns, "cannot establish namespace fence: {e}"); + fenced += 1; + mark_unavailable( + self.recovered.get_mut(), + ns, + FenceDetail::CorruptRecord, + format!("the namespace fence cannot be established: {e}"), + None, + ); continue; } }; @@ -376,12 +495,18 @@ impl MetaStoreInner { let config = fence_store::with_legacy_blocks(&config, &record.legacy_blocks); sender.send_modify(|c| c.config = Arc::new(config)); } - StoredFence::Unavailable { detail, reason, .. } => { + StoredFence::Unavailable { + detail, + reason, + marker, + } => { fenced += 1; - tracing::error!( - namespace = %ns, - %detail, - "namespace fence state is UNKNOWN_UNAVAILABLE: {reason}" + mark_unavailable( + self.recovered.get_mut(), + ns, + *detail, + reason.clone(), + marker.clone(), ); } } @@ -391,6 +516,109 @@ impl MetaStoreInner { } } +fn mark_unavailable( + recovered: &mut HashMap, + namespace: NamespaceName, + detail: FenceDetail, + reason: String, + marker: Option, +) { + tracing::error!( + namespace = %namespace, + %detail, + "namespace is UNKNOWN_UNAVAILABLE: {reason}" + ); + recovered.insert( + namespace, + StoredFence::Unavailable { + detail, + reason, + marker, + }, + ); +} + +/// The namespaces under `dbs_path` whose directory holds a fence marker. A marker in a +/// directory that is not a valid namespace name stops startup: the fence it records could be +/// neither enforced nor inspected. +fn marked_namespaces(dbs_path: &Path) -> Result> { + fence_store::scan_markers(dbs_path)? + .into_iter() + .map(|m| { + m.map_err(|raw| { + Error::Internal(format!( + "namespace directory `{raw}` holds a fence marker but is not a valid \ + namespace name; refusing to start" + )) + }) + }) + .collect() +} + +/// A namespace's fence as the metastore holds it now, for a metastore with the fence tables: +/// the live record and marker, unless they read as established while startup could not +/// recover the namespace (an undecodable config row, for instance), which stays unavailable. +fn established_fence( + inner: &MetaStoreInner, + conn: &rusqlite::Connection, + namespace: &NamespaceName, +) -> std::result::Result { + let (live, _) = fence_store::read_fence(conn, &inner.dbs_path, namespace)?; + if matches!(live, StoredFence::Unavailable { .. }) { + return Ok(live); + } + Ok(inner + .recovered + .lock() + .get(namespace) + .cloned() + .unwrap_or(live)) +} + +/// The error returned for a namespace whose fence state is `UNKNOWN_UNAVAILABLE`. +fn unavailable_error(stored: &StoredFence) -> FenceError { + stored + .permits(OperationClass::NormalRead) + .err() + .unwrap_or_else(|| { + FenceError::new( + FenceOutcome::FenceStateUnavailable, + "the namespace's fence state cannot be established", + ) + }) +} + +/// Why a name that has no config must not be created: its directory holds a marker, so it is a +/// target being created or a namespace the metastore lost (section 13.3). +fn marker_denial(dbs_path: &Path, namespace: &NamespaceName) -> Result> { + Ok(match fence_store::read_marker(dbs_path, namespace)? { + None => None, + Some(Ok(m)) => { + let revision = m.record.revision; + Some( + StoredFence::Record(m.record) + .permits(OperationClass::Lifecycle) + .err() + .unwrap_or_else(|| { + unavailable_error(&StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + reason: format!( + "the namespace directory holds a fence marker (revision \ + {revision}) but the metastore has no config for it" + ), + marker: None, + }) + }), + ) + } + Some(Err(e)) => Some(unavailable_error(&StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + reason: format!("the fence marker cannot be decoded: {e}"), + marker: None, + })), + }) +} + /// Handles config change updates by inserting them into the database and in-memory /// cache of configs. fn process(msg: ChangeMsg, inner: Arc) { @@ -442,6 +670,9 @@ fn try_process( namespace: &NamespaceName, config: &DatabaseConfig, ) -> Result<()> { + if let Some(e) = inner.recovery_denial(namespace) { + return Err(e.into()); + } let mut conn = inner.conn.blocking_lock(); // `BEGIN IMMEDIATE`: the write lock is what serialises this write with fence transitions // (docs/NAMESPACE_FENCE.md section 5.4), including those of other metastore connections. @@ -688,6 +919,9 @@ fn apply_fence_command( .map_or(request.operation_id, |r| r.operation_id); fence_store::prune_receipts(&tx, ns, owner, ctx.now_ms, inner.fence.receipt_retention)?; tx.commit()?; + // The command established the fence from the durable state; whatever startup could not + // recover about this name is settled. + inner.recovered.lock().remove(ns); let current = record.or_else(|| stored.record().cloned()); after_fence_commit( @@ -796,6 +1030,7 @@ fn complete_fence_drain( fence_store::write_record(&tx, &next, fence_store::stored_revision(&tx, ns)?)?; fence_store::write_receipt(&tx, &final_receipt)?; tx.commit()?; + inner.recovered.lock().remove(ns); after_fence_commit(inner, &conn, Some(&next), true); @@ -834,12 +1069,36 @@ impl MetaStore { if destroy_on_error { let db_path = base_path.join("metastore"); - tracing::info!( - "meta store set to destroy on restore error, removing metastore db path folder ({:?})", db_path - ); + // With fences in use the broken metastore may hold the only record of a + // fence, so it is kept aside for the operator rather than deleted, and the + // rebuilt metastore registers every marked namespace as unavailable + // (section 13.3). + let keep = config.namespace_fence + || marked_namespaces(&base_path.join("dbs")) + .map_or(true, |marked| !marked.is_empty()); + if keep { + let millis = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_millis()); + let aside = base_path.join(format!("metastore.broken-{millis}")); + tracing::error!( + "meta store failed to restore ({e}); moving it aside to {aside:?} \ + and rebuilding it" + ); + if let Err(rename) = std::fs::rename(&db_path, &aside) { + tracing::error!( + "failed to move the metastore aside ({rename}); not destroying it" + ); + return Err(e); + } + } else { + tracing::info!( + "meta store set to destroy on restore error, removing metastore db path folder ({:?})", db_path + ); - if let Err(e) = std::fs::remove_dir_all(&db_path) { - tracing::error!("failed to remove base path({:?}): {}", &db_path, e); + if let Err(e) = std::fs::remove_dir_all(&db_path) { + tracing::error!("failed to remove base path({:?}): {}", &db_path, e); + } } if let Err(e) = std::fs::create_dir_all(&db_path) { @@ -900,11 +1159,38 @@ impl MetaStore { Ok(Self { changes_tx, inner }) } - pub async fn handle(&self, namespace: NamespaceName) -> MetaStoreHandle { + /// The handle of an existing namespace, without creating one (section 13.3). `Ok(None)` + /// when the namespace does not exist; a fence error when its state could not be recovered. + /// Every path that only reads or serves a namespace uses this. + pub async fn lookup(&self, namespace: &NamespaceName) -> Result> { + if let Some(e) = self.inner.recovery_denial(namespace) { + return Err(e.into()); + } + let configs = self.inner.configs.lock().await; + Ok(configs.get(namespace).map(|sender| MetaStoreHandle { + namespace: namespace.clone(), + inner: HandleState::External(self.changes_tx.clone(), sender.subscribe()), + })) + } + + /// The handle of `namespace`, creating an empty in-memory entry when it does not exist. + /// Only paths that create a namespace (create, fork destination, reset, lazy creation) use + /// this. It refuses a namespace whose state could not be recovered, and a name without a + /// config whose directory holds a fence marker (a target being created, or a namespace the + /// metastore lost): creating either would publish a default config where a fence belongs. + pub async fn handle(&self, namespace: NamespaceName) -> Result { tracing::debug!("getting meta store handle"); + if let Some(e) = self.inner.recovery_denial(&namespace) { + return Err(e.into()); + } let change_tx = self.changes_tx.clone(); let mut configs = self.inner.configs.lock().await; + if !configs.contains_key(&namespace) { + if let Some(e) = marker_denial(&self.inner.dbs_path, &namespace)? { + return Err(e.into()); + } + } let sender = configs.entry(namespace.clone()).or_insert_with(|| { // TODO(lucio): if no entry exists we need to ensure we send the update to // the bg channel. @@ -916,14 +1202,17 @@ impl MetaStore { tracing::debug!("meta handle subscribed"); - MetaStoreHandle { + Ok(MetaStoreHandle { namespace, inner: HandleState::External(change_tx, rx), - } + }) } pub fn remove(&self, namespace: NamespaceName) -> Result>> { tracing::debug!("removing namespace `{}` from meta store", namespace); + if let Some(e) = self.inner.recovery_denial(&namespace) { + return Err(e.into()); + } // "configs" lock can be used in both async and sync contexts while "conn" lock always used // in blocking context @@ -1055,17 +1344,24 @@ impl MetaStore { tokio::task::spawn_blocking(move || -> std::result::Result<_, FenceStoreError> { let mut conn = inner.conn.blocking_lock(); if !inner.fence.tables { - let tx = conn.transaction()?; - let exists = fence_store::read_config_row(&tx, &namespace)?.is_some(); + let recovered = inner.recovered.lock().get(&namespace).cloned(); + let fence = match recovered { + Some(fence) => fence, + None => { + let tx = conn.transaction()?; + StoredFence::None { + namespace_exists: fence_store::read_config_row(&tx, &namespace)? + .is_some(), + } + } + }; return Ok(FenceInspection { - fence: StoredFence::None { - namespace_exists: exists, - }, + fence, receipts: Vec::new(), }); } let tx = conn.transaction()?; - let (fence, _) = fence_store::read_fence(&tx, &inner.dbs_path, &namespace)?; + let fence = established_fence(&inner, &tx, &namespace)?; let receipts = fence_store::read_receipts(&tx, &namespace)?; Ok(FenceInspection { fence, receipts }) }) @@ -1078,22 +1374,33 @@ impl MetaStore { pub async fn load_fences(&self) -> Result> { let inner = self.inner.clone(); tokio::task::spawn_blocking(move || -> std::result::Result<_, FenceStoreError> { + let recovered: Vec<(NamespaceName, StoredFence)> = inner + .recovered + .lock() + .iter() + .map(|(ns, fence)| (ns.clone(), fence.clone())) + .collect(); if !inner.fence.tables { - return Ok(Vec::new()); + return Ok(recovered); } let mut conn = inner.conn.blocking_lock(); let tx = conn.transaction()?; - let names: Vec = { + let mut names: Vec = { let mut stmt = tx.prepare("SELECT namespace FROM namespace_configs")?; let rows = stmt.query_map((), |r| r.get::<_, String>(0))?; - rows.collect::>()? + rows.collect::>>()? + .into_iter() + .filter_map(|name| NamespaceName::from_string(name).ok()) + .collect() }; + for (ns, _) in recovered { + if !names.contains(&ns) { + names.push(ns); + } + } let mut out = Vec::new(); - for name in names { - let Ok(ns) = NamespaceName::from_string(name) else { - continue; - }; - let (fence, _) = fence_store::read_fence(&tx, &inner.dbs_path, &ns)?; + for ns in names { + let fence = established_fence(&inner, &tx, &ns)?; if !matches!(fence, StoredFence::None { .. }) { out.push((ns, fence)); } @@ -1382,7 +1689,7 @@ mod fence_tests { } async fn create_namespace(store: &MetaStore, ns: &'static str) -> MetaStoreHandle { - let handle = store.handle(ns.into()).await; + let handle = store.handle(ns.into()).await.unwrap(); handle .store(DatabaseConfig { max_db_pages: 1234, @@ -1485,7 +1792,7 @@ mod fence_tests { ); // The in-memory config is the namespace's own, not the mirror. - let handle = store.handle("db".into()).await; + let handle = store.handle("db".into()).await.unwrap(); let config = handle.get(); assert!(!config.block_writes && !config.block_reads); assert_eq!(config.block_reason.as_deref(), Some("pre-fence")); @@ -1547,7 +1854,7 @@ mod fence_tests { .await, ); assert_eq!(e.detail(), Some(FenceDetail::FenceDisabled)); - let handle = store.handle("db".into()).await; + let handle = store.handle("db".into()).await.unwrap(); let e = fence_error(handle.store(DatabaseConfig::default()).await); assert_eq!(e.outcome(), FenceOutcome::MigrationWriteFenced); assert_eq!( @@ -1760,9 +2067,29 @@ mod fence_tests { .. } )); - // The config keeps the stored mirror, so statement-level checks stay closed too. - let handle = store.handle("db".into()).await; - assert!(handle.get().block_writes); + // The namespace is not served and cannot be recreated. Its in-memory config keeps the + // stored mirror, so statement-level checks would stay closed too. + assert_eq!( + fence_error(store.lookup(&"db".into()).await).detail(), + Some(FenceDetail::CorruptRecord) + ); + assert_eq!( + fence_error(store.handle("db".into()).await).detail(), + Some(FenceDetail::CorruptRecord) + ); + let config = store.inner.configs.lock().await[&NamespaceName::from("db")] + .borrow() + .config + .clone(); + assert!(config.block_writes); + // A handle taken before the fence became unavailable cannot write the config either. + let handle = MetaStoreHandle { + namespace: "db".into(), + inner: HandleState::External( + store.changes_tx.clone(), + store.inner.configs.lock().await[&NamespaceName::from("db")].subscribe(), + ), + }; let release = request( "db", @@ -2073,4 +2400,397 @@ mod fence_tests { ); assert_eq!(inspection.fence.revision(), 6); } + + /// Fail-closed metastore recovery (`docs/NAMESPACE_FENCE.md` section 13.3). + mod recovery { + use super::*; + + fn unavailable_detail(r: Result) -> FenceDetail { + let e = fence_error(r); + assert_eq!(e.outcome(), FenceOutcome::FenceStateUnavailable, "{e}"); + e.detail().expect("unavailable carries a detail") + } + + async fn open_err(dir: &Path, config: MetaStoreConfig) -> Error { + let (maker, manager) = metastore_connection_maker(None, dir).await.unwrap(); + let conn = maker().unwrap(); + match MetaStore::new(config, dir, conn, manager, DatabaseKind::Primary).await { + Ok(_) => panic!("the metastore opened"), + Err(e) => e, + } + } + + fn recover_from_fs(fence: bool) -> MetaStoreConfig { + MetaStoreConfig { + allow_recover_from_fs: true, + namespace_fence: fence, + ..Default::default() + } + } + + /// A fenced namespace `db` (SOURCE_DRAINING, revision 1, with its marker). + async fn fenced_db(dir: &Path) -> NamespaceFenceRecord { + let store = open(dir, true).await; + let _handle = create_namespace(&store, "db").await; + store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap() + .record + .unwrap() + } + + fn assert_marker_unavailable(fence: &StoredFence, record: &NamespaceFenceRecord) { + match fence { + StoredFence::Unavailable { + detail: FenceDetail::MetastoreBehindMarker, + marker: Some(m), + .. + } => assert_eq!(m, record), + other => panic!("expected metastore_behind_marker, got {other:?}"), + } + } + + #[tokio::test] + async fn lookup_never_creates() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + assert!(store.lookup(&"missing".into()).await.unwrap().is_none()); + assert!(!store.exists(&"missing".into()).await); + + let _handle = create_namespace(&store, "db").await; + let found = store.lookup(&"db".into()).await.unwrap().unwrap(); + assert_eq!(found.get().max_db_pages, 1234); + // Only the creating path adds an entry. + let created = store.handle("new".into()).await.unwrap(); + assert_eq!( + created.get().max_db_pages, + DatabaseConfig::default().max_db_pages + ); + assert!(store.exists(&"new".into()).await); + } + + #[tokio::test] + async fn fs_recovery_with_marker_unavailable() { + for fence in [true, false] { + let dir = tempdir().unwrap(); + let record = fenced_db(dir.path()).await; + // A legacy namespace directory without a marker. + std::fs::create_dir_all(dir.path().join("dbs").join("legacy")).unwrap(); + std::fs::remove_dir_all(dir.path().join("metastore")).unwrap(); + + let store = open_with(dir.path(), recover_from_fs(fence)).await; + // The legacy directory is recovered as before. + let legacy = store.lookup(&"legacy".into()).await.unwrap().unwrap(); + assert_eq!( + legacy.get().max_db_pages, + DatabaseConfig::default().max_db_pages + ); + // The fenced one is not recovered with a guessed config, and is unavailable. + assert_eq!( + unavailable_detail(store.lookup(&"db".into()).await), + FenceDetail::MetastoreBehindMarker, + "fence flag {fence}" + ); + assert_eq!( + unavailable_detail(store.handle("db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + assert_eq!( + unavailable_detail(remove_blocking(&store, "db")), + FenceDetail::MetastoreBehindMarker + ); + let inspection = store.inspect_fence("db".into()).await.unwrap(); + assert_marker_unavailable(&inspection.fence, &record); + let fences = store.load_fences().await.unwrap(); + assert_eq!(fences.len(), 1); + assert_marker_unavailable(&fences[0].1, &record); + // Nothing was written for it, and the marker is untouched. + let conn = raw(dir.path()).await; + assert!(fence_store::read_config_row(&conn, &"db".into()) + .unwrap() + .is_none()); + let marker = fence_store::read_marker(&dir.path().join("dbs"), &"db".into()) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(marker.record, record); + } + } + + #[tokio::test] + async fn destroy_on_error_keeps_fenced_unavailable() { + let dir = tempdir().unwrap(); + let record = fenced_db(dir.path()).await; + { + // Break the metastore so that restoring it fails. + let conn = raw(dir.path()).await; + conn.execute( + "ALTER TABLE namespace_configs RENAME COLUMN config TO broken", + (), + ) + .unwrap(); + } + let store = open_with( + dir.path(), + MetaStoreConfig { + destroy_on_error: true, + namespace_fence: true, + ..Default::default() + }, + ) + .await; + // The broken metastore is kept aside, not deleted. + let aside: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .filter(|n| n.starts_with("metastore.broken-")) + .collect(); + assert_eq!(aside.len(), 1, "{aside:?}"); + assert!(dir.path().join(&aside[0]).join("data").exists()); + // The rebuilt metastore knows nothing of `db`, so its marker makes it unavailable. + assert_eq!( + unavailable_detail(store.lookup(&"db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + assert_eq!( + unavailable_detail(store.handle("db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + assert_marker_unavailable( + &store.inspect_fence("db".into()).await.unwrap().fence, + &record, + ); + } + + #[tokio::test] + async fn destroy_on_error_without_fences_is_unchanged() { + let dir = tempdir().unwrap(); + { + let store = open(dir.path(), false).await; + let _handle = create_namespace(&store, "db").await; + } + { + let conn = raw(dir.path()).await; + conn.execute( + "ALTER TABLE namespace_configs RENAME COLUMN config TO broken", + (), + ) + .unwrap(); + } + let store = open_with( + dir.path(), + MetaStoreConfig { + destroy_on_error: true, + ..Default::default() + }, + ) + .await; + assert!(store.lookup(&"db".into()).await.unwrap().is_none()); + assert!(!std::fs::read_dir(dir.path()).unwrap().any(|e| e + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("metastore."))); + } + + #[tokio::test] + async fn undecodable_row_unavailable() { + let dir = tempdir().unwrap(); + { + let store = open(dir.path(), true).await; + let _handle = create_namespace(&store, "good").await; + } + let conn = raw(dir.path()).await; + conn.execute( + "INSERT INTO namespace_configs VALUES ('bad', X'FFFFFFFF')", + (), + ) + .unwrap(); + let store = open(dir.path(), true).await; + assert!(store.lookup(&"good".into()).await.unwrap().is_some()); + assert_eq!( + unavailable_detail(store.lookup(&"bad".into()).await), + FenceDetail::CorruptRecord + ); + // Never replaced by a default config, nor deleted. + assert_eq!( + unavailable_detail(store.handle("bad".into()).await), + FenceDetail::CorruptRecord + ); + assert_eq!( + unavailable_detail(remove_blocking(&store, "bad")), + FenceDetail::CorruptRecord + ); + assert!(matches!( + store.inspect_fence("bad".into()).await.unwrap().fence, + StoredFence::Unavailable { + detail: FenceDetail::CorruptRecord, + .. + } + )); + let bytes: Vec = conn + .query_row( + "SELECT config FROM namespace_configs WHERE namespace = 'bad'", + (), + |r| r.get(0), + ) + .unwrap(); + assert_eq!(bytes, vec![0xff; 4]); + } + + #[tokio::test] + async fn undecodable_row_without_fences_is_skipped_as_before() { + let dir = tempdir().unwrap(); + drop(open(dir.path(), false).await); + let conn = raw(dir.path()).await; + conn.execute( + "INSERT INTO namespace_configs VALUES ('bad', X'FFFFFFFF')", + (), + ) + .unwrap(); + let store = open(dir.path(), false).await; + assert!(store.lookup(&"bad".into()).await.unwrap().is_none()); + } + + #[tokio::test] + async fn undecodable_name_with_fence_fails_startup() { + let dir = tempdir().unwrap(); + drop(open(dir.path(), true).await); + let conn = raw(dir.path()).await; + let config = metadata::DatabaseConfig::from(&DatabaseConfig::default()).encode_to_vec(); + conn.execute("INSERT INTO namespace_configs VALUES ('', ?1)", [&config]) + .unwrap(); + // Without a fence it is skipped, as before. + let store = open(dir.path(), true).await; + assert!(store.load_fences().await.unwrap().is_empty()); + drop(store); + conn.execute("INSERT INTO namespace_fences VALUES ('', 1, 1, X'00')", ()) + .unwrap(); + let e = open_err(dir.path(), MetaStoreConfig::default()).await; + assert!(matches!(e, Error::Internal(_)), "{e}"); + } + + #[tokio::test] + async fn marker_in_invalid_directory_fails_startup() { + use std::os::unix::ffi::OsStrExt; + let dir = tempdir().unwrap(); + let bad = dir + .path() + .join("dbs") + .join(std::ffi::OsStr::from_bytes(b"\xff")); + std::fs::create_dir_all(&bad).unwrap(); + std::fs::write(bad.join(fence_store::MARKER_FILE_NAME), b"x").unwrap(); + let e = open_err(dir.path(), MetaStoreConfig::default()).await; + assert!(matches!(e, Error::Internal(_)), "{e}"); + } + + #[tokio::test] + async fn incomplete_target_unavailable() { + let dir = tempdir().unwrap(); + let store = open(dir.path(), true).await; + let commit = store + .apply_fence_command(create_target("tgt", 3), ctx(1_000)) + .await + .unwrap(); + let record = commit.record.unwrap(); + // Committed but not yet published: nothing can create a default namespace over it. + let e = fence_error(store.handle("tgt".into()).await); + assert_eq!(e.outcome(), FenceOutcome::MigrationTargetQuarantined); + assert!(store.lookup(&"tgt".into()).await.unwrap().is_none()); + drop(store); + + // A crash between the marker and the commit: the marker is all that is left. + let conn = raw(dir.path()).await; + for sql in [ + "DELETE FROM namespace_fence_receipts WHERE namespace = 'tgt'", + "DELETE FROM namespace_fences WHERE namespace = 'tgt'", + "DELETE FROM namespace_configs WHERE namespace = 'tgt'", + ] { + conn.execute(sql, ()).unwrap(); + } + let store = open(dir.path(), true).await; + assert_eq!( + unavailable_detail(store.lookup(&"tgt".into()).await), + FenceDetail::IncompleteTargetCreation + ); + assert_eq!( + unavailable_detail(store.handle("tgt".into()).await), + FenceDetail::IncompleteTargetCreation + ); + let fences = store.load_fences().await.unwrap(); + assert!(matches!( + fences.as_slice(), + [( + _, + StoredFence::Unavailable { + detail: FenceDetail::IncompleteTargetCreation, + .. + } + )] + )); + // The same command completes the creation, which settles the name. + let commit = store + .apply_fence_command(create_target("tgt", 3), ctx(2_000)) + .await + .unwrap(); + assert_eq!(commit.kind, FenceCommitKind::Committed); + assert_eq!(commit.record.as_ref().unwrap().identity, record.identity); + assert!(store.lookup(&"tgt".into()).await.unwrap().is_none()); + let e = fence_error(store.handle("tgt".into()).await); + assert_eq!(e.outcome(), FenceOutcome::MigrationTargetQuarantined); + assert_eq!( + store.inspect_fence("tgt".into()).await.unwrap().fence, + StoredFence::Record(commit.record.unwrap()) + ); + } + + #[tokio::test] + async fn metastore_rollback_detected_by_marker() { + let dir = tempdir().unwrap(); + let conn = raw(dir.path()).await; + let record = { + let store = open(dir.path(), true).await; + let _handle = create_namespace(&store, "db").await; + let record = store + .apply_fence_command(acquire("db", OP, 1), ctx(1_000)) + .await + .unwrap() + .record + .unwrap(); + drop(store); + // A metastore restored from a backup taken before the fence: the namespace is + // unfenced there, and only its marker remembers the fence. + conn.execute("DELETE FROM namespace_fence_receipts", ()) + .unwrap(); + conn.execute("DELETE FROM namespace_fences", ()).unwrap(); + record + }; + let store = open(dir.path(), true).await; + assert_eq!( + unavailable_detail(store.lookup(&"db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + // Neither served, nor deleted, nor given a new config. + assert_eq!( + unavailable_detail(store.handle("db".into()).await), + FenceDetail::MetastoreBehindMarker + ); + assert_eq!( + unavailable_detail(remove_blocking(&store, "db")), + FenceDetail::MetastoreBehindMarker + ); + assert_marker_unavailable( + &store.inspect_fence("db".into()).await.unwrap().fence, + &record, + ); + // A new operation cannot acquire over the lost fence either. + let e = fence_error( + store + .apply_fence_command(acquire("db", OTHER_OP, 2), ctx(2_000)) + .await, + ); + assert_eq!(e.detail(), Some(FenceDetail::MetastoreBehindMarker)); + } + } } diff --git a/libsql-server/src/namespace/store.rs b/libsql-server/src/namespace/store.rs index 86e9438ccd..1813ef5187 100644 --- a/libsql-server/src/namespace/store.rs +++ b/libsql-server/src/namespace/store.rs @@ -173,7 +173,7 @@ impl NamespaceStore { ns.destroy().await?; } - let db_config = self.inner.metadata.handle(namespace.clone()).await; + let db_config = self.inner.metadata.handle(namespace.clone()).await?; // destroy on-disk database self.cleanup( &namespace, @@ -240,7 +240,9 @@ impl NamespaceStore { return Err(crate::Error::NamespaceDoesntExist(from.to_string())); } - let from_config = self.inner.metadata.handle(from.clone()).await; + let Some(from_config) = self.inner.metadata.lookup(&from).await? else { + return Err(crate::Error::NamespaceDoesntExist(from.to_string())); + }; let from_entry = self .load_namespace(&from, from_config.clone(), RestoreOption::Latest) .await?; @@ -275,7 +277,7 @@ impl NamespaceStore { should_delete: true, }; - let handle = self.inner.metadata.handle(to.clone()).await; + let handle = self.inner.metadata.handle(to.clone()).await?; handle .store_and_maybe_flush(Some(to_config.into()), false) .await?; @@ -322,13 +324,6 @@ impl NamespaceStore { where Fun: FnOnce(&Namespace) -> R, { - if namespace != NamespaceName::default() - && !self.inner.metadata.exists(&namespace).await - && !self.inner.allow_lazy_creation - { - return Err(Error::NamespaceDoesntExist(namespace.to_string())); - } - let f = { let name = namespace.clone(); move |ns: NamespaceEntry| async move { @@ -341,7 +336,15 @@ impl NamespaceStore { } }; - let handle = self.inner.metadata.handle(namespace.to_owned()).await; + // A lookup that cannot create: only the default namespace and lazy creation create a + // namespace here, and those refuse a name whose fence state is not established. + let handle = match self.inner.metadata.lookup(&namespace).await? { + Some(handle) => handle, + None if namespace == NamespaceName::default() || self.inner.allow_lazy_creation => { + self.inner.metadata.handle(namespace.clone()).await? + } + None => return Err(Error::NamespaceDoesntExist(namespace.to_string())), + }; f(self .load_namespace(&namespace, handle, RestoreOption::Latest) .await?) @@ -440,7 +443,7 @@ impl NamespaceStore { } let db_config = Arc::new(db_config); - let handle = self.inner.metadata.handle(namespace.clone()).await; + let handle = self.inner.metadata.handle(namespace.clone()).await?; tracing::debug!("storing db config"); handle.store(db_config).await?; tracing::debug!("completed storing db config, loading namespace"); diff --git a/libsql-server/src/schema/db.rs b/libsql-server/src/schema/db.rs index ec8dcad840..d0bce10128 100644 --- a/libsql-server/src/schema/db.rs +++ b/libsql-server/src/schema/db.rs @@ -486,6 +486,7 @@ mod test { meta_store .handle(schema.into()) .await + .unwrap() .store(DatabaseConfig { is_shared_schema: true, ..Default::default() @@ -502,6 +503,7 @@ mod test { meta_store .handle(name.into()) .await + .unwrap() .store(DatabaseConfig { shared_schema_name: Some(schema.into()), ..Default::default() @@ -579,6 +581,7 @@ mod test { assert!(meta_store .handle("ns1".into()) .await + .unwrap() .store(DatabaseConfig { shared_schema_name: Some("schema1".into()), ..Default::default()