From ee7eba6bbb31065fd6a1dc1501d93d6234904e1a Mon Sep 17 00:00:00 2001 From: Kim Altintop Date: Thu, 8 Oct 2026 08:42:39 +0200 Subject: [PATCH] WIP: Blobs --- crates/bench/benches/special.rs | 2 +- crates/core/src/host/host_controller.rs | 15 +- crates/datastore/src/error.rs | 16 +- .../locking_tx_datastore/committed_state.rs | 98 ++-- .../src/locking_tx_datastore/datastore.rs | 38 +- .../src/locking_tx_datastore/mut_tx.rs | 35 +- .../src/locking_tx_datastore/replay.rs | 14 +- .../src/locking_tx_datastore/state_view.rs | 6 +- .../datastore/src/locking_tx_datastore/tx.rs | 4 +- .../src/locking_tx_datastore/tx_state.rs | 25 +- crates/dst/src/engine.rs | 4 +- crates/engine/src/relational_db.rs | 40 +- crates/execution/src/lib.rs | 2 +- crates/portable-datastore/src/lib.rs | 6 +- crates/snapshot/src/lib.rs | 51 +- crates/snapshot/src/remote.rs | 15 +- crates/snapshot/tests/remote.rs | 4 +- crates/table/benches/page.rs | 39 +- crates/table/benches/page_manager.rs | 41 +- crates/table/src/bflatn_from.rs | 53 +- crates/table/src/bflatn_to.rs | 47 +- crates/table/src/blob_store.rs | 271 +-------- crates/table/src/eq.rs | 6 +- crates/table/src/eq_to_pv.rs | 21 +- crates/table/src/page.rs | 163 +++--- crates/table/src/pages.rs | 9 +- crates/table/src/read_column.rs | 20 +- crates/table/src/row_hash.rs | 32 +- crates/table/src/static_bsatn_validator.rs | 6 +- crates/table/src/static_layout.rs | 10 +- crates/table/src/table.rs | 484 ++++++++-------- crates/table/src/table_index/mod.rs | 45 +- crates/table/src/tiered/blob_set.rs | 509 ++++++++++++++++ crates/table/src/tiered/budget.rs | 94 +-- crates/table/src/tiered/error.rs | 88 +++ crates/table/src/tiered/map.rs | 191 ++++++ crates/table/src/tiered/mem.rs | 445 ++++++++++++++ crates/table/src/tiered/mod.rs | 120 +++- crates/table/src/tiered/page_manager.rs | 544 ++++++------------ crates/table/src/tiered/page_set.rs | 121 +++- crates/table/src/tiered/storage.rs | 140 +++++ 41 files changed, 2551 insertions(+), 1323 deletions(-) create mode 100644 crates/table/src/tiered/blob_set.rs create mode 100644 crates/table/src/tiered/error.rs create mode 100644 crates/table/src/tiered/map.rs create mode 100644 crates/table/src/tiered/mem.rs create mode 100644 crates/table/src/tiered/storage.rs diff --git a/crates/bench/benches/special.rs b/crates/bench/benches/special.rs index 7e25f6e8488..70a3403948f 100644 --- a/crates/bench/benches/special.rs +++ b/crates/bench/benches/special.rs @@ -144,7 +144,7 @@ fn serialize_benchmarks< PageManager::new_for_test().into(), PageEvictionPolicy::NeverEvict, ); - let mut blob_store = spacetimedb_table::blob_store::HashMapBlobStore::default(); + let blob_store = spacetimedb_table::tiered::BlobSet::new_for_test(); let ptrs = data_pv .elements diff --git a/crates/core/src/host/host_controller.rs b/crates/core/src/host/host_controller.rs index 55e52350a41..f1f9679adb4 100644 --- a/crates/core/src/host/host_controller.rs +++ b/crates/core/src/host/host_controller.rs @@ -50,6 +50,7 @@ use spacetimedb_sats::hash::Hash; use spacetimedb_schema::auto_migrate::{ponder_migrate, AutoMigrateError, MigrationPolicy, PrettyPrintStyle}; use spacetimedb_schema::def::{ModuleDef, RawModuleDefVersion}; use spacetimedb_table::page_pool::PagePool; +use spacetimedb_table::tiered::{ByteBudget, MemoryObjectStore, TieredStorageConfig}; use std::future::Future; use std::ops::Deref; use std::sync::Arc; @@ -1251,7 +1252,7 @@ impl Host { EmptyHistory::new(), None, Some(tx_metrics_queue), - page_pool.clone(), + TieredStorageConfig::new_for_memory_database(page_pool.clone()), )?, db::Storage::Disk => { // Replay from the local state. @@ -1262,12 +1263,18 @@ impl Host { owner_identity: database.owner_identity, }; let persistence = persistence.persistence(&persistence_db, replica_id).await?; + // TODO(disk-tables): Figure out where the config comes from. + let storage = TieredStorageConfig { + page_pool: page_pool.clone(), + local_object_store: Box::new(MemoryObjectStore::default()), + remote_object_store: None, + resident_budget: ByteBudget::unlimited(), + }; // Loading a database from persistent storage involves heavy // blocking I/O. `asyncify` to avoid blocking the async worker. let (db, clients) = asyncify({ let database_identity = database.database_identity; let owner_identity = database.owner_identity; - let page_pool = page_pool.clone(); move || { RelationalDB::open( database_identity, @@ -1275,7 +1282,7 @@ impl Host { history, Some(persistence), Some(tx_metrics_queue), - page_pool, + storage, ) } }) @@ -1508,7 +1515,7 @@ impl Host { EmptyHistory::new(), None, None, - page_pool, + TieredStorageConfig::new_for_module_check(page_pool), )?; let memory_observer: Arc = Arc::new(()); diff --git a/crates/datastore/src/error.rs b/crates/datastore/src/error.rs index d389984ef5d..5b6112ffedc 100644 --- a/crates/datastore/src/error.rs +++ b/crates/datastore/src/error.rs @@ -8,10 +8,10 @@ use spacetimedb_sats::{AlgebraicType, AlgebraicValue}; use spacetimedb_schema::def::error::LibError; #[cfg(feature = "durability")] use spacetimedb_snapshot::SnapshotError; -use spacetimedb_table::tiered::PageError; use spacetimedb_table::{ bflatn_to, read_column, table::{self, ReadViaBsatnError, UniqueConstraintViolation}, + tiered::{BlobError, DeleteRowError, PageError}, }; use thiserror::Error; @@ -39,6 +39,12 @@ pub enum DatastoreError { #[error(transparent)] Page(#[from] PageError), + #[error(transparent)] + TableStorage(#[from] table::TableError), + + #[error(transparent)] + Blob(#[from] BlobError), + #[error(transparent)] Other(#[from] anyhow::Error), } @@ -146,7 +152,7 @@ impl From for DatastoreError { table::InsertError::Duplicate(e) => TableError::from(e).into(), table::InsertError::Bflatn(e) => TableError::from(e).into(), table::InsertError::IndexError(e) => IndexError::from(e).into(), - table::InsertError::Page(e) => e.into(), + table::InsertError::Table(e) => e.into(), } } } @@ -157,6 +163,12 @@ impl From for DatastoreError { } } +impl From for DatastoreError { + fn from(err: DeleteRowError) -> Self { + table::TableError::from(err).into() + } +} + #[cfg(feature = "durability")] impl From for DatastoreError { fn from(e: SnapshotError) -> Self { diff --git a/crates/datastore/src/locking_tx_datastore/committed_state.rs b/crates/datastore/src/locking_tx_datastore/committed_state.rs index 6692bb5fdd1..818467d425b 100644 --- a/crates/datastore/src/locking_tx_datastore/committed_state.rs +++ b/crates/datastore/src/locking_tx_datastore/committed_state.rs @@ -53,11 +53,9 @@ use spacetimedb_schema::{ table_name::TableName, }; use spacetimedb_table::{ - blob_store::{BlobStore, HashMapBlobStore}, indexes::{RowPointer, SquashedOffset}, - page_pool::PagePool, table::{RowRef, Table, TableAndIndex}, - tiered::{ByteBudget, ByteBudgetConfig, PageEvictionPolicy, PageManager, PreparedCommit}, + tiered::{BlobHandle, BlobSet, PageEvictionPolicy, PreparedCommit, TieredDatabaseStorage}, }; use std::sync::Arc; use std::{collections::BTreeMap, iter}; @@ -76,10 +74,9 @@ use std::{collections::BTreeMap, iter}; pub struct CommittedState { pub(crate) next_tx_offset: u64, pub(crate) tables: IntMap, - pub(crate) blob_store: HashMapBlobStore, + pub(crate) storage: TieredDatabaseStorage, /// Provides fast lookup for index id -> an index. pub(super) index_id_map: IndexIdMap, - pub(super) page_manager: Arc, /// Total bytes occupied by physical pages in committed tables. datastore_page_bytes: u64, /// We track the read sets for each view in the committed state. @@ -135,9 +132,8 @@ impl MemoryUsage for CommittedState { let Self { next_tx_offset, tables, - blob_store, + storage, index_id_map, - page_manager: _, datastore_page_bytes, read_sets, view_instances, @@ -149,7 +145,7 @@ impl MemoryUsage for CommittedState { // NOTE(centril): We do not want to include the heap usage of `page_pool` as it's a shared resource. next_tx_offset.heap_usage() + tables.heap_usage() - + blob_store.heap_usage() + + storage.heap_usage() + index_id_map.heap_usage() + datastore_page_bytes.heap_usage() + read_sets.heap_usage() @@ -215,20 +211,14 @@ impl StateView for CommittedState { } impl CommittedState { - pub(super) fn new(page_pool: PagePool) -> Self { - let page_manager = Arc::new(PageManager::new( - page_pool.clone(), - Arc::new(()), - ByteBudget::new(ByteBudgetConfig::unlimited()).unwrap(), - )); + pub(super) fn new(storage: TieredDatabaseStorage) -> Self { Self { next_tx_offset: <_>::default(), tables: <_>::default(), - blob_store: <_>::default(), + storage, index_id_map: <_>::default(), read_sets: <_>::default(), view_instances: <_>::default(), - page_manager, datastore_page_bytes: 0, ephemeral_tables: <_>::default(), sequence_advance_simulate_reallocation_rng: { @@ -252,12 +242,12 @@ impl CommittedState { /// Returns committed datastore bytes. /// /// This currently includes the pages allocated by the datastore - /// and the objects stored in the [`BlobStore`]. + /// and the objects stored in the [`BlobSet`]. /// /// TODO: Eventually this should also include index bytes /// once indexes are managed by the page cache. pub fn datastore_memory_bytes(&self) -> u64 { - self.datastore_page_bytes + self.blob_store.physical_bytes_used_by_blobs() + self.datastore_page_bytes + self.storage.physical_bytes_used_by_blobs() } /// Recomputes `datastore_page_bytes` from committed tables. @@ -464,14 +454,14 @@ impl CommittedState { /// Returns an iterator doing a full table scan on `table_id`. pub(super) fn table_scan<'a>(&'a self, table_id: TableId) -> Option> { - Some(self.get_table(table_id)?.scan_rows(&self.blob_store).err_into()) + Some(self.get_table(table_id)?.scan_rows(self.storage.blobs()).err_into()) } /// Returns an index for `table_id` on `cols`, if any. pub(super) fn get_index_by_cols(&self, table_id: TableId, cols: &ColList) -> Option> { self.tables .get(&table_id)? - .get_index_by_cols_with_table(&self.blob_store, cols) + .get_index_by_cols_with_table(self.storage.blobs(), cols) } /// Returns the table associated with the given `index_id`, if any. @@ -483,7 +473,7 @@ impl CommittedState { pub(super) fn get_index_by_id_with_table(&self, table_id: TableId, index_id: IndexId) -> Option> { self.tables .get(&table_id)? - .get_index_by_id_with_table(&self.blob_store, index_id) + .get_index_by_id_with_table(self.storage.blobs(), index_id) } // TODO(perf, deep-integration): Make this method `unsafe`. Add the following to the docs: @@ -509,7 +499,7 @@ impl CommittedState { .expect("Attempt to get COMMITTED_STATE row from table not present in tables."); // TODO(perf, deep-integration): Use `get_row_ref_unchecked`. Ok(table - .get_row_ref(&self.blob_store, row_ptr)? + .get_row_ref(self.storage.blobs(), row_ptr)? .expect("row pointer to be present")) } @@ -577,7 +567,10 @@ impl CommittedState { } fn prepare_merge_inner(&mut self, tx_state: &TxState) -> Result { - let mut prepared = PreparedMerge::default(); + // Prepare transaction-local blobs before planning page mutations. This + // installs and pins their frames without publishing logical references. + let blobs: Box<[BlobHandle]> = self.storage.merge_blobs_from(&tx_state.blob_store)?; + let mut prepared = PreparedMerge::with_pinned_blobs(blobs); // If a table was dropped, collect its rows. // Subscription updates contain all the deleted rows, instead of a drop @@ -588,7 +581,7 @@ impl CommittedState { if let PendingSchemaChange::TableRemoved(table_id, table) = change { assert!(!prepared.mutated_tables.contains_key(table_id)); let rows = table - .scan_rows(&self.blob_store) + .scan_rows(self.storage.blobs()) .map(|row| Ok(row?.to_product_value())) .collect::>()?; prepared.removed_tables.push(PreparedRemovedTable { @@ -622,12 +615,12 @@ impl CommittedState { Some(tx_insert_table) => Either::Left( tx_insert_table .scan_rows(&tx_state.blob_store) - .map(|row| Ok(commit_table.prepare_insert(row?))), + .map(|row| row.map(|row| commit_table.prepare_insert(row))), ), None => Either::Right(iter::empty()), }; - let commit = commit_table.prepare_commit(deletes, inserts)?; + let commit = commit_table.prepare_commit(self.storage.blobs(), deletes, inserts)?; prepared.insert(table_id, commit); } @@ -644,7 +637,13 @@ impl CommittedState { let mut tx_data = TxData::default(); let mut truncates = IntSet::default(); - for (table_id, commit) in prepared.mutated_tables { + let PreparedMerge { + mutated_tables, + removed_tables, + pinned_blobs, + } = prepared; + + for (table_id, commit) in mutated_tables { let (table, blob_store, _) = self .get_table_and_blob_store_mut(table_id) .expect("table must have been created during prepare phase"); @@ -679,7 +678,11 @@ impl CommittedState { } } - for PreparedRemovedTable { id, name, rows } in prepared.removed_tables { + // All page mutations have finished, so pinned blob frames no longer + // need to be pinned by the commit guard. + drop(pinned_blobs); + + for PreparedRemovedTable { id, name, rows } in removed_tables { truncates.insert(id); tx_data.set_deletes_for_table(id, &name, rows); } @@ -894,13 +897,13 @@ impl CommittedState { let table = self .get_table(table_id) .ok_or_else(|| TableError::IdNotFoundState(table_id))?; - Ok((table, &self.blob_store as &dyn BlobStore, &self.index_id_map)) + Ok((table, self.storage.blobs(), &self.index_id_map)) } pub(super) fn get_table_and_blob_store_mut( &mut self, table_id: TableId, - ) -> Result<(&mut Table, &mut dyn BlobStore, &mut IndexIdMap)> { + ) -> Result<(&mut Table, &BlobSet, &mut IndexIdMap)> { // NOTE(centril): `TableError` is a fairly large type. // Not making this lazy made `TableError::drop` show up in perf. // TODO(centril): Box all the errors. @@ -909,18 +912,14 @@ impl CommittedState { .tables .get_mut(&table_id) .ok_or_else(|| TableError::IdNotFoundState(table_id))?; - Ok(( - table, - &mut self.blob_store as &mut dyn BlobStore, - &mut self.index_id_map, - )) + Ok((table, self.storage.blobs(), &mut self.index_id_map)) } fn make_table(&self, schema: Arc) -> Table { Table::new( schema, SquashedOffset::COMMITTED_STATE, - self.page_manager.clone(), + self.storage.pages().clone(), PageEvictionPolicy::NeverEvict, ) } @@ -933,8 +932,8 @@ impl CommittedState { &'this mut self, table_id: TableId, schema: &Arc, - ) -> (&'this mut Table, &'this mut dyn BlobStore) { - let page_manager = self.page_manager.clone(); + ) -> (&'this mut Table, &'this BlobSet) { + let page_manager = self.storage.pages().clone(); let table = self.tables.entry(table_id).or_insert_with(|| { Table::new( schema.clone(), @@ -943,19 +942,18 @@ impl CommittedState { PageEvictionPolicy::NeverEvict, ) }); - let blob_store = &mut self.blob_store; - (table, blob_store) + (table, self.storage.blobs()) } /// Returns an iterator over all persistent tables (i.e., non-ephemeral tables) #[cfg(feature = "durability")] - pub(super) fn persistent_tables_and_blob_store(&mut self) -> (impl Iterator, &HashMapBlobStore) { + pub(super) fn persistent_tables_and_blob_store(&mut self) -> (impl Iterator, &BlobSet) { ( self.tables .iter_mut() .filter(|(table_id, _)| !self.ephemeral_tables.contains(*table_id)) .map(|(_, table)| table), - &self.blob_store, + self.storage.blobs(), ) } @@ -986,20 +984,20 @@ impl CommittedState { DATA_SIZE_METRICS .data_size_blob_store_num_blobs .with_label_values(&database_identity) - .set(self.blob_store.num_blobs() as _); + .set(self.storage.blobs().num_blobs() as _); DATA_SIZE_METRICS .data_size_blob_store_bytes_used_by_blobs .with_label_values(&database_identity) - .set(self.blob_store.bytes_used_by_blobs() as _); + .set(self.storage.blobs().bytes_used_by_blobs() as _); } } -pub(super) type CommitTableForInsertion<'a> = (&'a Table, &'a dyn BlobStore, &'a IndexIdMap); +pub(super) type CommitTableForInsertion<'a> = (&'a Table, &'a BlobSet, &'a IndexIdMap); -#[derive(Default)] pub(super) struct PreparedMerge { mutated_tables: BTreeMap, removed_tables: Vec, + pinned_blobs: Box<[BlobHandle]>, } struct PreparedRemovedTable { @@ -1009,6 +1007,14 @@ struct PreparedRemovedTable { } impl PreparedMerge { + fn with_pinned_blobs(pinned_blobs: Box<[BlobHandle]>) -> Self { + Self { + mutated_tables: <_>::default(), + removed_tables: <_>::default(), + pinned_blobs, + } + } + fn insert(&mut self, table: TableId, commit: PreparedCommit) -> bool { self.mutated_tables.insert(table, commit).is_none() } diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index 72e43f9ce03..f1e3830d8b9 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -43,7 +43,8 @@ use spacetimedb_schema::{ }; #[cfg(feature = "durability")] use spacetimedb_snapshot::{BoxedPendingSnapshot, DynSnapshotRepo, ReconstructedSnapshot}; -use spacetimedb_table::{indexes::RowPointer, page_pool::PagePool, table::RowRef}; +use spacetimedb_table::tiered::{TieredDatabaseStorage, TieredStorageConfig}; +use spacetimedb_table::{indexes::RowPointer, table::RowRef}; use std::borrow::Cow; use std::sync::Arc; use std::time::Duration; @@ -80,21 +81,21 @@ impl MemoryUsage for Locking { } impl Locking { - pub fn new(database_identity: Identity, page_pool: PagePool) -> Self { + pub fn new(database_identity: Identity, storage: TieredDatabaseStorage) -> Self { Self { - committed_state: Arc::new(RwLock::new(CommittedState::new(page_pool))), + committed_state: Arc::new(RwLock::new(CommittedState::new(storage))), database_identity, } } /// IMPORTANT! This the most delicate function in the entire codebase. /// DO NOT CHANGE UNLESS YOU KNOW WHAT YOU'RE DOING!!! - pub fn bootstrap(database_identity: Identity, page_pool: PagePool) -> Result { + pub fn bootstrap(database_identity: Identity, config: TieredStorageConfig) -> Result { log::trace!("DATABASE: BOOTSTRAPPING SYSTEM TABLES..."); // NOTE! The bootstrapping process does not take plan in a transaction. // This is intentional. - let datastore = Self::new(database_identity, page_pool); + let datastore = Self::new(database_identity, TieredDatabaseStorage::new(config)); let mut commit_state = datastore.committed_state.write_arc(); // TODO(cloutiertyler): One thing to consider in the future is, should // we persist the bootstrap transaction in the message log? My intuition @@ -167,13 +168,13 @@ impl Locking { /// - Construct all the tables referenced by `snapshot`, computing their schemas /// either from known system table schemas or from `st_table` and friends. /// - Populate those tables with all rows in `snapshot`. - /// - Construct a [`HashMapBlobStore`] containing all the large blobs referenced by `snapshot`, + /// - Construct a [`spacetimedb_table::tiered::BlobSet`] containing all the large blobs referenced by `snapshot`, /// with reference counts specified in `snapshot`. /// - Notably, **do not** construct indexes or sequences. /// This should be done by [`Self::rebuild_state_after_replay`], /// after replaying the suffix of the commitlog. #[cfg(feature = "durability")] - pub fn restore_from_snapshot(snapshot: ReconstructedSnapshot, page_pool: PagePool) -> Result { + pub fn restore_from_snapshot(snapshot: ReconstructedSnapshot, config: TieredStorageConfig) -> Result { let ReconstructedSnapshot { database_identity, tx_offset, @@ -182,9 +183,8 @@ impl Locking { .. } = snapshot; - let datastore = Self::new(database_identity, page_pool); + let datastore = Self::new(database_identity, TieredDatabaseStorage::with_blobs(config, blob_store)); let mut committed_state = datastore.committed_state.write_arc(); - committed_state.blob_store = blob_store; // Note that `tables` is a `BTreeMap`, and so iterates in increasing order. // This means that we will instantiate and populate the system tables before any user tables. @@ -1021,10 +1021,11 @@ impl MutTx for Locking { let timer = Instant::now(); let committed_state_write_lock = self.committed_state.write_arc(); let lock_wait_time = timer.elapsed(); + let tx_state = TxState::new(&committed_state_write_lock.storage); MutTxId { committed_state_write_lock, - tx_state: TxState::default(), + tx_state, lock_wait_time, read_sets: <_>::default(), view_instances: <_>::default(), @@ -1055,10 +1056,11 @@ impl Locking { let timer = Instant::now(); let committed_state_write_lock = self.committed_state.try_write_arc()?; let lock_wait_time = timer.elapsed(); + let tx_state = TxState::new(&committed_state_write_lock.storage); Some(MutTxId { committed_state_write_lock, - tx_state: TxState::default(), + tx_state, lock_wait_time, read_sets: <_>::default(), view_instances: <_>::default(), @@ -1253,7 +1255,7 @@ pub(crate) mod tests { } fn get_datastore() -> Result { - Locking::bootstrap(Identity::ZERO, PagePool::new_for_test()) + Locking::bootstrap(Identity::ZERO, TieredStorageConfig::new_for_test()) } fn col(col: u16) -> ColList { @@ -2117,6 +2119,18 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn test_large_blob_insert_post_commit() -> ResultTest<()> { + let (datastore, mut tx, table_id) = setup_table()?; + let large = "x".repeat(2048); + insert(&datastore, &mut tx, table_id, &u32_str_u32(0, &large, 18))?; + datastore.commit_mut_tx(tx)?; + + let tx = begin_mut_tx(&datastore); + assert_eq!(all_rows(&datastore, &tx, table_id), vec![u32_str_u32(1, &large, 18)]); + Ok(()) + } + #[test] fn test_insert_post_rollback() -> ResultTest<()> { let (datastore, tx, table_id) = setup_table()?; diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 68e26a6481f..ca6791a5d7c 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -73,7 +73,6 @@ use spacetimedb_schema::{ table_name::TableName, }; use spacetimedb_table::{ - blob_store::BlobStore, indexes::{RowPointer, SquashedOffset}, static_assert_size, table::{ @@ -81,6 +80,7 @@ use spacetimedb_table::{ TableAndIndex, UniqueConstraintViolation, }, table_index::{IndexCannotSeekRange, IndexKey, IndexSeekRangeResult, PointOrRange, TableIndex}, + tiered::BlobSet, tiered::PageEvictionPolicy, }; use std::{marker::PhantomData, sync::Arc, time::Duration}; @@ -460,7 +460,7 @@ pub struct MutTxId { // Grew by one word when `ReducerName` became fully qualified: it now holds a // `NamespacedIdentifier` (segments + joined rendering) rather than a single `Identifier`. // One per transaction, not per row. -static_assert_size!(MutTxId, 576); +static_assert_size!(MutTxId, 592); impl MutTxId { /// Record that a view performs a table scan in this transaction's read set @@ -1138,7 +1138,7 @@ impl MutTxId { let commit_table = Table::new( schema, SquashedOffset::COMMITTED_STATE, - self.committed_state_write_lock.page_manager.clone(), + self.committed_state_write_lock.storage.pages().clone(), PageEvictionPolicy::NeverEvict, ); let tx_table = commit_table.clone_structure(SquashedOffset::TX_STATE); @@ -1386,10 +1386,7 @@ impl MutTxId { fn get_or_create_insert_table_mut( &mut self, table_id: TableId, - ) -> Result<( - TxTableForInsertion<'_>, - (&mut Table, &mut dyn BlobStore, &mut IndexIdMap), - )> { + ) -> Result<(TxTableForInsertion<'_>, (&mut Table, &BlobSet, &mut IndexIdMap))> { let (commit_table, commit_bs, idx_map) = self.committed_state_write_lock.get_table_and_blob_store_mut(table_id)?; // Get the insert table, so we can write the row into it. @@ -1842,20 +1839,18 @@ impl MutTxId { self.get_or_create_insert_table_mut(table_id)?; // Create and build the indices. - let map_violation = - |violation, index: &TableIndex, table: &Table, bs: &dyn BlobStore| -> Result { - let violation = table - .get_row_ref(bs, violation)? - .expect("row came from scanning the table"); - let violation = index.project_row(&violation); - let schema = table.get_schema(); - let violation = - UniqueConstraintViolation::build_with_index_schema(schema, index, &index_schema, violation); - Ok(IndexError::from(violation).into()) - }; + let map_violation = |violation, index: &TableIndex, table: &Table, bs: &BlobSet| -> Result { + let violation = table + .get_row_ref(bs, violation)? + .expect("row came from scanning the table"); + let violation = index.project_row(&violation); + let schema = table.get_schema(); + let violation = UniqueConstraintViolation::build_with_index_schema(schema, index, &index_schema, violation); + Ok(IndexError::from(violation).into()) + }; // Builds the index and ensures that `table`'s row won't cause a unique constraint violation // due to the existing rows having the same value for some column(s). - let build_from_rows = |index: &mut TableIndex, table: &Table, bs: &dyn BlobStore| -> Result<()> { + let build_from_rows = |index: &mut TableIndex, table: &Table, bs: &BlobSet| -> Result<()> { let rows = table.scan_rows(bs); // SAFETY: (1) `tx_index` / `commit_index` was derived from `table` / `commit_table` // which in turn was derived from `commit_table`. @@ -3771,7 +3766,7 @@ pub(super) fn insert<'a, const GENERATE: bool>( // Unwrap these error into `TableError::{IndexError, Bflatn}`: Err(InsertError::IndexError(e)) => Err(IndexError::from(e).into()), Err(InsertError::Bflatn(e)) => Err(TableError::Bflatn(e).into()), - Err(InsertError::Page(e)) => Err(e.into()), + Err(InsertError::Table(e)) => Err(e.into()), } } diff --git a/crates/datastore/src/locking_tx_datastore/replay.rs b/crates/datastore/src/locking_tx_datastore/replay.rs index 3338bdb0e56..1aeabdb44ef 100644 --- a/crates/datastore/src/locking_tx_datastore/replay.rs +++ b/crates/datastore/src/locking_tx_datastore/replay.rs @@ -661,13 +661,13 @@ impl<'cs> ReplayCommittedState<'cs> { pub(super) fn build_indexes(&mut self) -> Result<()> { let st_indexes = self.tables.get(&ST_INDEX_ID).unwrap(); let rows = st_indexes - .scan_rows(&self.blob_store) + .scan_rows(self.storage.blobs()) .map(|row_ref| row_ref.map_err(Into::into).and_then(StIndexRow::try_from)) .collect::>>()?; let st_constraints = self.tables.get(&ST_CONSTRAINT_ID).unwrap(); let unique_constraints: HashSet<(TableId, ColSet)> = st_constraints - .scan_rows(&self.blob_store) + .scan_rows(self.storage.blobs()) .map(|row_ref| row_ref.map_err(Into::into).and_then(StConstraintRow::try_from)) .filter_map(Result::ok) .filter_map(|constraint| match constraint.constraint_data { @@ -707,7 +707,7 @@ impl<'cs> ReplayCommittedState<'cs> { return Ok(tables); }; let backing_tables = st_view - .scan_rows(&self.blob_store) + .scan_rows(self.storage.blobs()) .map(|row_ref| { let StViewRow { table_id, view_id, .. } = StViewRow::try_from(row_ref?)?; table_id.ok_or_else(|| DatastoreError::View(ViewError::TableNotFound(view_id))) @@ -741,7 +741,7 @@ impl<'cs> ReplayCommittedState<'cs> { pub(super) fn build_missing_tables(&mut self) -> Result<()> { // Find all ids of tables that are in `st_tables` but haven't been built. let mut table_ids = Vec::new(); - for row in self.get_table(ST_TABLE_ID).unwrap().scan_rows(&self.blob_store) { + for row in self.get_table(ST_TABLE_ID).unwrap().scan_rows(self.storage.blobs()) { let row_ref = row?; let table_id = row_ref.read_col(StTableFields::TableId).unwrap(); if self.get_table(table_id).is_none() { @@ -782,7 +782,7 @@ impl<'cs> ReplayCommittedState<'cs> { } Err(InsertError::Bflatn(e)) => return Err(TableError::Bflatn(e).into()), Err(InsertError::IndexError(e)) => return Err(IndexError::UniqueConstraintViolation(e).into()), - Err(InsertError::Page(e)) => return Err(DatastoreError::Page(e)), + Err(InsertError::Table(e)) => return Err(e.into()), }; // `row_ref` is treated as having a mutable borrow on `self` @@ -1164,7 +1164,7 @@ mod tests { use spacetimedb_lib::db::auth::{StAccess, StTableType}; use spacetimedb_lib::error::ResultTest; use spacetimedb_sats::{product, AlgebraicType}; - use spacetimedb_table::page_pool::PagePool; + use spacetimedb_table::tiered::TieredStorageConfig; use super::*; @@ -1199,7 +1199,7 @@ mod tests { #[test] fn test_replay_updated_st_table_lookup_reads_st_table() -> ResultTest<()> { - let datastore = Locking::bootstrap(Identity::ZERO, PagePool::new_for_test())?; + let datastore = Locking::bootstrap(Identity::ZERO, TieredStorageConfig::new_for_test())?; let mut tx = begin_mut_tx(&datastore); let schema = TableSchema::new( diff --git a/crates/datastore/src/locking_tx_datastore/state_view.rs b/crates/datastore/src/locking_tx_datastore/state_view.rs index 85e9812b17a..577cc92cc50 100644 --- a/crates/datastore/src/locking_tx_datastore/state_view.rs +++ b/crates/datastore/src/locking_tx_datastore/state_view.rs @@ -18,8 +18,8 @@ use spacetimedb_primitives::{ColList, TableId}; use spacetimedb_sats::AlgebraicValue; use spacetimedb_schema::schema::{ColumnSchema, IndexSchema, TableSchema, ViewDefInfo}; use spacetimedb_table::{ - blob_store::HashMapBlobStore, table::{self, RowRef, Table}, + tiered::BlobSet, }; use std::marker::PhantomData; use std::sync::Arc; @@ -374,7 +374,7 @@ pub(crate) fn iter_st_column_for_table<'a>( } pub struct IterMutTx<'a> { - tx_state_ins: Option<(&'a Table, &'a HashMapBlobStore)>, + tx_state_ins: Option<(&'a Table, &'a BlobSet)>, stage: ScanStage<'a>, } @@ -391,7 +391,7 @@ impl<'a> IterMutTx<'a> { .get(&table_id) .map(|table| (table, &tx_state.blob_store)); - let iter = commit_table.scan_rows(&committed_state.blob_store).err_into(); + let iter = commit_table.scan_rows(committed_state.storage.blobs()).err_into(); let stage = if let Some(deletes) = tx_state.get_delete_table(table_id) { // There are deletes in the tx state // so we must exclude those (1b). diff --git a/crates/datastore/src/locking_tx_datastore/tx.rs b/crates/datastore/src/locking_tx_datastore/tx.rs index d038324aea0..6fbf5ab4907 100644 --- a/crates/datastore/src/locking_tx_datastore/tx.rs +++ b/crates/datastore/src/locking_tx_datastore/tx.rs @@ -1,7 +1,7 @@ use super::{ committed_state::CommittedState, datastore::{Result, TxMetrics}, - state_view::{IterByColRangeTx, StateView}, + state_view::{IterByColRangeTx, StateView, TableScanIter}, time::Instant, IterByColEqTx, SharedReadGuard, }; @@ -144,7 +144,7 @@ impl TxId { .ok_or_else(|| anyhow::anyhow!("TableId `{table_id}` does not exist")) .and_then(|table| { table - .get_index_by_id_with_table(&self.committed_state_shared_lock.blob_store, index_id) + .get_index_by_id_with_table(self.committed_state_shared_lock.storage.blobs(), index_id) .map(seek) .ok_or_else(|| anyhow::anyhow!("IndexId `{index_id}` does not exist")) }) diff --git a/crates/datastore/src/locking_tx_datastore/tx_state.rs b/crates/datastore/src/locking_tx_datastore/tx_state.rs index 91f69d7a239..e679451af85 100644 --- a/crates/datastore/src/locking_tx_datastore/tx_state.rs +++ b/crates/datastore/src/locking_tx_datastore/tx_state.rs @@ -4,16 +4,17 @@ use spacetimedb_lib::db::auth::StAccess; use spacetimedb_primitives::{ColId, ColList, ConstraintId, IndexId, SequenceId, TableId}; use spacetimedb_sats::memory_usage::MemoryUsage; use spacetimedb_sats::raw_identifier::RawNamespacedIdentifier; -use spacetimedb_schema::identifier::{Identifier, NamespacedIdentifier}; -use spacetimedb_schema::schema::{ColumnSchema, ConstraintSchema, IndexSchema, OutboxSchema, SequenceSchema}; +use spacetimedb_schema::{ + identifier::{Identifier, NamespacedIdentifier}, + schema::{ColumnSchema, ConstraintSchema, IndexSchema, OutboxSchema, SequenceSchema}, +}; use spacetimedb_table::{ - blob_store::{BlobStore, HashMapBlobStore}, indexes::{RowPointer, SquashedOffset}, pointer_map::PointerMap, static_assert_size, - table::{RowRef, Table, TableAndIndex}, + table::{RowRef, Table, TableAndIndex, TableError}, table_index::TableIndex, - tiered::PageError, + tiered::{BlobSet, TieredDatabaseStorage}, }; use std::collections::{btree_map, BTreeMap}; use std::iter::Peekable; @@ -48,7 +49,6 @@ pub(super) type IndexIdMap = IntMap; /// - any row in `insert_tables` must not be in the associated `CommittedState` /// - any row in `delete_tables` must be in the associated `CommittedState` /// - any row cannot be in both `insert_tables` and `delete_tables` -#[derive(Default)] pub(super) struct TxState { //NOTE: Need to preserve order to correctly restore the db after reopen /// For any `TableId` that has had a row inserted into it in this TX @@ -72,7 +72,7 @@ pub(super) struct TxState { /// - Maintain the set of newly-referenced blob hashes in the `TxState`, /// and free each of them during rollback. /// - Traverse all rows in the `insert_tables` and free each of their blobs during rollback. - pub(super) blob_store: HashMapBlobStore, + pub(super) blob_store: BlobSet, /// All of the immediately applied schema changes to the committed state during this transaction. /// @@ -80,7 +80,7 @@ pub(super) struct TxState { pub(super) pending_schema_changes: ThinVec, } -static_assert_size!(TxState, 96); +static_assert_size!(TxState, 112); impl MemoryUsage for TxState { fn heap_usage(&self) -> usize { @@ -107,6 +107,7 @@ impl MemoryUsage for TxState { /// Architecting this way should benefit performance both during transactions and merge. /// On rollback, it should be fairly cheap to e.g., just re-add an index or drop it on the floor. #[derive(Debug)] +#[allow(clippy::large_enum_variant)] pub enum PendingSchemaChange { /// The [`TableIndex`] / [`IndexSchema`] with `IndexId` /// was removed from the table with [`TableId`]. @@ -251,7 +252,7 @@ impl TxState { // with `table_id` // and has not been passed to `self.delete` // is sufficient to demonstrate that a call to `self.get` is safe. - pub(super) fn get(&self, table_id: TableId, row_ptr: RowPointer) -> Result, PageError> { + pub(super) fn get(&self, table_id: TableId, row_ptr: RowPointer) -> Result, TableError> { debug_assert!( row_ptr.squashed_offset().is_tx_state(), "Cannot get COMMITTED_STATE row_ptr from TxState.", @@ -288,7 +289,7 @@ impl TxState { get_delete_table_mut(&mut self.delete_tables, table_id, commit_table) } - pub(super) fn get_table_and_blob_store(&mut self, table_id: TableId) -> Option<(&mut Table, &mut dyn BlobStore)> { + pub(super) fn get_table_and_blob_store(&mut self, table_id: TableId) -> Option<(&mut Table, &mut BlobSet)> { let table = self.insert_tables.get_mut(&table_id)?; let blob_store = &mut self.blob_store; Some((table, blob_store)) @@ -318,7 +319,7 @@ impl TxState { /// /// The insert and delete tables must exist. pub unsafe fn assume_present_get_mut_table(&mut self, table_id: TableId) -> TxTableForInsertion<'_> { - let tx_blob_store: &mut dyn BlobStore = &mut self.blob_store; + let tx_blob_store = &mut self.blob_store; let tx_table = self.insert_tables.get_mut(&table_id); // SAFETY: we successfully got a `tx_table` before and haven't removed it since. let tx_table = unsafe { tx_table.unwrap_unchecked() }; @@ -415,7 +416,7 @@ impl<'a> Iterator for TxTables<'a> { } } -pub(super) type TxTableForInsertion<'a> = (&'a mut Table, &'a mut dyn BlobStore, &'a mut DeleteTable); +pub(super) type TxTableForInsertion<'a> = (&'a mut Table, &'a mut BlobSet, &'a mut DeleteTable); fn get_delete_table_mut<'a>( delete_tables: &'a mut BTreeMap, diff --git a/crates/dst/src/engine.rs b/crates/dst/src/engine.rs index 451b7417e41..3a121188d5a 100644 --- a/crates/dst/src/engine.rs +++ b/crates/dst/src/engine.rs @@ -12,7 +12,7 @@ use spacetimedb_runtime::sim::{Rng, Runtime as SimRuntime}; use spacetimedb_runtime::Handle; use spacetimedb_schema::def::ModuleDef; use spacetimedb_schema::schema::{Schema, TableSchema}; -use spacetimedb_table::page_pool::PagePool; +use spacetimedb_table::tiered::TieredStorageConfig; mod model; mod properties; @@ -66,7 +66,7 @@ impl EngineTarget { history, Some(persistence), None, - PagePool::new_for_test(), + TieredStorageConfig::new_for_test(), )?; anyhow::ensure!(connected_clients.is_empty(), "replay produced connected clients"); Ok(db) diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index af2a76e8eb7..ae31bf25e66 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -64,7 +64,7 @@ use spacetimedb_table::indexes::RowPointer; use spacetimedb_table::page_pool::PagePool; use spacetimedb_table::table::RowRef; use spacetimedb_table::table_index::IndexKey; -use spacetimedb_table::tiered::PageError; +use spacetimedb_table::tiered::TieredStorageConfig; use std::borrow::Cow; use std::io; use std::ops::RangeBounds; @@ -286,7 +286,7 @@ impl RelationalDB { history: impl durability::History, mut persistence: Option, metrics_recorder_queue: Option, - page_pool: PagePool, + storage_config: TieredStorageConfig, ) -> Result<(Self, ConnectedClients), DBError> { log::trace!("[{database_identity}] DATABASE: OPEN"); @@ -310,7 +310,7 @@ impl RelationalDB { snapshot_repo.as_deref(), durable_tx_offset, min_commitlog_offset, - page_pool, + storage_config, )?; if let Some(persistence) = &mut persistence { // Sanity check because the snapshot worker could've been used before. @@ -565,7 +565,7 @@ impl RelationalDB { snapshot_repo: Option<&DynSnapshotRepo>, durable_tx_offset: Option, min_commitlog_offset: TxOffset, - page_pool: PagePool, + config: TieredStorageConfig, ) -> Result { // Try to load the `ReconstructedSnapshot` at `snapshot_offset`. fn try_load_snapshot( @@ -622,11 +622,11 @@ impl RelationalDB { fn restore_from_snapshot( database_identity: &Identity, snapshot: ReconstructedSnapshot, - page_pool: PagePool, + config: TieredStorageConfig, ) -> Result> { let start = std::time::Instant::now(); let snapshot_offset = snapshot.tx_offset; - Locking::restore_from_snapshot(snapshot, page_pool) + Locking::restore_from_snapshot(snapshot, config) .inspect(|_| { let elapsed_time = start.elapsed(); @@ -664,15 +664,9 @@ impl RelationalDB { | SnapshotError::BadMagic { .. } | SnapshotError::BadVersion { .. } => false, - SnapshotError::Page(inner) => match inner { - PageError::MemoryLimitExceeded(_) | PageError::Io(_) => true, - - PageError::TooManyPages - | PageError::Page(_) - | PageError::MissingPage(_) - | PageError::MissingObject(_) - | PageError::Deserialize(_) => false, - }, + SnapshotError::Page(inner) => inner.is_transient(), + SnapshotError::Blob(inner) => inner.is_transient(), + SnapshotError::ObjectStore(inner) => inner.is_transient(), } } @@ -698,7 +692,7 @@ impl RelationalDB { log::debug!("snapshot_offset={snapshot_offset} min_commitlog_offset={min_commitlog_offset}"); break; } - match try_load_snapshot(&database_identity, snapshot_repo, snapshot_offset, &page_pool) { + match try_load_snapshot(&database_identity, snapshot_repo, snapshot_offset, &config.page_pool) { Ok(snapshot) if snapshot.database_identity != database_identity => { return Err(RestoreSnapshotError::IdentityMismatch { expected: database_identity, @@ -706,7 +700,7 @@ impl RelationalDB { }); } Ok(snapshot) => { - return restore_from_snapshot(&database_identity, snapshot, page_pool) + return restore_from_snapshot(&database_identity, snapshot, config) .map_err(RestoreSnapshotError::Datastore); } Err(e) => { @@ -743,7 +737,7 @@ impl RelationalDB { return Err(RestoreSnapshotError::NoConnectedSnapshot { min_commitlog_offset }); } - Locking::bootstrap(database_identity, page_pool) + Locking::bootstrap(database_identity, config) .map_err(DBError::from) .map_err(Box::new) .map_err(RestoreSnapshotError::Bootstrap) @@ -2203,7 +2197,7 @@ pub mod tests_utils { local.as_history(), Some(persistence), None, - PagePool::new_for_test(), + TieredStorageConfig::new_for_test(), )?; let db = db.with_row_count(Self::row_count_fn()); Ok((db, local)) @@ -2338,7 +2332,7 @@ pub mod tests_utils { history, persistence, metrics_recorder_queue, - PagePool::new_for_test(), + TieredStorageConfig::new_for_test(), )?; assert_eq!(connected_clients.len(), expected_num_clients); let db = db.with_row_count(Self::row_count_fn()); @@ -4179,7 +4173,7 @@ mod tests { Some(repo.as_ref()), Some(last_compress), 0, - PagePool::new_for_test(), + TieredStorageConfig::new_for_test(), )?; Ok(()) @@ -4209,7 +4203,7 @@ mod tests { Some(repo.as_ref()), last, 0, - PagePool::new_for_test(), + TieredStorageConfig::new_for_test(), )?; let out = TempDir::with_prefix("snapshot_test")?; @@ -4259,7 +4253,7 @@ mod tests { Some(&repo), Some(durable_tx_offset), min_commitlog_offset, - PagePool::new_for_test(), + TieredStorageConfig::new_for_test(), ) }; diff --git a/crates/execution/src/lib.rs b/crates/execution/src/lib.rs index e3eceb93ed2..c69e231e32e 100644 --- a/crates/execution/src/lib.rs +++ b/crates/execution/src/lib.rs @@ -297,7 +297,7 @@ pub enum Tuple<'a> { Join(Vec>), } -static_assert_size!(Tuple, 48); +static_assert_size!(Tuple, 40); impl ProjectField for Tuple<'_> { fn project(&self, field: &TupleField) -> AlgebraicValue { diff --git a/crates/portable-datastore/src/lib.rs b/crates/portable-datastore/src/lib.rs index b4f70ba39e0..a9519788fa9 100644 --- a/crates/portable-datastore/src/lib.rs +++ b/crates/portable-datastore/src/lib.rs @@ -22,6 +22,7 @@ use spacetimedb_schema::schema::{Schema, TableOrViewSchema, TableSchema}; use spacetimedb_table::indexes::RowPointer; use spacetimedb_table::page_pool::PagePool; use spacetimedb_table::table::RowRef; +use spacetimedb_table::tiered::TieredStorageConfig; use thiserror::Error; /// Commit-time behavior for a pending transaction. @@ -61,7 +62,10 @@ impl PortableDatastore { /// Create an in-memory datastore initialized with the tables in `raw`. pub fn from_module_def(raw: RawModuleDef, module_identity: Identity) -> Result { let module_def = ModuleDef::try_from(raw.clone())?; - let datastore = Locking::bootstrap(module_identity, PagePool::new(None))?; + let datastore = Locking::bootstrap( + module_identity, + TieredStorageConfig::new_for_memory_database(PagePool::new(None)), + )?; let mut tx = datastore.begin_mut_tx(IsolationLevel::Serializable, Workload::Internal); let mut event_table_ids = Vec::new(); let mut table_ids = HashMap::new(); diff --git a/crates/snapshot/src/lib.rs b/crates/snapshot/src/lib.rs index a150d418b1f..a6d8f0469b8 100644 --- a/crates/snapshot/src/lib.rs +++ b/crates/snapshot/src/lib.rs @@ -38,15 +38,11 @@ use spacetimedb_paths::server::{ArchivedSnapshotDirPath, SnapshotDirPath, Snapsh use spacetimedb_paths::FromPathUnchecked; use spacetimedb_primitives::TableId; use spacetimedb_sats::{bsatn, de::Deserialize, ser::Serialize}; -use spacetimedb_table::tiered::PageError; -use spacetimedb_table::{ - blob_store::{BlobHash, BlobStore, HashMapBlobStore}, - page::Page, - page_pool::PagePool, - table::Table, -}; +use spacetimedb_table::tiered::{BlobError, BlobMap, BlobSet, ObjectStoreError, PageError}; +use spacetimedb_table::{blob_store::BlobHash, page::Page, page_pool::PagePool, table::Table}; use std::fs::{self, File}; use std::io; +use std::num::NonZeroUsize; use std::ops::{Range, RangeBounds}; use std::path::Path; use std::time::{Duration, Instant}; @@ -175,6 +171,10 @@ pub enum SnapshotError { Io(#[from] std::io::Error), #[error(transparent)] Page(#[from] PageError), + #[error(transparent)] + Blob(#[from] BlobError), + #[error(transparent)] + ObjectStore(#[from] ObjectStoreError), } impl SnapshotError { @@ -363,7 +363,7 @@ pub struct Snapshot { } impl Snapshot { - /// Insert a single large blob from a [`BlobStore`] + /// Insert a single large blob from a [`BlobSet`] /// into the in-memory snapshot `self` /// and the on-disk object repository `object_repo`. /// @@ -399,13 +399,14 @@ impl Snapshot { fn write_all_blobs( &mut self, object_repo: &DirTrie, - blobs: &dyn BlobStore, + blobs: &BlobSet, prev_snapshot: Option<&DirTrie>, counter: &mut CountCreated, ) -> Result<(), SnapshotError> { - for (hash, uses, blob) in blobs.iter_blobs() { - self.write_blob(object_repo, hash, uses, blob, prev_snapshot, counter)?; - } + blobs.try_for_each(|hash, uses, blob| { + self.write_blob(object_repo, &hash, uses.get(), blob, prev_snapshot, counter) + })?; + Ok(()) } @@ -570,7 +571,7 @@ impl Snapshot { Ok((buf, disk_bytes)) } - /// Construct a [`HashMapBlobStore`] containing all the blobs referenced in `self`, + /// Construct a [`BlobMap`] containing all the blobs referenced in `self`, /// reading their data from files in the `object_repo`. /// /// Fails if any of the object files is missing or corrupted, @@ -579,8 +580,8 @@ impl Snapshot { &self, object_repo: &DirTrie, metrics: &mut SnapshotReadKindMetrics, - ) -> Result { - let mut blob_store = HashMapBlobStore::default(); + ) -> Result { + let mut blob_store = BlobMap::default(); for BlobEntry { hash, uses } in &self.blobs { // Read the bytes of the blob object. @@ -604,7 +605,9 @@ impl Snapshot { }); } - blob_store.insert_with_uses(hash, *uses as usize, buf.into_boxed_slice()); + if let Some(uses) = NonZeroUsize::new(*uses as usize) { + blob_store.insert_with_uses(hash, uses, buf.into()); + } } Ok(blob_store) @@ -884,7 +887,7 @@ impl SnapshotRepository { pub fn create_snapshot<'db>( &self, tables: impl Iterator, - blobs: &'db dyn BlobStore, + blobs: &BlobSet, tx_offset: TxOffset, ) -> Result { // Invalidate equal to or newer than `tx_offset`. @@ -1506,10 +1509,10 @@ pub trait SnapshotRepo: Send + Sync { fn database_identity(&self) -> Identity; /// Start creating a snapshot at `tx_offset` from the provided tables and blob store. - fn create_snapshot<'db>( + fn create_snapshot( &self, - tables: &mut dyn Iterator, - blobs: &'db dyn BlobStore, + tables: &mut dyn Iterator, + blobs: &BlobSet, tx_offset: TxOffset, ) -> Result; @@ -1547,10 +1550,10 @@ impl SnapshotRepo for SnapshotRepository { SnapshotRepository::database_identity(self) } - fn create_snapshot<'db>( + fn create_snapshot( &self, - tables: &mut dyn Iterator, - blobs: &'db dyn BlobStore, + tables: &mut dyn Iterator, + blobs: &BlobSet, tx_offset: TxOffset, ) -> Result { Ok(Box::new(SnapshotRepository::create_snapshot( @@ -1594,7 +1597,7 @@ pub struct ReconstructedSnapshot { pub module_abi_version: [u16; 2], /// The blob store of the snapshotted state. - pub blob_store: HashMapBlobStore, + pub blob_store: BlobMap, /// All the tables from the snapshotted state, sans schema information and indexes. /// diff --git a/crates/snapshot/src/remote.rs b/crates/snapshot/src/remote.rs index 2c35e1ada79..0a66cda3a2c 100644 --- a/crates/snapshot/src/remote.rs +++ b/crates/snapshot/src/remote.rs @@ -833,9 +833,7 @@ mod tests { use spacetimedb_paths::{server::SnapshotsPath, FromPathUnchecked}; use spacetimedb_sats::bsatn; use spacetimedb_sats::layout::{row_size_for_type, Size}; - use spacetimedb_table::{ - blob_store::NullBlobStore, indexes::PageOffset, page::Page, var_len::AlignedVarLenOffsets, - }; + use spacetimedb_table::{indexes::PageOffset, page::Page, tiered::BlobSet, var_len::AlignedVarLenOffsets}; use std::io::Cursor; use tempfile::tempdir; use zstd_framed::AsyncZstdWriter; @@ -981,8 +979,15 @@ mod tests { fn insert_u64(page: &mut Page, val: u64) -> PageOffset { let val_slice = val.to_le_bytes(); - unsafe { page.insert_row(&val_slice, &[] as &[&[u8]], u64_var_len_visitor(), &mut NullBlobStore) } - .expect("Failed to insert first row") + unsafe { + page.insert_row( + &val_slice, + &[] as &[&[u8]], + u64_var_len_visitor(), + &mut BlobSet::new_for_test(), + ) + } + .expect("Failed to insert first row") } const U64_VL_VISITOR: AlignedVarLenOffsets<'_> = AlignedVarLenOffsets::from_offsets(&[]); diff --git a/crates/snapshot/tests/remote.rs b/crates/snapshot/tests/remote.rs index 7edd20002d4..331f3312020 100644 --- a/crates/snapshot/tests/remote.rs +++ b/crates/snapshot/tests/remote.rs @@ -32,7 +32,7 @@ use spacetimedb_snapshot::{ remote::{synchronize_snapshot, verify_snapshot}, Snapshot, SnapshotError, SnapshotRepository, }; -use spacetimedb_table::page_pool::PagePool; +use spacetimedb_table::{page_pool::PagePool, tiered::TieredStorageConfig}; use tempfile::{tempdir, TempDir}; use tokio::{sync::OnceCell, task::spawn_blocking}; @@ -65,7 +65,7 @@ async fn can_sync_a_snapshot() -> anyhow::Result<()> { assert!(read_metrics.metadata.disk_bytes > 0); assert_eq!(read_metrics.page.files + read_metrics.blob.files, total_objects); assert!(read_metrics.page.disk_bytes + read_metrics.blob.disk_bytes > 0); - Locking::restore_from_snapshot(dst_snapshot_full, pool)?; + Locking::restore_from_snapshot(dst_snapshot_full, TieredStorageConfig::new_for_module_check(pool))?; // Let's also check that running `synchronize_snapshot` again does nothing. let stats = synchronize_snapshot(blob_provider.clone(), dst_path.clone(), src_snapshot.clone()).await?; diff --git a/crates/table/benches/page.rs b/crates/table/benches/page.rs index 668685a5dd7..e177b2c0df0 100644 --- a/crates/table/benches/page.rs +++ b/crates/table/benches/page.rs @@ -11,13 +11,13 @@ use spacetimedb_sats::layout::{row_size_for_type, RowTypeLayout}; use spacetimedb_sats::{product, AlgebraicType, AlgebraicValue, ArrayValue, ProductType, ProductValue}; use spacetimedb_table::bflatn_from::serialize_row_from_page; use spacetimedb_table::bflatn_to::write_row_to_page; -use spacetimedb_table::blob_store::NullBlobStore; use spacetimedb_table::eq::eq_row_in_page; use spacetimedb_table::indexes::{Byte, Bytes, PageOffset, RowHash}; use spacetimedb_table::page::Page; use spacetimedb_table::row_hash::hash_row_in_page; use spacetimedb_table::row_type_visitor::{row_type_visitor, VarLenVisitorProgram}; use spacetimedb_table::static_layout::StaticLayout; +use spacetimedb_table::tiered::{BlobReadSet, BlobSet}; use spacetimedb_table::var_len::{AlignedVarLenOffsets, NullVarLenVisitor, VarLenGranule, VarLenMembers, VarLenRef}; fn time(acc: &mut Duration, body: impl FnOnce() -> R) -> R { @@ -157,7 +157,7 @@ fn var_len_rows_per_page(data_size_in_bytes: usize) -> usize { fn fill_with_fixed_len(page: &mut Page, val: Row, visitor: &impl VarLenMembers) { while black_box(&*page).has_space_for_row(row_size_for_type::(), 0) { let _ = black_box(unsafe { - black_box(&mut *page).insert_row(val.as_bytes(), &[] as &[&[u8]], visitor, &mut NullBlobStore) + black_box(&mut *page).insert_row(val.as_bytes(), &[] as &[&[u8]], visitor, &mut BlobSet::new_for_test()) }); } } @@ -211,7 +211,7 @@ fn delete_all_shuffled(page: &mut Page, rng: &mut impl Rng) { let visitor = R::var_len_visitor(); for row in row_offsets { - unsafe { page.delete_row(row, row_size_for_type::(), &visitor, &mut NullBlobStore) }; + let _ = unsafe { page.delete_row(row, row_size_for_type::(), &visitor, &BlobSet::new_for_test()) }; } assert_eq!(page.num_rows(), 0); } @@ -219,7 +219,12 @@ fn delete_all_shuffled(page: &mut Page, rng: &mut impl Rng) { fn fill_with_var_len(page: &mut Page, var_len: &[u8], visitor: &impl VarLenMembers) { while black_box(&*page).has_space_for_row_with_objects(row_size_for_type::(), &[var_len]) { let _ = black_box(unsafe { - black_box(&mut *page).insert_row(as_bytes(&VarLenRef::NULL), &[var_len], visitor, &mut NullBlobStore) + black_box(&mut *page).insert_row( + as_bytes(&VarLenRef::NULL), + &[var_len], + visitor, + &mut BlobSet::new_for_test(), + ) }); } } @@ -341,14 +346,14 @@ fn insert_opt_str(c: &mut Criterion) { break; } let _ = unsafe { - page.insert_row(&variant_none, &[] as &[&[u8]], &visitor, &mut NullBlobStore) + page.insert_row(&variant_none, &[] as &[&[u8]], &visitor, &mut BlobSet::new_for_test()) }; } else { if !page.has_space_for_row_with_objects(fixed_row_size, &[&var_len_data]) { break; } let _ = unsafe { - page.insert_row(&variant_some, &[&var_len_data], &visitor, &mut NullBlobStore) + page.insert_row(&variant_some, &[&var_len_data], &visitor, &mut BlobSet::new_for_test()) }; } }; @@ -376,7 +381,7 @@ fn delete_to_approx_fullness_ratio(page: &mut Page, fullness: for row in row_offsets.into_iter() { let should_keep = rng.random_bool(fullness); if !should_keep { - unsafe { page.delete_row(row, row_size_for_type::(), &visitor, &mut NullBlobStore) }; + let _ = unsafe { page.delete_row(row, row_size_for_type::(), &visitor, &BlobSet::new_for_test()) }; } } } @@ -694,7 +699,7 @@ fn time_insert_one( value.clone() }; iter_time_with_page(b, page, pre, |val, _, page| unsafe { - let _ = black_box(write_row_to_page(page, &mut NullBlobStore, visitor, ty, &val)); + let _ = black_box(write_row_to_page(page, &BlobSet::new_for_test(), visitor, ty, &val)); }); } @@ -730,8 +735,9 @@ fn insert_product_value_into_page(c: &mut Criterion) { } fn time_extract_one(b: &mut Bencher, page: &mut Page, offset: PageOffset, ty: &RowTypeLayout) { - let body = - |_, _, page: &mut _| unsafe { serialize_row_from_page(ValueSerializer, page, &NullBlobStore, offset, ty) }; + let body = |_, _, page: &mut _| unsafe { + serialize_row_from_page(ValueSerializer, page, &BlobReadSet::empty(), offset, ty) + }; iter_time_with_page(b, page, |_| (), body); } @@ -740,7 +746,8 @@ fn extract_product_value_from_page(c: &mut Criterion) { for (name, ty, value, _null_visitor, _aligned_offsets_visitor) in product_value_test_cases() { let mut group = c.benchmark_group("extract_product_value"); let (ty, mut page, visitor) = ty_page_visitor(ty); - let (offset, _) = unsafe { write_row_to_page(&mut page, &mut NullBlobStore, &visitor, &ty, &value) }.unwrap(); + let (offset, _) = + unsafe { write_row_to_page(&mut page, &BlobSet::new_for_test(), &visitor, &ty, &value) }.unwrap(); group.bench_function(name, |b| time_extract_one(b, &mut page, offset, &ty)); } } @@ -755,8 +762,9 @@ fn eq_in_page_same(c: &mut Criterion) { let (ty, mut page, visitor) = ty_page_visitor(ty); let static_bsatn_layout = StaticLayout::for_row_type(&ty); - let (offset_0, _) = unsafe { write_row_to_page(&mut page, &mut NullBlobStore, &visitor, &ty, &value) }.unwrap(); - let (offset_1, _) = unsafe { write_row_to_page(&mut page, &mut NullBlobStore, &visitor, &ty, &value) }.unwrap(); + let blobs = BlobSet::new_for_test(); + let (offset_0, _) = unsafe { write_row_to_page(&mut page, &blobs, &visitor, &ty, &value) }.unwrap(); + let (offset_1, _) = unsafe { write_row_to_page(&mut page, &blobs, &visitor, &ty, &value) }.unwrap(); group.bench_function(name, |b| { b.iter(|| { @@ -784,7 +792,8 @@ fn hash_in_page(c: &mut Criterion) { for (name, ty, value, _null_visitor, _aligned_offsets_visitor) in product_value_test_cases() { let (ty, mut page, visitor) = ty_page_visitor(ty); - let (offset, _) = unsafe { write_row_to_page(&mut page, &mut NullBlobStore, &visitor, &ty, &value) }.unwrap(); + let (offset, _) = + unsafe { write_row_to_page(&mut page, &BlobSet::new_for_test(), &visitor, &ty, &value) }.unwrap(); group.bench_function(name, |b| { let mut hasher = RowHash::hasher_builder().build_hasher(); b.iter(|| { @@ -792,7 +801,7 @@ fn hash_in_page(c: &mut Criterion) { hash_row_in_page( &mut hasher, black_box(&page), - black_box(&NullBlobStore), + black_box(&BlobReadSet::empty()), black_box(offset), black_box(&ty), ) diff --git a/crates/table/benches/page_manager.rs b/crates/table/benches/page_manager.rs index 55c72864cce..ce96994b1d5 100644 --- a/crates/table/benches/page_manager.rs +++ b/crates/table/benches/page_manager.rs @@ -15,13 +15,12 @@ use spacetimedb_schema::def::BTreeAlgorithm; use spacetimedb_schema::def::ModuleDef; use spacetimedb_schema::schema::TableSchema; use spacetimedb_schema::table_name::TableName; -use spacetimedb_table::blob_store::NullBlobStore; use spacetimedb_table::indexes::{Byte, Bytes, PageOffset, RowPointer, SquashedOffset}; use spacetimedb_table::page_pool::PagePool; use spacetimedb_table::pages::Pages; use spacetimedb_table::row_type_visitor::{row_type_visitor, VarLenVisitorProgram}; use spacetimedb_table::table::Table; -use spacetimedb_table::tiered::{PageEvictionPolicy, PageManager}; +use spacetimedb_table::tiered::{BlobSet, PageEvictionPolicy, PageManager}; use spacetimedb_table::var_len::{NullVarLenVisitor, VarLenGranule, VarLenMembers, VarLenRef}; fn time(acc: &mut Duration, body: impl FnOnce() -> R) -> R { @@ -192,7 +191,7 @@ fn fill_page_with_fixed_len_collect_row_pointers( row_size_for_type::(), val.as_bytes(), &[], - &mut NullBlobStore, + &mut BlobSet::new_for_test(), ) } .unwrap(); @@ -307,9 +306,9 @@ fn table_insert_one_row(c: &mut Criterion) { let val = black_box(val.to_product()); // Insert before benching to alloc and fault in a page. - let mut ctx = (table, NullBlobStore); - let ptr = ctx.0.insert(&mut ctx.1, &val).unwrap().1.pointer(); - let pre = |_, (table, bs): &mut (Table, NullBlobStore)| { + let mut ctx = (table, BlobSet::new_for_test()); + let ptr = ctx.0.insert(&ctx.1, &val).unwrap().1.pointer(); + let pre = |_, (table, bs): &mut (Table, BlobSet)| { table.delete(bs, ptr, |_| ()).unwrap(); }; group.bench_function(name, |b| { @@ -357,10 +356,9 @@ fn table_delete_one_row(c: &mut Criterion) { let val = val.to_product(); // Insert before benching to alloc and fault in a page. - let mut ctx = (table, NullBlobStore, PagePool::new_for_test()); - let insert = |_: u64, (table, bs, _pool): &mut (Table, NullBlobStore, PagePool)| { - table.insert(bs, &val).unwrap().1.pointer() - }; + let mut ctx = (table, BlobSet::new_for_test(), PagePool::new_for_test()); + let insert = + |_: u64, (table, bs, _pool): &mut (Table, BlobSet, PagePool)| table.insert(bs, &val).unwrap().1.pointer(); group.bench_function(name, |b| { iter_time_with(b, &mut ctx, insert, |row, _, (table, bs, _)| { @@ -406,8 +404,8 @@ fn table_extract_one_row(c: &mut Criterion) { let mut table = make_table_for_row_type::(name); let val = val.to_product(); - let mut blob_store = NullBlobStore; - let row = black_box(table.insert(&mut blob_store, &val).unwrap().1); + let blob_store = BlobSet::new_for_test(); + let row = black_box(table.insert(&blob_store, &val).unwrap().1); group.bench_function(name, |b| { b.iter_with_large_drop(|| black_box(row.to_product_value())); }); @@ -535,7 +533,7 @@ fn make_table_with_index(unique: bool) -> (Table, IndexId) { let algo = BTreeAlgorithm { columns: cols }.into(); let idx = tbl.new_index(&algo, unique).unwrap(); // SAFETY: index was derived from the table. - unsafe { tbl.insert_index(&NullBlobStore, index_id, idx) } + unsafe { tbl.insert_index(&BlobSet::new_for_test(), index_id, idx) } .unwrap() .unwrap(); @@ -563,7 +561,9 @@ fn insert_num_same( if let Some(slot) = row.elements.get_mut(1) { *slot = n.into(); } - tbl.insert(&mut NullBlobStore, &row).map(|(_, row)| row.pointer()).ok() + tbl.insert(&BlobSet::new_for_test(), &row) + .map(|(_, row)| row.pointer()) + .ok() }) .last() .flatten() @@ -575,7 +575,7 @@ fn clear_all_same(tbl: &mut Table, index_id: IndexId, val_same: u let key = index.key_from_algebraic_value(&key); let ptrs = index.seek_point(&key).collect::>(); for ptr in ptrs { - tbl.delete(&mut NullBlobStore, ptr, |_| ()).unwrap(); + tbl.delete(&BlobSet::new_for_test(), ptr, |_| ()).unwrap(); } } @@ -622,13 +622,13 @@ fn index_insert(c: &mut Criterion) { let pool = PagePool::new_for_test(); let (tbl, index_id, num_same, _) = make_table_with_same_ratio::(&pool, make_row_move, num_rows, same_ratio, false); - let mut ctx = (tbl, NullBlobStore, pool); + let mut ctx = (tbl, BlobSet::new_for_test(), pool); group.bench_with_input( bench_id_for_index(name, num_rows, same_ratio, num_same, false), &num_rows, |b, &num_rows| { - let pre = |_, (tbl, _, pool): &mut (Table, NullBlobStore, PagePool)| { + let pre = |_, (tbl, _, pool): &mut (Table, BlobSet, PagePool)| { clear_all_same::(tbl, index_id, num_rows); insert_num_same(pool, tbl, || make_row(num_rows), num_same - 1); make_row(num_rows).to_product() @@ -683,9 +683,8 @@ fn index_seek(c: &mut Criterion) { &num_diff, |b, &num_diff| { let col_to_seek = black_box(R::column_value_from_u64(num_diff / 2)); - let index = black_box(&tbl) - .get_index_by_id_with_table(&NullBlobStore, index_id) - .unwrap(); + let blobs = BlobSet::new_for_test(); + let index = black_box(&tbl).get_index_by_id_with_table(&blobs, index_id).unwrap(); b.iter_custom(|num_iters| { let mut elapsed = WallTime.zero(); for _ in 0..num_iters { @@ -758,7 +757,7 @@ fn index_delete(c: &mut Criterion) { insert_num_same(&pool, tbl, || make_row(num_rows), num_same).unwrap() }; iter_time_with(b, &mut tbl, pre, |ptr, _, tbl| { - tbl.delete(&mut NullBlobStore, ptr, |_| ()) + tbl.delete(&BlobSet::new_for_test(), ptr, |_| ()) }); }, ); diff --git a/crates/table/src/bflatn_from.rs b/crates/table/src/bflatn_from.rs index 9d0a6b6f30b..d48a399518c 100644 --- a/crates/table/src/bflatn_from.rs +++ b/crates/table/src/bflatn_from.rs @@ -2,8 +2,9 @@ //! which serializes `value = page.get_row_data(fixed_offset, fixed_row_size)` typed at `ty` //! and associated var len objects in `value` into the serializer `ser`. +use crate::tiered::{BlobHandle, BlobReadSet}; + use super::{ - blob_store::BlobStore, indexes::{Bytes, PageOffset}, page::Page, row_hash, @@ -34,7 +35,7 @@ use spacetimedb_sats::{ pub unsafe fn serialize_row_from_page( ser: S, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, fixed_offset: PageOffset, ty: &RowTypeLayout, ) -> Result { @@ -43,7 +44,7 @@ pub unsafe fn serialize_row_from_page( // - Per 1. and 2., `fixed_bytes` points at a row in `page` valid for `ty`. // - Per 3., for any `vlr: VarLenRef` stored in `fixed_bytes`, // `vlr.first_offset` is either `NULL` or points to a valid granule in `page`. - unsafe { serialize_product(ser, fixed_bytes, page, blob_store, &Cell::new(0), ty.product()) } + unsafe { serialize_product(ser, fixed_bytes, page, blobs, &Cell::new(0), ty.product()) } } /// Serializes the columns `cols` of the row in `page` @@ -60,7 +61,7 @@ pub unsafe fn serialize_row_from_page( pub unsafe fn serialize_columns_from_page( ser: S, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, fixed_offset: PageOffset, ty: &RowTypeLayout, cols: &ColList, @@ -80,7 +81,7 @@ pub unsafe fn serialize_columns_from_page( // `sub_val = &bytes[range_move(0..elem_ty.ty.size(), offset)]` // is valid at `elem_ty.ty`, as `elem_ty`. // 2. forward caller requirement. - unsafe { serialize_product_field(&mut ser, bytes, page, blob_store, offset, elem_ty) }?; + unsafe { serialize_product_field(&mut ser, bytes, page, blobs, offset, elem_ty) }?; } ser.end() @@ -108,7 +109,7 @@ unsafe fn serialize_product( ser: S, bytes: &Bytes, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, curr_offset: CurrOffset<'_>, ty: ProductTypeLayoutView<'_>, ) -> Result { @@ -123,7 +124,7 @@ unsafe fn serialize_product( // `sub_val = &bytes[range_move(0..elem_ty.ty.size(), offset)]` // is valid at `elem_ty.ty`, as `elem_ty`. // 2. forward caller requirement. - unsafe { serialize_product_field(&mut ser, bytes, page, blob_store, offset, elem_ty) }?; + unsafe { serialize_product_field(&mut ser, bytes, page, blobs, offset, elem_ty) }?; } ser.end() @@ -140,7 +141,7 @@ unsafe fn serialize_product_field( ser: &mut S, bytes: &Bytes, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, offset: usize, elem_ty: &ProductTypeElementLayout, ) -> Result<(), S::Error> { @@ -151,7 +152,7 @@ unsafe fn serialize_product_field( let value = Value { bytes, page, - blob_store, + blobs, curr_offset: &Cell::new(offset), ty: &elem_ty.ty, }; @@ -169,7 +170,7 @@ unsafe fn serialize_sum( ser: S, bytes: &Bytes, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, curr_offset: CurrOffset<'_>, ty: &SumTypeLayout, ) -> Result { @@ -185,7 +186,7 @@ unsafe fn serialize_sum( let data_value = Value { bytes, page, - blob_store, + blobs, curr_offset: data_offset, ty: data_ty, }; @@ -213,13 +214,13 @@ pub fn read_tag<'ty>(bytes: &Bytes, ty: &'ty SumTypeLayout, curr_offset: usize) struct Value<'a> { bytes: &'a Bytes, page: &'a Page, - blob_store: &'a dyn BlobStore, + blobs: &'a BlobReadSet, curr_offset: CurrOffset<'a>, ty: &'a AlgebraicTypeLayout, } impl_serialize!(['a] Value<'a>, (self, ser) => { - unsafe { serialize_value(ser, self.bytes, self.page, self.blob_store, self.curr_offset, self.ty) } + unsafe { serialize_value(ser, self.bytes, self.page, self.blobs, self.curr_offset, self.ty) } }); /// Serialize `value = &bytes[range_move(0..ty.size(), *curr_offset)]` into a `ser`, @@ -235,7 +236,7 @@ pub(crate) unsafe fn serialize_value( ser: S, bytes: &Bytes, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, curr_offset: CurrOffset<'_>, ty: &AlgebraicTypeLayout, ) -> Result { @@ -248,11 +249,11 @@ pub(crate) unsafe fn serialize_value( match ty { AlgebraicTypeLayout::Sum(ty) => { // SAFETY: `value` was valid at `ty` and `VarLenRef`s won't be dangling. - unsafe { serialize_sum(ser, bytes, page, blob_store, curr_offset, ty) } + unsafe { serialize_sum(ser, bytes, page, blobs, curr_offset, ty) } } AlgebraicTypeLayout::Product(ty) => { // SAFETY: `value` was valid at `ty` and `VarLenRef`s won't be dangling. - unsafe { serialize_product(ser, bytes, page, blob_store, curr_offset, ty.view()) } + unsafe { serialize_product(ser, bytes, page, blobs, curr_offset, ty.view()) } } // The primitive types: // @@ -303,11 +304,11 @@ pub(crate) unsafe fn serialize_value( // The var-len cases. &AlgebraicTypeLayout::String => { // SAFETY: `value` was valid at `::String` and `VarLenRef`s won't be dangling. - unsafe { serialize_string(ser, bytes, page, blob_store, curr_offset) } + unsafe { serialize_string(ser, bytes, page, blobs, curr_offset) } } AlgebraicTypeLayout::VarLen(VarLenType::Array(ty)) => { // SAFETY: `value` was valid at `ty` and `VarLenRef`s won't be dangling. - unsafe { serialize_array(ser, bytes, page, blob_store, curr_offset, ty) } + unsafe { serialize_array(ser, bytes, page, blobs, curr_offset, ty) } } } } @@ -324,7 +325,7 @@ unsafe fn serialize_string( ser: S, bytes: &Bytes, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, curr_offset: CurrOffset<'_>, ) -> Result { // SAFETY: `value` was valid at and aligned for `::String` @@ -334,9 +335,9 @@ unsafe fn serialize_string( if vlr.is_large_blob() { // SAFETY: As `vlr` a blob, `vlr.first_granule` always points to a valid granule. - let blob = unsafe { vlr_blob_bytes(page, blob_store, vlr) }; + let blob = unsafe { vlr_blob_bytes(page, blobs, vlr) }; // SAFETY: For `::String`, the blob will always be valid UTF-8. - let str = unsafe { str::from_utf8_unchecked(blob) }; + let str = unsafe { str::from_utf8_unchecked(&blob) }; ser.serialize_str(str) } else { // SAFETY: `vlr.first_granule` is either NULL or points to a valid granule. @@ -353,7 +354,7 @@ unsafe fn serialize_array( ser: S, bytes: &Bytes, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, curr_offset: CurrOffset<'_>, ty: &ArrayType, ) -> Result { @@ -364,9 +365,9 @@ unsafe fn serialize_array( if vlr.is_large_blob() { // SAFETY: As `vlr` is a blob, `vlr.first_granule` always points to a valid granule. - let blob = unsafe { vlr_blob_bytes(page, blob_store, vlr) }; + let blob = unsafe { vlr_blob_bytes(page, blobs, vlr) }; // SAFETY: The BSATN in `blob` is encoded from an `AlgebraicValue`. - unsafe { ser.serialize_bsatn(ty, blob) } + unsafe { ser.serialize_bsatn(ty, &blob) } } else { // SAFETY: `vlr.first_granule` is either NULL or points to a valid granule. let var_iter = unsafe { page.iter_vlo_data(vlr.first_granule) }; @@ -384,7 +385,7 @@ unsafe fn serialize_array( /// - `vlr.first_granule` must point to a valid granule in `page`. #[cold] #[inline(never)] -pub(crate) unsafe fn vlr_blob_bytes<'b>(page: &Page, blob_store: &'b dyn BlobStore, vlr: VarLenRef) -> &'b [u8] { +pub(crate) unsafe fn vlr_blob_bytes(page: &Page, blobs: &BlobReadSet, vlr: VarLenRef) -> BlobHandle { // Read the blob hash. // SAFETY: `vlr.first_granule` points to a valid granule. let mut var_iter = unsafe { page.iter_var_len_object(vlr.first_granule) }; @@ -395,7 +396,7 @@ pub(crate) unsafe fn vlr_blob_bytes<'b>(page: &Page, blob_store: &'b dyn BlobSto let hash = granule.blob_hash(); // Find the blob. - blob_store.retrieve_blob(&hash).unwrap() + blobs.get(&hash).unwrap().clone() } /// Read a `T` from `bytes` at the `curr_offset` and advance by `size` bytes. diff --git a/crates/table/src/bflatn_to.rs b/crates/table/src/bflatn_to.rs index c832eaaaa6c..6713b7d51b6 100644 --- a/crates/table/src/bflatn_to.rs +++ b/crates/table/src/bflatn_to.rs @@ -4,11 +4,11 @@ use crate::{ indexes::PageIndex, - tiered::{PageError, PageSet, ReservedPage}, + table::TableError, + tiered::{BlobError, BlobSet, PageError, PageSet, ReservedPage}, }; use super::{ - blob_store::BlobStore, indexes::{Bytes, PageOffset, RowPointer, SquashedOffset}, page::{GranuleOffsetIter, Page, VarView}, table::BlobNumBytes, @@ -34,9 +34,13 @@ pub enum Error { #[error("Expected a value of type {0:?}, but found {1:?}")] WrongType(AlgebraicType, AlgebraicValue), #[error(transparent)] - PageError(#[from] super::page::Error), + Page(#[from] super::page::Error), #[error(transparent)] - PagesError(#[from] PageError), + Pages(#[from] PageError), + #[error(transparent)] + Blobs(#[from] BlobError), + #[error(transparent)] + Table(#[from] TableError), } /// Writes `row` typed at `ty` to `pages` @@ -53,13 +57,13 @@ pub enum Error { pub unsafe fn write_row_to_pages_bsatn( pages: &mut PageSet, visitor: &impl VarLenMembers, - blob_store: &mut dyn BlobStore, + blobs: &mut BlobSet, ty: &RowTypeLayout, mut bytes: &[u8], squashed_offset: SquashedOffset, ) -> Result<(RowPointer, BlobNumBytes), Error> { let val = ty.product().deserialize(bsatn::Deserializer::new(&mut bytes))?; - unsafe { write_row_to_pages(pages, None, visitor, blob_store, ty, &val, squashed_offset) } + unsafe { write_row_to_pages(pages, None, visitor, blobs, ty, &val, squashed_offset) } } /// Writes `row` typed at `ty` to `pages` @@ -77,7 +81,7 @@ pub unsafe fn write_row_to_pages( pages: &mut PageSet, reservation: Option<(PageIndex, ReservedPage)>, visitor: &impl VarLenMembers, - blob_store: &mut dyn BlobStore, + blobs: &BlobSet, ty: &RowTypeLayout, val: &ProductValue, squashed_offset: SquashedOffset, @@ -97,7 +101,7 @@ pub unsafe fn write_row_to_pages( // - Caller promised that `visitor` is prepared to visit for `ty` // and in the same order as a `VarLenVisitorProgram` for `ty` would. // - `visitor` came from `pages` which we can trust to visit in the right order. - unsafe { write_row_to_page(page, blob_store, visitor, ty, val) } + unsafe { write_row_to_page(page, blobs, visitor, ty, val) } })? { (page, Ok((offset, blob_inserted))) => { Ok((RowPointer::new(false, page, offset, squashed_offset), blob_inserted)) @@ -123,7 +127,7 @@ pub unsafe fn write_row_to_pages( /// - `page` must use a var-len visitor which visits the same var-len members in the same order. pub unsafe fn write_row_to_page( page: &mut Page, - blob_store: &mut dyn BlobStore, + blobs: &BlobSet, visitor: &impl VarLenMembers, ty: &RowTypeLayout, val: &ProductValue, @@ -160,7 +164,7 @@ pub unsafe fn write_row_to_page( } else { // Haven't stored large blobs or init those granules with blob hashes yet, // so do it now. - serialized.write_large_blobs(blob_store) + serialized.write_large_blobs(blobs)? }; Ok((fixed_offset, blob_store_inserted_bytes)) @@ -213,7 +217,7 @@ impl BflatnSerializedRowBuffer<'_> { /// Insert all large blobs into `blob_store` and their hashes to their granules. #[cold] #[inline(never)] - fn write_large_blobs(mut self, blob_store: &mut dyn BlobStore) -> BlobNumBytes { + fn write_large_blobs(mut self, blobs: &BlobSet) -> Result { let mut blob_store_inserted_bytes = BlobNumBytes::default(); for (vlr, value) in self.large_blob_insertions { // SAFETY: `vlr` was given to us by `alloc_for_slice` @@ -221,10 +225,10 @@ impl BflatnSerializedRowBuffer<'_> { // However, as it was added to `self.large_blob_insertions`, // we have not yet written the hash to that granule. unsafe { - blob_store_inserted_bytes += self.var_view.write_large_blob_hash_to_granule(blob_store, &value, vlr); + blob_store_inserted_bytes += self.var_view.write_large_blob_hash_to_granule(blobs, &value, vlr)?; } } - blob_store_inserted_bytes + Ok(blob_store_inserted_bytes) } /// Write an `val`, an [`AlgebraicValue`], typed at `ty`, to the buffer. @@ -556,8 +560,8 @@ fn bsatn_len(val: &AlgebraicValue) -> usize { pub mod test { use super::*; use crate::{ - bflatn_from::serialize_row_from_page, blob_store::HashMapBlobStore, page::tests::hash_unmodified_save_get, - row_type_visitor::row_type_visitor, + bflatn_from::serialize_row_from_page, page::tests::hash_unmodified_save_get, + row_type_visitor::row_type_visitor, tiered::BlobReadSet, }; use proptest::{prelude::*, prop_assert_eq, proptest}; use spacetimedb_sats::algebraic_value::ser::ValueSerializer; @@ -570,16 +574,23 @@ pub mod test { let ty: RowTypeLayout = ty.into(); let mut page = Page::new(ty.size()); let visitor = row_type_visitor(&ty); - let blob_store = &mut HashMapBlobStore::default(); + let blobs = &mut BlobSet::new_for_test(); let hash_pre_ins = hash_unmodified_save_get(&mut page); - let (offset, _) = unsafe { write_row_to_page(&mut page, blob_store, &visitor, &ty, &val).unwrap() }; + let (offset, _) = unsafe { write_row_to_page(&mut page, blobs, &visitor, &ty, &val).unwrap() }; let hash_pre_ser = hash_unmodified_save_get(&mut page); assert_ne!(hash_pre_ins, hash_pre_ser); - let read_val = unsafe { serialize_row_from_page(ValueSerializer, &page, blob_store, offset, &ty) } + let blobs = BlobReadSet::new( + &visitor, + &page, + blobs, + &ty, + offset + ).unwrap(); + let read_val = unsafe { serialize_row_from_page(ValueSerializer, &page, &blobs, offset, &ty) } .unwrap().into_product().unwrap(); prop_assert_eq!(val, read_val); diff --git a/crates/table/src/blob_store.rs b/crates/table/src/blob_store.rs index ba0665b228c..36b45939005 100644 --- a/crates/table/src/blob_store.rs +++ b/crates/table/src/blob_store.rs @@ -1,21 +1,10 @@ -//! Provides the interface [`BlobStore`] that tables use to talk to -//! a blob store engine for large var-len objects. -//! -//! These blob objects are referred to by their [`BlobHash`], -//! which is currently defined through BLAKE3 on the bytes of the blob object. -//! -//! Two simple implementations are provided, -//! primarily for tests and benchmarking. -//! - [`NullBlobStore`], a blob store that always panics. -//! Used when ensuring that the blob store is unreachable in a scenario. -//! - [`HashMapBlobStore`], a blob store backed by a `HashMap` that refcounts blob objects. -//! It is not optimize and is mainly intended for testing purposes. +//! Defines the content address used for large var-len objects. use blake3::hash; -use core::mem; -use spacetimedb_data_structures::map::{hash_map::Entry, HashMap}; +use core::fmt; use spacetimedb_lib::{de::Deserialize, ser::Serialize}; use spacetimedb_memory_usage::MemoryUsage; +use std::fmt::Write; /// The content address of a blob-stored object. #[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Hash, Debug, Serialize, Deserialize)] @@ -29,13 +18,12 @@ pub struct BlobHash { impl MemoryUsage for BlobHash {} impl BlobHash { - /// The size of the hash function's output in bytes. pub const SIZE: usize = 32; - /// Returns the blob hash for `bytes`. pub fn hash_from_bytes(bytes: &[u8]) -> Self { - let data = hash(bytes).into(); - Self { data } + Self { + data: hash(bytes).into(), + } } } @@ -43,248 +31,19 @@ impl TryFrom<&[u8]> for BlobHash { type Error = (); fn try_from(data: &[u8]) -> Result { - let data: [u8; Self::SIZE] = data.try_into().map_err(drop)?; - Ok(Self { data }) - } -} - -/// An error that signifies that a [`BlobHash`] wasn't associated with a large blob object. -#[derive(Debug)] -pub struct NoSuchBlobError; - -/// Iterator returned by [`BlobStore::iter_blobs`]. -/// -/// Each element is a tuple `(hash, uses, data)`, -/// where `hash` is a blob's content-addressed [`BlobHash`], -/// `uses` is the number of references to that blob, -/// and `data` is the data itself. -pub type BlobsIter<'a> = Box + 'a>; - -/// The interface that tables use to talk to the blob store engine for large var-len objects. -/// -/// These blob objects are referred to by their [`BlobHash`], -/// which is currently defined through BLAKE3 on the bytes of the blob object. -pub trait BlobStore: Sync { - /// Mark the `hash` as used. - /// - /// This is a more efficient way of doing: - /// ```ignore - /// let bytes = self.retrieve_blob(&hash); - /// let _ = self.insert_blob(&bytes); - /// ``` - fn clone_blob(&mut self, hash: &BlobHash) -> Result<(), NoSuchBlobError>; - - /// Insert `bytes` into the blob store. - /// - /// Returns the content address of `bytes` a `BlobHash` - /// which can be used in [`retrieve_blob`] to fetch it. - fn insert_blob(&mut self, bytes: &[u8]) -> BlobHash; - - /// Insert `hash` referring to `bytes` and mark its refcount as `uses`. - /// - /// Used when restoring from a snapshot. - fn insert_with_uses(&mut self, hash: &BlobHash, uses: usize, bytes: Box<[u8]>); - - /// Returns the bytes stored at the content address `hash`. - fn retrieve_blob(&self, hash: &BlobHash) -> Result<&[u8], NoSuchBlobError>; - - /// Marks the `hash` as unused. - /// - /// Depending on the strategy employed by the blob store, - /// this might not actually free the data, - /// but rather just decrement a reference count. - fn free_blob(&mut self, hash: &BlobHash) -> Result<(), NoSuchBlobError>; - - /// Iterate over all blobs present in the blob store. - /// - /// Each element is a tuple `(hash, uses, data)`, - /// where `hash` is a blob's content-addressed [`BlobHash`], - /// `uses` is the number of references to that blob, - /// and `data` is the data itself. - /// - /// Used when capturing a snapshot. - fn iter_blobs(&self) -> BlobsIter<'_>; - - /// Returns the amount of memory in bytes used by blobs in this `BlobStore`. - /// - /// Duplicate blobs are counted a number of times equal to their refcount. - /// This is in order to preserve the property that inserting a large blob - /// causes this quantity to increase by that blob's size, - /// and deleting a large blob causes it to decrease the same amount. - fn bytes_used_by_blobs(&self) -> u64 { - self.iter_blobs() - .map(|(_, uses, data)| data.len() as u64 * uses as u64) - .sum() - } - - /// Returns the physical bytes represented by unique blob entries. - /// - /// This counts each unique [`BlobHash`] key once plus the payload bytes for that key. - /// It intentionally does not consider refcounts, as it is primarily used for memory tracking. - fn physical_bytes_used_by_blobs(&self) -> u64 { - self.iter_blobs() - .map(|(_, _, data)| data.len() as u64 + mem::size_of::() as u64) - .sum() - } - - /// Returns the number of blobs, or more precisely, blob-usages, recorded in this `BlobStore`. - /// - /// Duplicate blobs are counted a number of times equal to their refcount. - /// This is in order to preserve the property that inserting a large blob - /// causes this quantity to increase by 1, and deleting a large blob causes it to decrease by 1. - fn num_blobs(&self) -> u64 { - self.iter_blobs().map(|(_, uses, _)| uses as u64).sum() + Ok(Self { + data: data.try_into().map_err(drop)?, + }) } } -/// A blob store that panics on all operations. -/// Used for tests when you want to ensure that the blob store isn't used. -#[derive(Default)] -pub struct NullBlobStore; - -impl BlobStore for NullBlobStore { - fn clone_blob(&mut self, _hash: &BlobHash) -> Result<(), NoSuchBlobError> { - unimplemented!("NullBlobStore doesn't do anything") - } - - fn insert_blob(&mut self, _bytes: &[u8]) -> BlobHash { - unimplemented!("NullBlobStore doesn't do anything") - } - - fn insert_with_uses(&mut self, _hash: &BlobHash, _uses: usize, _bytes: Box<[u8]>) { - unimplemented!("NullBlobStore doesn't do anything") - } - - fn retrieve_blob(&self, _hash: &BlobHash) -> Result<&[u8], NoSuchBlobError> { - unimplemented!("NullBlobStore doesn't do anything") - } - - fn free_blob(&mut self, _hash: &BlobHash) -> Result<(), NoSuchBlobError> { - unimplemented!("NullBlobStore doesn't do anything") - } - - fn iter_blobs(&self) -> BlobsIter<'_> { - unimplemented!("NullBlobStore doesn't do anything") - } -} - -/// A blob store that is backed by a hash map with a reference counted value. -/// Used for tests when you need an actual blob store. -#[derive(Default, PartialEq, Eq, Debug)] -pub struct HashMapBlobStore { - /// For testing, we use a hash map with a reference count - /// to handle freeing and cloning correctly. - map: HashMap, - /// Total number of bytes in the map used by blobs and their keys - physical_blob_payload_bytes: u64, -} - -impl MemoryUsage for HashMapBlobStore { - fn heap_usage(&self) -> usize { - let Self { - map, - physical_blob_payload_bytes: _, - } = self; - map.heap_usage() - } -} - -/// A blob object including a reference count and the data. -#[derive(PartialEq, Eq, Debug)] -struct BlobObject { - /// Reference count of the blob. - uses: usize, - /// The blob data. - blob: Box<[u8]>, -} - -impl MemoryUsage for BlobObject { - fn heap_usage(&self) -> usize { - let Self { uses, blob } = self; - uses.heap_usage() + blob.heap_usage() - } -} - -impl BlobStore for HashMapBlobStore { - fn clone_blob(&mut self, hash: &BlobHash) -> Result<(), NoSuchBlobError> { - self.map.get_mut(hash).ok_or(NoSuchBlobError)?.uses += 1; - Ok(()) - } - - fn insert_blob(&mut self, bytes: &[u8]) -> BlobHash { - let hash = BlobHash::hash_from_bytes(bytes); - match self.map.entry(hash) { - Entry::Occupied(mut entry) => entry.get_mut().uses += 1, - Entry::Vacant(entry) => { - self.physical_blob_payload_bytes += bytes.len() as u64; - entry.insert(BlobObject { - blob: bytes.into(), - uses: 1, - }); - } - } - hash - } - - fn insert_with_uses(&mut self, hash: &BlobHash, uses: usize, bytes: Box<[u8]>) { - debug_assert_eq!(hash, &BlobHash::hash_from_bytes(&bytes)); - match self.map.entry(*hash) { - Entry::Occupied(mut entry) => entry.get_mut().uses += uses, - Entry::Vacant(entry) => { - self.physical_blob_payload_bytes += bytes.len() as u64; - entry.insert(BlobObject { blob: bytes, uses }); - } - } - } - - fn retrieve_blob(&self, hash: &BlobHash) -> Result<&[u8], NoSuchBlobError> { - self.map.get(hash).map(|obj| &*obj.blob).ok_or(NoSuchBlobError) - } - - fn free_blob(&mut self, hash: &BlobHash) -> Result<(), NoSuchBlobError> { - match self.map.entry(*hash) { - Entry::Vacant(_) => return Err(NoSuchBlobError), - Entry::Occupied(entry) if entry.get().uses == 1 => { - let removed = entry.remove(); - self.physical_blob_payload_bytes -= removed.blob.len() as u64; - } - Entry::Occupied(mut entry) => entry.get_mut().uses -= 1, +impl fmt::Display for BlobHash { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + const ALPHABET: &[u8; 16] = b"0123456789abcdef"; + for &b in &self.data { + f.write_char(ALPHABET[(b >> 4) as usize] as char)?; + f.write_char(ALPHABET[(b & 0xf) as usize] as char)?; } Ok(()) } - - fn iter_blobs(&self) -> BlobsIter<'_> { - Box::new(self.map.iter().map(|(hash, obj)| (hash, obj.uses, &obj.blob[..]))) - } - - fn physical_bytes_used_by_blobs(&self) -> u64 { - self.physical_blob_payload_bytes + self.map.len() as u64 * mem::size_of::() as u64 - } -} - -impl HashMapBlobStore { - /// Merge `src_bs` into `self`. - pub fn merge_from(&mut self, src_bs: Self) { - let Self { - map, - physical_blob_payload_bytes: _, - } = src_bs; - for (hash, obj) in map { - match self.map.entry(hash) { - Entry::Occupied(mut entry) => entry.get_mut().uses += obj.uses, - Entry::Vacant(entry) => { - self.physical_blob_payload_bytes += obj.blob.len() as u64; - entry.insert(obj); - } - } - } - } -} - -#[cfg(test)] -impl HashMapBlobStore { - /// Returns a map relating blob hashes to the usage count in this blob store. - pub fn usage_counter(&self) -> HashMap { - self.iter_blobs().map(|(hash, uses, _)| (*hash, uses)).collect() - } } diff --git a/crates/table/src/eq.rs b/crates/table/src/eq.rs index aab706e747c..b62cba4454e 100644 --- a/crates/table/src/eq.rs +++ b/crates/table/src/eq.rs @@ -229,7 +229,7 @@ fn eq_byte_array(ctx: &mut EqCtx<'_, '_>, len: usize) -> bool { #[cfg(test)] mod test { - use crate::blob_store::NullBlobStore; + use crate::tiered::BlobSet; use spacetimedb_sats::{product, AlgebraicType, AlgebraicValue, ProductType}; #[test] @@ -241,7 +241,7 @@ mod test { AlgebraicType::product([AlgebraicType::U8, AlgebraicType::U32]), // xpppxxxx ])]); - let bs = &mut NullBlobStore; + let bs = &mut BlobSet::new_for_test(); let mut table_a = crate::table::test::table(ty.clone()); let mut table_b = crate::table::test::table(ty); @@ -262,7 +262,7 @@ mod test { // They should occupy the spaces of the previous rows. let v1 = product![AlgebraicValue::sum(1, product![0u8, 0u32].into())]; let (_, a1_rr) = table_a.insert(bs, &v1).unwrap(); - let bs = &mut NullBlobStore; + let bs = &mut BlobSet::new_for_test(); let (_, b1_rr) = table_b.insert(bs, &v1).unwrap(); assert_eq!(a0_ptr, a1_rr.pointer()); assert_eq!(b0_ptr, b1_rr.pointer()); diff --git a/crates/table/src/eq_to_pv.rs b/crates/table/src/eq_to_pv.rs index f2252e844c6..775892db16f 100644 --- a/crates/table/src/eq_to_pv.rs +++ b/crates/table/src/eq_to_pv.rs @@ -5,11 +5,11 @@ use crate::{ bflatn_from::{read_tag, vlr_blob_bytes}, - blob_store::BlobStore, eq::BytesPage, indexes::PageOffset, page::Page, row_hash::{read_from_bytes, run_vlo_bytes}, + tiered::BlobReadSet, var_len::{VarLenGranule, VarLenRef}, }; use core::str; @@ -32,7 +32,7 @@ use spacetimedb_sats::{AlgebraicValue, ProductValue}; /// 2. for any `vlr: VarLenRef` in the fixed parts of row `lhs`, /// `vlr.first_offset` must either be `NULL` or point to a valid granule in `page`. pub unsafe fn eq_row_in_page_to_pv( - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, page: &Page, fixed_offset: PageOffset, rhs: &ProductValue, @@ -41,7 +41,7 @@ pub unsafe fn eq_row_in_page_to_pv( // Context for the whole comparison. let mut ctx = EqCtx { lhs: BytesPage::new(page, fixed_offset, ty), - blob_store, + blobs, curr_offset: 0, }; // Test for equality! @@ -61,7 +61,7 @@ struct EqCtx<'page> { /// The view into the fixed part of row `lhs` in its page. lhs: BytesPage<'page>, /// The blob store that `lhs.page` uses for its large blob VLOs. - blob_store: &'page dyn BlobStore, + blobs: &'page BlobReadSet, /// The current offset at which some sub-object of `lhs` exists. curr_offset: usize, } @@ -168,7 +168,7 @@ unsafe fn eq_value(ctx: &mut EqCtx<'_>, ty: &AlgebraicTypeLayout, rhs: &Algebrai run_vlo_bytes( ctx.lhs.page, ctx.lhs.bytes, - ctx.blob_store, + ctx.blobs, &mut ctx.curr_offset, |mut bsatn| { let lhs = Deserializer::new(&mut bsatn); @@ -196,9 +196,9 @@ unsafe fn eq_str(ctx: &mut EqCtx<'_>, rhs: &str) -> bool { if vlr.is_large_blob() { // SAFETY: As `vlr` is a blob, `vlr.first_granule` always points to a valid granule. - let bytes = unsafe { vlr_blob_bytes(ctx.lhs.page, ctx.blob_store, vlr) }; + let blob = unsafe { vlr_blob_bytes(ctx.lhs.page, ctx.blobs, vlr) }; // SAFETY: For `ty = String`, the blob will always be valid UTF-8. - rhs == unsafe { str::from_utf8_unchecked(bytes) } + rhs == unsafe { str::from_utf8_unchecked(&blob) } } else { // SAFETY: `vlr.first_granule` is either NULL or points to a valid granule. let lhs_chunks = unsafe { ctx.lhs.page.iter_vlo_data(vlr.first_granule) }; @@ -226,18 +226,19 @@ unsafe fn eq_at(ctx: &mut EqCtx<'_>, rhs: &T) -> bool { #[cfg(test)] mod tests { - use crate::blob_store::HashMapBlobStore; use proptest::prelude::*; use spacetimedb_sats::proptest::generate_typed_row; + use crate::tiered::BlobSet; + proptest! { #![proptest_config(ProptestConfig::with_cases(if cfg!(miri) { 8 } else { 2048 }))] #[test] fn pv_row_ref_eq((ty, val) in generate_typed_row()) { // Turn `val` into a `RowRef`. let mut table = crate::table::test::table(ty); - let blob_store = &mut HashMapBlobStore::default(); - let (_, row) = table.insert(blob_store, &val).unwrap(); + let blobs = &mut BlobSet::new_for_test(); + let (_, row) = table.insert(blobs, &val).unwrap(); // Check eq algo. prop_assert_eq!(row, val); diff --git a/crates/table/src/page.rs b/crates/table/src/page.rs index 5e105b4ba05..0b82c6d14c0 100644 --- a/crates/table/src/page.rs +++ b/crates/table/src/page.rs @@ -32,8 +32,10 @@ //! See the post [Two Kinds of Invariants: Safety and Validity][ralf_safe_valid] //! for a discussion on safety and validity invariants. +use crate::tiered::{BlobError, BlobSet}; + use super::{ - blob_store::{BlobHash, BlobStore}, + blob_store::BlobHash, fixed_bit_set::FixedBitSet, fixed_bit_set::IterSet, indexes::{max_rows_in_page, Byte, Bytes, PageOffset}, @@ -739,19 +741,17 @@ impl<'page> VarView<'page> { /// Inserts `var_len_obj` into `blob_store` /// and stores the blob hash in the granule pointed to by `vlr.first_granule`. /// - /// This insertion will never fail. - /// /// # Safety /// /// `vlr.first_granule` must point to an unused `VarLenGranule` in bounds of this page, /// which must be valid for writes. pub unsafe fn write_large_blob_hash_to_granule( &mut self, - blob_store: &mut dyn BlobStore, + blobs: &BlobSet, var_len_obj: &impl AsRef<[u8]>, vlr: VarLenRef, - ) -> BlobNumBytes { - let hash = blob_store.insert_blob(var_len_obj.as_ref()); + ) -> Result { + let hash = blobs.insert(var_len_obj.as_ref())?.hash(); let granule = vlr.first_granule; // SAFETY: @@ -759,7 +759,7 @@ impl<'page> VarView<'page> { // 2. The null granule is trivially initialized. // 3. The caller promised that `granule` is safe to overwrite. unsafe { self.write_chunk_to_granule(&hash.data, hash.data.len(), granule, PageOffset::VAR_LEN_NULL) }; - var_len_obj.as_ref().len().into() + Ok(var_len_obj.as_ref().len().into()) } /// Write the `chunk` (data) to the [`VarLenGranule`] pointed to by `granule`, @@ -902,29 +902,14 @@ impl<'page> VarView<'page> { /// SAFETY: `offset` must point to a valid [`VarLenGranule`] or be NULL. #[cold] #[inline(never)] - unsafe fn free_blob(&self, offset: PageOffset, blob_store: &mut dyn BlobStore) -> BlobNumBytes { + unsafe fn free_blob(&self, offset: PageOffset, blobs: &BlobSet) -> Result { assert!(!offset.is_var_len_null()); // SAFETY: Per caller contract + the assertion above, // we know `offset` refers to a valid `VarLenGranule`. let granule = unsafe { self.get_granule_ref(offset) }; - // Actually free the blob. - let hash = granule.blob_hash(); - - // The size of `deleted_bytes` is calculated here instead of requesting it from `blob_store`. - // This is because the actual number of bytes deleted depends on the `blob_store`'s logic. - // We prefer to measure it from the datastore's point of view. - let blob_store_deleted_bytes = blob_store - .retrieve_blob(&hash) - .expect("failed to free var-len blob") - .len() - .into(); - - // Actually free the blob. - blob_store.free_blob(&hash).expect("failed to free var-len blob"); - - blob_store_deleted_bytes + blobs.free_ref(granule.blob_hash()).map(BlobNumBytes::from) } /// Frees an entire var-len linked-list object. @@ -963,14 +948,14 @@ impl<'page> VarView<'page> { /// Frees an entire var-len linked-list object. /// /// SAFETY: `var_len_obj.first_granule` must point to a valid [`VarLenGranule`] or be NULL. - unsafe fn free_object(&mut self, var_len_obj: VarLenRef, blob_store: &mut dyn BlobStore) -> BlobNumBytes { + unsafe fn free_object(&mut self, var_len_obj: VarLenRef, blobs: &BlobSet) -> Result { let mut blob_store_deleted_bytes = BlobNumBytes::default(); // For large blob objects, extract the hash and tell `blob_store` to discard it. if var_len_obj.is_large_blob() { // SAFETY: `var_len_obj.first_granule` was promised to // point to a valid [`VarLenGranule`] or be NULL, as required. unsafe { - blob_store_deleted_bytes = self.free_blob(var_len_obj.first_granule, blob_store); + blob_store_deleted_bytes = self.free_blob(var_len_obj.first_granule, blobs)?; } } @@ -979,7 +964,7 @@ impl<'page> VarView<'page> { self.free_object_ignore_blob(var_len_obj); } - blob_store_deleted_bytes + Ok(blob_store_deleted_bytes) } } @@ -1246,7 +1231,7 @@ impl Page { /// /// Note that large blobs themselves are not counted. /// The caller should obtain a count of the bytes used by large blobs - /// from the [`super::blob_store::BlobStore`]. + /// from the [`super::tiered::BlobSet`]. /// /// This method runs in constant time. pub fn bytes_used_by_rows(&self, fixed_row_size: Size) -> usize { @@ -1384,7 +1369,7 @@ impl Page { fixed_row: &Bytes, var_len_objects: &[impl AsRef<[u8]>], var_len_visitor: &impl VarLenMembers, - blob_store: &mut dyn BlobStore, + blobs: &mut BlobSet, ) -> Result { // Allocate the fixed-len row. let fixed_row_size = Size(fixed_row.len() as u16); @@ -1408,13 +1393,13 @@ impl Page { for (var_len_ref_slot, var_len_obj) in vlr_slot_iter.zip(var_len_objects) { let (var_len_ref, in_blob) = var.alloc_for_slice(var_len_obj.as_ref())?; if in_blob { - // The blob store insertion will never fail. // SAFETY: `alloc_for_slice` always returns a pointer // to a `VarLenGranule` in bounds of this page. // As `in_blob` holds, it is also unused, as required. // We'll now make that granule valid. unsafe { - var.write_large_blob_hash_to_granule(blob_store, var_len_obj, var_len_ref); + var.write_large_blob_hash_to_granule(blobs, var_len_obj, var_len_ref) + .expect("legacy Page::insert_row cannot propagate blob errors"); } } *var_len_ref_slot = var_len_ref; @@ -1553,8 +1538,8 @@ impl Page { fixed_row: PageOffset, fixed_row_size: Size, var_len_visitor: &impl VarLenMembers, - blob_store: &mut dyn BlobStore, - ) -> BlobNumBytes { + blobs: &BlobSet, + ) -> Result { // We're modifying the page, so clear the unmodified hash. self.header.unmodified_hash = None; @@ -1571,7 +1556,7 @@ impl Page { // which we've justified that the above is, // returns an iterator, that will only yield `var_len_ref`s, // where `var_len_ref.first_granule` points to a valid `VarLenGranule` or is NULL. - blob_store_deleted_bytes += unsafe { var.free_object(*var_len_ref, blob_store) } + blob_store_deleted_bytes += unsafe { var.free_object(*var_len_ref, blobs)? } } // SAFETY: Caller promised that `fixed_row` points to a valid row in the page. @@ -1583,7 +1568,7 @@ impl Page { fixed.free(fixed_row, fixed_row_size); } - blob_store_deleted_bytes + Ok(blob_store_deleted_bytes) } /// Returns the total number of granules used by the fixed row at `fixed_row_offset` @@ -1915,26 +1900,17 @@ impl Page { } pub fn metadata(&self, fixed_row_size: Size) -> PageMetadata { - PageMetadata { - num_rows: self.num_rows() as _, - bytes_used_by_rows: self.bytes_used_by_rows(fixed_row_size) as _, - has_free_fixed_slot: self.header.fixed.next_free.has(), - gap_bytes: gap_remaining_size(self.header.var.first, self.header.fixed.last).0 as _, - available_granules: self.available_var_len_granules() as _, - } - } - - pub fn capacity(&self, fixed_row_size: Size) -> PageCapacity { let allocated_fixed_slots = self.header.fixed.last / fixed_row_size; let free_fixed_slots = allocated_fixed_slots .checked_sub(self.header.fixed.num_rows as usize) .expect("live row count exceeds allocated fixed slots"); - PageCapacity { - num_rows: self.num_rows(), - gap_size: gap_remaining_size(self.header.var.first, self.header.fixed.last), - free_fixed_slots, - available_granules: self.available_var_len_granules(), + PageMetadata { + num_rows: self.num_rows() as _, + bytes_used_by_rows: self.bytes_used_by_rows(fixed_row_size) as _, + free_fixed_slots: free_fixed_slots as _, + gap_bytes: gap_remaining_size(self.header.var.first, self.header.fixed.last).0 as _, + available_granules: self.available_var_len_granules() as _, } } } @@ -1999,7 +1975,7 @@ impl<'page> Iterator for VarLenGranulesIter<'page> { pub struct PageMetadata { pub num_rows: u16, pub bytes_used_by_rows: u32, - has_free_fixed_slot: bool, + free_fixed_slots: u16, gap_bytes: u16, available_granules: u16, } @@ -2007,7 +1983,7 @@ pub struct PageMetadata { impl PageMetadata { pub fn has_space_for_row(&self, fixed_row_size: Size, num_var_len_granules: usize) -> bool { has_space_for_row( - self.has_free_fixed_slot, + self.free_fixed_slots != 0, Size(self.gap_bytes), self.available_granules as _, fixed_row_size, @@ -2126,10 +2102,29 @@ impl PageCapacity { } } +impl From for PageCapacity { + fn from( + PageMetadata { + num_rows, + free_fixed_slots, + gap_bytes, + available_granules, + .. + }: PageMetadata, + ) -> Self { + Self { + num_rows: num_rows as _, + gap_size: Size(gap_bytes), + free_fixed_slots: free_fixed_slots as _, + available_granules: available_granules as _, + } + } +} + #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::{blob_store::NullBlobStore, page_pool::PagePool, var_len::AlignedVarLenOffsets}; + use crate::{page_pool::PagePool, var_len::AlignedVarLenOffsets}; use proptest::{collection::vec, prelude::*}; use spacetimedb_sats::bsatn; use spacetimedb_sats::layout::row_size_for_type; @@ -2147,8 +2142,15 @@ pub(crate) mod tests { fn insert_u64(page: &mut Page, val: u64) -> PageOffset { let val_slice = val.to_le_bytes(); - unsafe { page.insert_row(&val_slice, &[] as &[&[u8]], u64_var_len_visitor(), &mut NullBlobStore) } - .expect("Failed to insert first row") + unsafe { + page.insert_row( + &val_slice, + &[] as &[&[u8]], + u64_var_len_visitor(), + &mut BlobSet::new_for_test(), + ) + } + .expect("Failed to insert first row") } fn insert_u64_drop(page: &mut Page, val: u64) { @@ -2156,7 +2158,7 @@ pub(crate) mod tests { } fn assert_page_summaries_match(page: &Page, capacity: &PageCapacity, fixed_row_size: Size) { - let canonical_capacity = page.capacity(fixed_row_size); + let canonical_capacity = PageCapacity::from(page.metadata(fixed_row_size)); assert_eq!(capacity.num_rows, canonical_capacity.num_rows); assert_eq!(capacity.gap_size, canonical_capacity.gap_size); assert_eq!(capacity.free_fixed_slots, canonical_capacity.free_fixed_slots); @@ -2168,7 +2170,7 @@ pub(crate) mod tests { metadata.bytes_used_by_rows as usize, page.bytes_used_by_rows(fixed_row_size) ); - assert_eq!(metadata.has_free_fixed_slot, page.header.fixed.next_free.has()); + assert_eq!(metadata.free_fixed_slots != 0, page.header.fixed.next_free.has()); assert_eq!( Size(metadata.gap_bytes), gap_remaining_size(page.header.var.first, page.header.fixed.last) @@ -2186,7 +2188,7 @@ pub(crate) mod tests { fn page_summaries_match_page_after_reservations() { let fixed_row_size = u64_row_size(); let mut page = Page::new(fixed_row_size); - let mut capacity = page.capacity(fixed_row_size); + let mut capacity = PageCapacity::from(page.metadata(fixed_row_size)); assert_page_summaries_match(&page, &capacity, fixed_row_size); for val in 0..64 { @@ -2283,8 +2285,9 @@ pub(crate) mod tests { assert_ne!(hash_pre_ins, hash_pre_del); // Delete rows. + let blobs = &mut BlobSet::new_for_test(); for row_offset in odds { - unsafe { page.delete_row(row_offset, u64_row_size(), u64_var_len_visitor(), &mut NullBlobStore) }; + unsafe { page.delete_row(row_offset, u64_row_size(), u64_var_len_visitor(), blobs) }.unwrap(); } // The hash should have been cleared. @@ -2325,7 +2328,15 @@ pub(crate) mod tests { assert_ne!(hash_pre_ins, hash_pre_del); // Delete first row. - unsafe { page.delete_row(offset_0, u64_row_size(), u64_var_len_visitor(), &mut NullBlobStore) }; + unsafe { + page.delete_row( + offset_0, + u64_row_size(), + u64_var_len_visitor(), + &BlobSet::new_for_test(), + ) + } + .unwrap(); assert_eq!(page.header.fixed.last, u64_row_size() * 2); @@ -2360,14 +2371,22 @@ pub(crate) mod tests { fn insert_str(page: &mut Page, data: &[u8]) -> PageOffset { let fixed_len_data = [0u8; STR_ROW_SIZE.len()]; - unsafe { page.insert_row(&fixed_len_data, &[data], str_var_len_visitor(), &mut NullBlobStore) } - .expect("Failed to insert row") + unsafe { + page.insert_row( + &fixed_len_data, + &[data], + str_var_len_visitor(), + &mut BlobSet::new_for_test(), + ) + } + .expect("Failed to insert row") } #[test] fn page_summaries_match_page_with_var_len_freelist() { let mut page = Page::new(STR_ROW_SIZE); - let mut capacity = page.capacity(STR_ROW_SIZE); + let blobs = &mut BlobSet::new_for_test(); + let mut capacity = PageCapacity::from(page.metadata(STR_ROW_SIZE)); let one_granule = [0xa5; 1]; let one_granule_count = Page::total_granules_required_for_objects(&[one_granule]); let mut offsets = Vec::new(); @@ -2380,7 +2399,7 @@ pub(crate) mod tests { for offset in offsets.into_iter().step_by(2) { capacity.release_row(one_granule_count); - unsafe { page.delete_row(offset, STR_ROW_SIZE, str_var_len_visitor(), &mut NullBlobStore) }; + unsafe { page.delete_row(offset, STR_ROW_SIZE, str_var_len_visitor(), blobs) }.unwrap(); assert_page_summaries_match(&page, &capacity, STR_ROW_SIZE); } @@ -2487,6 +2506,8 @@ pub(crate) mod tests { let data_2 = b"Glad to hear it."; let mut page = Page::new(STR_ROW_SIZE); + let blobs = &mut BlobSet::new_for_test(); + let offset_0 = insert_str(&mut page, data_0); let offset_1 = insert_str(&mut page, data_1); @@ -2506,7 +2527,7 @@ pub(crate) mod tests { let hash_pre_del = hash_unmodified_save_get(&mut page); - unsafe { page.delete_row(offset_0, STR_ROW_SIZE, str_var_len_visitor(), &mut NullBlobStore) }; + unsafe { page.delete_row(offset_0, STR_ROW_SIZE, str_var_len_visitor(), blobs) }.unwrap(); let hash_pre_ins = hash_unmodified_save_get(&mut page); @@ -2530,6 +2551,7 @@ pub(crate) mod tests { #[test] fn free_var_len_obj_multiple_granules() { let mut page = Page::new(STR_ROW_SIZE); + let blobs = &mut BlobSet::new_for_test(); // Allocate a 4-granule var-len object. let data_0 = [0xa5u8].repeat(VarLenGranule::DATA_SIZE * 4); @@ -2547,7 +2569,7 @@ pub(crate) mod tests { assert_eq!(page.header.var.first.idx(), data_sub_n_vlg(4)); // Delete the row. - unsafe { page.delete_row(offset_0, STR_ROW_SIZE, str_var_len_visitor(), &mut NullBlobStore) }; + unsafe { page.delete_row(offset_0, STR_ROW_SIZE, str_var_len_visitor(), blobs) }.unwrap(); // Allocate a new 4-granule var-len object. // This should use the same storage as the original row. @@ -2582,6 +2604,7 @@ pub(crate) mod tests { let data_2 = data_2.as_ref(); let mut page = Page::new(STR_ROW_SIZE); + let blobs = &mut BlobSet::new_for_test(); // First the hash is not saved, so compute it. let hash_pre_ins = hash_unmodified_save_get(&mut page); @@ -2597,7 +2620,7 @@ pub(crate) mod tests { assert_ne!(hash_pre_ins, hash_pre_del); // Delete the first row. - unsafe { page.delete_row(offset_0, STR_ROW_SIZE, str_var_len_visitor(), &mut NullBlobStore) }; + unsafe { page.delete_row(offset_0, STR_ROW_SIZE, str_var_len_visitor(), blobs) }.unwrap(); // Hash is cleared by deleting. let hash_post_del = hash_unmodified_save_get(&mut page); @@ -2677,6 +2700,7 @@ pub(crate) mod tests { #[test] fn fill_delete_iter_var_len_str() { let mut page = Page::new(STR_ROW_SIZE); + let blobs = &mut BlobSet::new_for_test(); // First the hash is not saved, so compute it. let hash_pre_ins = hash_unmodified_save_get(&mut page); @@ -2700,7 +2724,7 @@ pub(crate) mod tests { // Delete the rows. for row_offset in odds { - unsafe { page.delete_row(row_offset, STR_ROW_SIZE, str_var_len_visitor(), &mut NullBlobStore) }; + unsafe { page.delete_row(row_offset, STR_ROW_SIZE, str_var_len_visitor(), blobs) }.unwrap(); } // Hash was cleared by deleting and is different now. @@ -2728,6 +2752,7 @@ pub(crate) mod tests { fn serde_round_trip_whole_page() { let pool = PagePool::new_for_test(); let mut page = Page::new(u64_row_size()); + let blobs = &mut BlobSet::new_for_test(); // Construct an empty page, ser/de it, and assert that it's still empty. let hash_pre_ins = hash_unmodified_save_get(&mut page); @@ -2761,7 +2786,7 @@ pub(crate) mod tests { .enumerate() .filter_map(|(i, offset)| { if i % 2 == 0 { - unsafe { page.delete_row(offset, u64_row_size(), u64_var_len_visitor(), &mut NullBlobStore) }; + unsafe { page.delete_row(offset, u64_row_size(), u64_var_len_visitor(), blobs) }.unwrap(); None } else { Some(offset) diff --git a/crates/table/src/pages.rs b/crates/table/src/pages.rs index 975d9715b5b..95c9918b56e 100644 --- a/crates/table/src/pages.rs +++ b/crates/table/src/pages.rs @@ -1,6 +1,7 @@ //! Provides [`Pages`], a page manager dealing with [`Page`]s as a collection. -use super::blob_store::BlobStore; +use crate::tiered::{BlobError, BlobSet}; + use super::indexes::{Bytes, PageIndex, PageOffset, RowPointer}; use super::page::Page; use super::page_pool::PagePool; @@ -217,7 +218,7 @@ impl Pages { fixed_row_size: Size, fixed_len: &Bytes, var_len: &[&[u8]], - blob_store: &mut dyn BlobStore, + blob_store: &mut BlobSet, ) -> Result<(PageIndex, PageOffset), Error> { debug_assert!(fixed_len.len() == fixed_row_size.len()); @@ -261,8 +262,8 @@ impl Pages { var_len_visitor: &impl VarLenMembers, fixed_row_size: Size, row_ptr: RowPointer, - blob_store: &mut dyn BlobStore, - ) -> BlobNumBytes { + blob_store: &mut BlobSet, + ) -> Result { let page_index = row_ptr.page_index(); self.with_updating_non_full_pages(page_index, fixed_row_size, |this| { diff --git a/crates/table/src/read_column.rs b/crates/table/src/read_column.rs index 22c4cf9e549..6763417d142 100644 --- a/crates/table/src/read_column.rs +++ b/crates/table/src/read_column.rs @@ -378,8 +378,8 @@ unsafe impl ReadColumn for SumTag { #[cfg(test)] mod test { use super::*; - use crate::blob_store::HashMapBlobStore; use crate::table::test::table; + use crate::tiered::BlobSet; use proptest::{prelude::*, prop_assert_eq, proptest, test_runner::TestCaseResult}; use spacetimedb_sats::{product, proptest::generate_typed_row}; @@ -393,10 +393,10 @@ mod test { /// inserting the row, then doing `AlgebraicValue::read_column` on each column of the row /// returns the expected value. fn read_column_same_value((ty, val) in generate_typed_row()) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty); - let (_, row_ref) = table.insert(&mut blob_store, &val).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &val).unwrap(); for (idx, orig_col_value) in val.into_iter().enumerate() { let read_col_value = row_ref.read_col::(idx).unwrap(); @@ -409,10 +409,10 @@ mod test { /// which does not match the actual column type /// returns an appropriate error. fn read_column_wrong_type((ty, val) in generate_typed_row()) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty.clone()); - let (_, row_ref) = table.insert(&mut blob_store, &val).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &val).unwrap(); for (idx, col_ty) in ty.elements.iter().enumerate() { assert_wrong_type_error::(&row_ref, idx, &col_ty.algebraic_type, AlgebraicType::U8)?; @@ -439,10 +439,10 @@ mod test { /// i.e. with an out-of-bounds index, /// returns an appropriate error. fn read_column_out_of_bounds((ty, val) in generate_typed_row()) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty.clone()); - let (_, row_ref) = table.insert(&mut blob_store, &val).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &val).unwrap(); let oob = ty.elements.len(); @@ -496,7 +496,7 @@ mod test { ($name:ident { $algebraic_type:expr => $rust_type:ty = $val:expr }) => { #[test] fn $name() { - let mut blob_store = HashMapBlobStore::default(); + let mut blob_store = BlobSet::new_for_test(); let mut table = table(ProductType::from_iter([$algebraic_type])); let val: $rust_type = $val; @@ -557,11 +557,11 @@ mod test { fn read_sum_tag_from_sum_with_payload() { let algebraic_type = AlgebraicType::sum([("a", AlgebraicType::U8), ("b", AlgebraicType::U16)]); - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ProductType::from([algebraic_type])); let val = SumValue::new(1, 42u16); - let (_, row_ref) = table.insert(&mut blob_store, &product![val.clone()]).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &product![val.clone()]).unwrap(); assert_eq!(val.tag, row_ref.read_col::(0).unwrap().0); } diff --git a/crates/table/src/row_hash.rs b/crates/table/src/row_hash.rs index 0cf2cd3cc83..135b2b91561 100644 --- a/crates/table/src/row_hash.rs +++ b/crates/table/src/row_hash.rs @@ -8,7 +8,7 @@ use super::{ page::Page, var_len::VarLenRef, }; -use crate::{bflatn_from::vlr_blob_bytes, blob_store::BlobStore}; +use crate::{bflatn_from::vlr_blob_bytes, tiered::BlobReadSet}; use core::hash::{Hash as _, Hasher}; use core::mem; use core::str; @@ -32,7 +32,7 @@ use spacetimedb_sats::{algebraic_value::ser::concat_byte_chunks_buf, bsatn::Dese pub unsafe fn hash_row_in_page( hasher: &mut impl Hasher, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, fixed_offset: PageOffset, ty: &RowTypeLayout, ) { @@ -42,7 +42,7 @@ pub unsafe fn hash_row_in_page( // - Per 1. and 2., `fixed_bytes` points at a row in `page` valid for `ty`. // - Per 3., for any `vlr: VarLenRef` stored in `fixed_bytes`, // `vlr.first_offset` is either `NULL` or points to a valid granule in `page`. - unsafe { hash_product(hasher, fixed_bytes, page, blob_store, &mut 0, ty.product()) }; + unsafe { hash_product(hasher, fixed_bytes, page, blobs, &mut 0, ty.product()) }; } /// Hashes every product field in `value = &bytes[range_move(0..ty.size(), *curr_offset)]` @@ -56,7 +56,7 @@ unsafe fn hash_product( hasher: &mut impl Hasher, bytes: &Bytes, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, curr_offset: &mut usize, ty: ProductTypeLayoutView<'_>, ) { @@ -69,7 +69,7 @@ unsafe fn hash_product( // are valid `elem_ty.ty`s. // By 2., and the above, it follows that sub-`value`s won't have dangling `VarLenRef`s. unsafe { - hash_value(hasher, bytes, page, blob_store, curr_offset, &elem_ty.ty); + hash_value(hasher, bytes, page, blobs, curr_offset, &elem_ty.ty); } } } @@ -85,7 +85,7 @@ unsafe fn hash_value( hasher: &mut impl Hasher, bytes: &Bytes, page: &Page, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, curr_offset: &mut usize, ty: &AlgebraicTypeLayout, ) { @@ -109,12 +109,12 @@ unsafe fn hash_value( // we know `data_value = &bytes[range_move(0..data_ty.size(), data_offset))` // is valid at `data_ty`. // By 2., and the above, we also know that `data_value` won't have dangling `VarLenRef`s. - unsafe { hash_value(hasher, bytes, page, blob_store, &mut data_offset, data_ty) }; + unsafe { hash_value(hasher, bytes, page, blobs, &mut data_offset, data_ty) }; *curr_offset += ty.size(); } AlgebraicTypeLayout::Product(ty) => { // SAFETY: `value` was valid at `ty` and `VarLenRef`s won't be dangling. - unsafe { hash_product(hasher, bytes, page, blob_store, curr_offset, ty.view()) } + unsafe { hash_product(hasher, bytes, page, blobs, curr_offset, ty.view()) } } // The primitive types: @@ -152,7 +152,7 @@ unsafe fn hash_value( // Moreover, `vlr.first_granule` was promised by the caller // to either be `NULL` or point to a valid granule in `page`. unsafe { - run_vlo_bytes(page, bytes, blob_store, curr_offset, |bytes| { + run_vlo_bytes(page, bytes, blobs, curr_offset, |bytes| { // SAFETY: For `::String`, the blob will always be valid UTF-8. let string = str::from_utf8_unchecked(bytes); string.hash(hasher) @@ -168,7 +168,7 @@ unsafe fn hash_value( // Moreover, `vlr.first_granule` was promised by the caller // to either be `NULL` or point to a valid granule in `page`. unsafe { - run_vlo_bytes(page, bytes, blob_store, curr_offset, |mut bsatn| { + run_vlo_bytes(page, bytes, blobs, curr_offset, |mut bsatn| { let de = Deserializer::new(&mut bsatn); spacetimedb_sats::hash_bsatn_array(hasher, ty, de).unwrap(); }); @@ -187,7 +187,7 @@ unsafe fn hash_value( pub(crate) unsafe fn run_vlo_bytes( page: &Page, bytes: &Bytes, - blob_store: &dyn BlobStore, + blobs: &BlobReadSet, curr_offset: &mut usize, run: impl FnOnce(&[u8]) -> R, ) -> R { @@ -198,8 +198,8 @@ pub(crate) unsafe fn run_vlo_bytes( if vlr.is_large_blob() { // SAFETY: As `vlr` is a blob, `vlr.first_granule` always points to a valid granule. - let bytes = unsafe { vlr_blob_bytes(page, blob_store, vlr) }; - run(bytes) + let bytes = unsafe { vlr_blob_bytes(page, blobs, vlr) }; + run(&bytes) } else { // SAFETY: `vlr.first_granule` is either NULL or points to a valid granule. let var_iter = unsafe { page.iter_vlo_data(vlr.first_granule) }; @@ -228,7 +228,7 @@ pub unsafe fn read_from_bytes(bytes: &Bytes, curr_offset: &mut usize) - #[cfg(test)] mod tests { - use crate::blob_store::HashMapBlobStore; + use crate::tiered::BlobSet; use core::hash::BuildHasher; use proptest::prelude::*; use spacetimedb_sats::proptest::generate_typed_row; @@ -239,7 +239,7 @@ mod tests { fn pv_row_ref_hash_same_std_random_state((ty, val) in generate_typed_row()) { // Turn `val` into a `RowRef`. let mut table = crate::table::test::table(ty); - let blob_store = &mut HashMapBlobStore::default(); + let blob_store = &mut BlobSet::new_for_test(); let (_, row) = table.insert(blob_store, &val).unwrap(); // Check hashing algos. @@ -250,7 +250,7 @@ mod tests { #[test] fn pv_row_ref_hash_same_ahash((ty, val) in generate_typed_row()) { // Turn `val` into a `RowRef`. - let blob_store = &mut HashMapBlobStore::default(); + let blob_store = &mut BlobSet::new_for_test(); let mut table = crate::table::test::table(ty); let (_, row) = table.insert(blob_store, &val).unwrap(); diff --git a/crates/table/src/static_bsatn_validator.rs b/crates/table/src/static_bsatn_validator.rs index 6c86f71b607..f12700264e2 100644 --- a/crates/table/src/static_bsatn_validator.rs +++ b/crates/table/src/static_bsatn_validator.rs @@ -371,9 +371,7 @@ pub(crate) unsafe fn validate_bsatn( #[cfg(test)] pub mod test { use super::*; - use crate::{ - bflatn_to::write_row_to_page, blob_store::HashMapBlobStore, page::Page, row_type_visitor::row_type_visitor, - }; + use crate::{bflatn_to::write_row_to_page, page::Page, row_type_visitor::row_type_visitor, tiered::BlobSet}; use proptest::{prelude::*, prop_assert_eq, proptest}; use spacetimedb_sats::bsatn::to_vec; use spacetimedb_sats::proptest::generate_typed_row; @@ -400,7 +398,7 @@ pub mod test { let mut page = Page::new(ty.size()); let visitor = row_type_visitor(&ty); - let blob_store = &mut HashMapBlobStore::default(); + let blob_store = &mut BlobSet::new_for_test(); let res_write = unsafe { write_row_to_page(&mut page, blob_store, &visitor, &ty, &val) }; prop_assert_eq!(res_validate.is_ok(), res_write.is_ok()); diff --git a/crates/table/src/static_layout.rs b/crates/table/src/static_layout.rs index 0f8ac12c117..891fcf5bb52 100644 --- a/crates/table/src/static_layout.rs +++ b/crates/table/src/static_layout.rs @@ -424,7 +424,7 @@ impl LayoutBuilder { #[cfg(test)] mod test { use super::*; - use crate::blob_store::HashMapBlobStore; + use crate::tiered::BlobSet; use proptest::prelude::*; use spacetimedb_sats::{bsatn, proptest::generate_typed_row, AlgebraicType, ProductType}; @@ -641,7 +641,7 @@ mod test { #[test] fn known_bsatn_same_as_bflatn_from((ty, val) in generate_typed_row()) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = crate::table::test::table(ty); let Some(static_layout) = table.static_layout().cloned() else { // `ty` has a var-len member or a sum with different payload lengths, @@ -649,7 +649,7 @@ mod test { return Err(TestCaseError::reject("Var-length type")); }; - let (_, row_ref) = table.insert(&mut blob_store, &val).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &val).unwrap(); row_ref.with_row_data(|bytes| { let slow_path = bsatn::to_vec(&row_ref).unwrap(); @@ -669,7 +669,7 @@ mod test { #[test] fn known_bflatn_same_as_pv_from((ty, val) in generate_typed_row()) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = crate::table::test::table(ty); let Some(static_layout) = table.static_layout().cloned() else { // `ty` has a var-len member or a sum with different payload lengths, @@ -678,7 +678,7 @@ mod test { }; let bsatn = bsatn::to_vec(&val).unwrap(); - let (_, row_ref) = table.insert(&mut blob_store, &val).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &val).unwrap(); row_ref.with_row_data(|slow_path| { let mut fast_path = vec![0u8; slow_path.len()]; unsafe { diff --git a/crates/table/src/table.rs b/crates/table/src/table.rs index ad029ce9e04..6a533e14ce0 100644 --- a/crates/table/src/table.rs +++ b/crates/table/src/table.rs @@ -1,13 +1,15 @@ use crate::{ bflatn_to::required_var_len_granules_for_row, page, - tiered::{PageError, PageEvictionPolicy, PageHandle, PageManager, PageSet, PreparedCommit, ReservedPage}, + tiered::{ + BlobError, BlobReadSet, BlobSet, DeleteRowError, PageError, PageEvictionPolicy, PageHandle, PageManager, + PageSet, PreparedCommit, ReservedPage, + }, }; use super::{ bflatn_from::{serialize_columns_from_page, serialize_row_from_page}, bflatn_to::{write_row_to_pages, write_row_to_pages_bsatn, Error}, - blob_store::{BlobStore, NullBlobStore}, eq::eq_row_in_page, eq_to_pv::eq_row_in_page_to_pv, indexes::{Bytes, PageIndex, PageOffset, RowHash, RowPointer, SquashedOffset, PAGE_DATA_SIZE}, @@ -20,7 +22,6 @@ use super::{ static_bsatn_validator::{static_bsatn_validator, validate_bsatn, StaticBsatnValidator}, static_layout::StaticLayout, table_index::{IndexCannotSeekRange, IndexKey, IndexKind, TableIndex, TableIndexPointIter, TableIndexRangeIter}, - var_len::VarLenMembers, }; use core::{fmt, ptr}; use core::{ @@ -62,15 +63,38 @@ use std::{ }; use thiserror::Error; -/// The number of bytes used by, added to, or removed from a [`Table`]'s share of a [`BlobStore`]. +/// The number of bytes used by, added to, or removed from a [`Table`]'s share of a [`BlobSet`]. #[derive(Copy, Clone, PartialEq, Eq, Debug, Default, From, Add, Sub, AddAssign, SubAssign)] pub struct BlobNumBytes(usize); +impl From for BlobNumBytes { + fn from(v: u64) -> Self { + Self(v as usize) + } +} + impl MemoryUsage for BlobNumBytes {} pub type SeqIdList = SmallVec<[SequenceId; 4]>; static_assert_size!(SeqIdList, 24); +#[derive(Debug, Error)] +pub enum TableError { + #[error(transparent)] + Page(#[from] PageError), + #[error(transparent)] + Blob(#[from] BlobError), +} + +impl From for TableError { + fn from(e: DeleteRowError) -> Self { + match e { + DeleteRowError::Blob(e) => Self::Blob(e), + DeleteRowError::Page(e) => Self::Page(e), + } + } +} + /// A database table containing the row schema, the rows, and indices. /// /// The table stores the rows into a page manager @@ -100,7 +124,7 @@ pub struct Table { pub row_count: u64, /// Stores the sum total number of bytes that each blob object in the table occupies. /// - /// Note that the [`HashMapBlobStore`] does ref-counting and de-duplication, + /// Note that the [`BlobSet`] does ref-counting and de-duplication, /// but this sum will count an object each time its hash is mentioned, rather than just once. blob_store_bytes: BlobNumBytes, /// Indicates whether this is a scheduler table or not. @@ -162,16 +186,16 @@ impl TableInner { /// is sufficient to demonstrate all of these properties. unsafe fn get_row_ref_unchecked<'a>( &'a self, - blob_store: &'a dyn BlobStore, + blobs: &'a BlobSet, squashed_offset: SquashedOffset, ptr: RowPointer, - ) -> Result, PageError> { + ) -> Result, TableError> { let page = self .pages .get_page(ptr.page_index())? .expect("page for get_row_ref_unchecked to be present"); // SAFETY: Forward caller requirements. - Ok(unsafe { RowRef::new(self, blob_store, squashed_offset, ptr, page) }) + unsafe { RowRef::new(self, blobs, squashed_offset, ptr, page) }.map_err(Into::into) } /// Returns whether the row at `ptr` is present or not. @@ -214,7 +238,7 @@ impl TableInner { } } -static_assert_size!(Table, 304); +static_assert_size!(Table, 328); impl MemoryUsage for Table { fn heap_usage(&self) -> usize { @@ -274,7 +298,7 @@ pub enum InsertError { IndexError(#[from] UniqueConstraintViolation), #[error(transparent)] - Page(#[from] PageError), + Table(#[from] TableError), } /// Errors that can occur while trying to read a value via bsatn. @@ -367,7 +391,7 @@ pub enum TableNotEmptyError { #[error("Table is not empty")] TableNotEmpty, #[error(transparent)] - Page(#[from] PageError), + Table(#[from] TableError), } // Public API: @@ -420,7 +444,7 @@ impl Table { // Remove and drop any pages, as even though they must be empty, // they may have residual layout-derived data which conflicts with the new schema. // Safety: there aren't any pages here, so they cannot conflict with the schema or row layout. - unsafe { self.set_pages(Vec::new(), &NullBlobStore)? }; + unsafe { self.set_pages(Vec::new(), &BlobSet::new_for_test())? }; // Safety: the table has no rows according to its row count, // and no pages 'cause we just did `set_pages` to the empty vec. @@ -660,21 +684,21 @@ impl Table { /// TODO(error-handling): describe errors from `write_row_to_pages` and return meaningful errors. pub fn insert<'a>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, row: &ProductValue, ) -> Result<(Option, RowRef<'a>), InsertError> { - self.insert_with_reservation(blob_store, row, None) + self.insert_with_reservation(blobs, row, None) } pub(crate) fn insert_with_reservation<'a>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, row: &ProductValue, reservation: Option<(PageIndex, ReservedPage)>, ) -> Result<(Option, RowRef<'a>), InsertError> { // Optimistically insert the `row` before checking any constraints // under the assumption that errors (unique constraint & set semantic violations) are rare. - let (row_ref, blob_bytes) = self.insert_physically_pv(blob_store, row, reservation)?; + let (row_ref, blob_bytes) = self.insert_physically_pv(blobs, row, reservation)?; let row_ptr = row_ref.into_pointer(); // Confirm the insertion, checking any constraints, removing the physical row on error. @@ -683,24 +707,24 @@ impl Table { // where `insert` is called, we are not dealing with transactions, // and we already know there cannot be a duplicate row error, // but we check just in case it isn't. - let (hash, row_ptr) = unsafe { self.confirm_insertion::(blob_store, row_ptr, blob_bytes) }?; + let (hash, row_ptr) = unsafe { self.confirm_insertion::(blobs, row_ptr, blob_bytes) }?; // SAFETY: Per post-condition of `confirm_insertion`, `row_ptr` refers to a valid row. - let row_ref = unsafe { self.get_row_ref_unchecked(blob_store, row_ptr) }?; + let row_ref = unsafe { self.get_row_ref_unchecked(blobs, row_ptr) }?; Ok((hash, row_ref)) } pub(crate) fn insert_prepared<'a>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, prepared: &PreparedInsert, reservation: Option<(PageIndex, ReservedPage)>, ) -> Result<(Option, RowRef<'a>), InsertError> { let (row_ref, blob_bytes) = match &prepared.inner { - PreparedInsertInner::Product(row) => self.insert_physically_pv(blob_store, row, reservation), + PreparedInsertInner::Product(row) => self.insert_physically_pv(blobs, row, reservation), PreparedInsertInner::CopyFixed { source_page, source_offset, - } => self.insert_physically_copy_bflatn(blob_store, source_page, *source_offset, reservation), + } => self.insert_physically_copy_bflatn(blobs, source_page, *source_offset, reservation), }?; let row_ptr = row_ref.into_pointer(); @@ -711,16 +735,16 @@ impl Table { // where `insert` is called, we are not dealing with transactions, // and we already know there cannot be a duplicate row error, // but we check just in case it isn't. - let (hash, row_ptr) = unsafe { self.confirm_insertion::(blob_store, row_ptr, blob_bytes) }?; + let (hash, row_ptr) = unsafe { self.confirm_insertion::(blobs, row_ptr, blob_bytes) }?; // SAFETY: Per post-condition of `confirm_insertion`, `row_ptr` refers to a valid row. - let row_ref = unsafe { self.get_row_ref_unchecked(blob_store, row_ptr) }?; + let row_ref = unsafe { self.get_row_ref_unchecked(blobs, row_ptr) }?; Ok((hash, row_ref)) } fn insert_physically_copy_bflatn<'a>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, source_page: &PageHandle, source_offset: PageOffset, reservation: Option<(PageIndex, ReservedPage)>, @@ -746,7 +770,7 @@ impl Table { let ptr = RowPointer::new(false, page_index, fixed_offset, squashed_offset); // SAFETY: We just inserted `ptr`, so it must be present. - let row_ref = unsafe { self.inner.get_row_ref_unchecked(blob_store, squashed_offset, ptr) }?; + let row_ref = unsafe { self.inner.get_row_ref_unchecked(blobs, squashed_offset, ptr) }?; Ok((row_ref, BlobNumBytes::default())) } @@ -758,7 +782,7 @@ impl Table { /// A call to this method should be followed by a call to [`delete_internal_skip_pointer_map`]. pub fn insert_physically_pv<'a>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, row: &ProductValue, reservation: Option<(PageIndex, ReservedPage)>, ) -> Result<(RowRef<'a>, BlobNumBytes), Error> { @@ -769,14 +793,14 @@ impl Table { &mut self.inner.pages, reservation, &self.inner.visitor_prog, - blob_store, + blobs, &self.inner.row_layout, row, self.squashed_offset, ) }?; // SAFETY: We just inserted `ptr`, so it must be present. - let row_ref = unsafe { self.inner.get_row_ref_unchecked(blob_store, self.squashed_offset, ptr) }?; + let row_ref = unsafe { self.inner.get_row_ref_unchecked(blobs, self.squashed_offset, ptr) }?; Ok((row_ref, blob_bytes)) } @@ -796,7 +820,7 @@ impl Table { /// an error is returned and there will be nothing for the caller to revert. pub fn insert_physically_bsatn<'a>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a mut BlobSet, row: &[u8], ) -> Result<(RowRef<'a>, BlobNumBytes), Error> { // Got a static layout? => Use fast-path insertion. @@ -813,7 +837,7 @@ impl Table { .with_page_to_insert_row(fixed_row_size, 0, None, |page| { // SAFETY: We've used the right `row_size` and we trust that others have too. // `RowTypeLayout` also ensures that we satisfy the minimum row size. - let fixed_offset = unsafe { page.alloc_fixed_len(fixed_row_size) }.map_err(Error::PageError)?; + let fixed_offset = unsafe { page.alloc_fixed_len(fixed_row_size) }.map_err(Error::Page)?; let (mut fixed, _) = page.split_fixed_var_mut(); let fixed_buf = fixed.get_row_mut(fixed_offset, fixed_row_size); // SAFETY: @@ -822,9 +846,9 @@ impl Table { unsafe { static_layout.deserialize_row_into(fixed_buf, row) }; Ok(fixed_offset) }) - .map_err(Error::PagesError)?; + .map_err(Error::Pages)?; match res { - (page, Ok(offset)) => (RowPointer::new(false, page, offset, squashed_offset), 0.into()), + (page, Ok(offset)) => (RowPointer::new(false, page, offset, squashed_offset), 0usize.into()), (_, Err(e)) => return Err(e), } } else { @@ -834,7 +858,7 @@ impl Table { write_row_to_pages_bsatn( &mut self.inner.pages, &self.inner.visitor_prog, - blob_store, + blobs, &self.inner.row_layout, row, self.squashed_offset, @@ -843,7 +867,7 @@ impl Table { }; // SAFETY: We just inserted `ptr`, so it must be present. - let row_ref = unsafe { self.inner.get_row_ref_unchecked(blob_store, self.squashed_offset, ptr) }?; + let row_ref = unsafe { self.inner.get_row_ref_unchecked(blobs, self.squashed_offset, ptr) }?; Ok((row_ref, blob_bytes)) } @@ -855,14 +879,14 @@ impl Table { /// `self.is_row_present(row)` must hold. pub unsafe fn sequence_triggers_for<'a>( &'a self, - blob_store: &'a dyn BlobStore, + blobs: &'a BlobSet, row: RowPointer, ) -> Result<(ColList, SeqIdList), Error> { let sequences = &*self.get_schema().sequences; let row_ty = self.row_layout().product(); // SAFETY: Caller promised that `self.is_row_present(row)` holds. - let row_ref = unsafe { self.get_row_ref_unchecked(blob_store, row) }?; + let row_ref = unsafe { self.get_row_ref_unchecked(blobs, row) }?; Ok(sequences .iter() @@ -951,14 +975,14 @@ impl Table { /// `self.is_row_present(row)` must hold. pub unsafe fn confirm_insertion<'a, const CHECK_SAME_ROW: bool>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, ptr: RowPointer, blob_bytes: BlobNumBytes, ) -> Result<(Option, RowPointer), InsertError> { // SAFETY: Caller promised that `self.is_row_present(ptr)` holds. - let hash = unsafe { self.insert_into_pointer_map(blob_store, ptr) }??; + let hash = unsafe { self.insert_into_pointer_map(blobs, ptr) }??; // SAFETY: Caller promised that `self.is_row_present(ptr)` holds. - unsafe { self.insert_into_indices::(blob_store, ptr) }?; + unsafe { self.insert_into_indices::(blobs, ptr) }?; self.update_statistics_added_row(blob_bytes); Ok((hash, ptr)) @@ -979,28 +1003,29 @@ impl Table { /// `self.is_row_present(new_row)` and `self.is_row_present(old_row)` must hold. pub unsafe fn confirm_update<'a>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a mut BlobSet, new_ptr: RowPointer, old_ptr: RowPointer, blob_bytes_added: BlobNumBytes, ) -> Result { // (1) Remove old row from indices. // SAFETY: Caller promised that `self.is_row_present(old_ptr)` holds. - unsafe { self.delete_from_indices(blob_store, old_ptr) }?; + unsafe { self.delete_from_indices(blobs, old_ptr) }?; // Insert new row into indices. // SAFETY: Caller promised that `self.is_row_present(ptr)` holds. - let res = unsafe { self.insert_into_indices::(blob_store, new_ptr) }; + let res = unsafe { self.insert_into_indices::(blobs, new_ptr) }; if let Err(e) = res { // Undo (1). - unsafe { self.insert_into_indices::(blob_store, old_ptr) } + unsafe { self.insert_into_indices::(blobs, old_ptr) } .expect("re-inserting the old row into indices should always work"); return Err(e); } // Remove the old row physically. // SAFETY: The physical `old_ptr` still exists. - let blob_bytes_removed = unsafe { self.delete_internal_skip_pointer_map(blob_store, old_ptr) }?; + let blob_bytes_removed = + unsafe { self.delete_internal_skip_pointer_map(blobs, old_ptr) }.map_err(TableError::from)?; self.update_statistics_deleted_row(blob_bytes_removed); // Update statistics. @@ -1034,13 +1059,13 @@ impl Table { /// Post-condition: If this method returns `Ok(_)`, the row still exists. unsafe fn insert_into_indices<'a, const CHECK_SAME_ROW: bool>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, new: RowPointer, ) -> Result<(), InsertError> { let mut violation = None; for (index_id, index) in self.indexes.iter_mut() { // SAFETY: We just inserted `ptr`, so it must be present. - let new = unsafe { self.inner.get_row_ref_unchecked(blob_store, self.squashed_offset, new) }?; + let new = unsafe { self.inner.get_row_ref_unchecked(blobs, self.squashed_offset, new) }?; // SAFETY: any index in this table was constructed with the same row type as this table. if let Err(old) = unsafe { index.check_and_insert(&new) } { violation = Some((*index_id, old, new)); @@ -1060,7 +1085,7 @@ impl Table { // - The row layouts are the same as it's the same table. // - We know `old` exists in `self` as we just found it in an index. // - Caller promised that `new` is valid for `self`. - && unsafe { Self::eq_row_in_page(self, old_ptr, self, new_ref.pointer()) }? + && unsafe { Self::eq_row_in_page(self, old_ptr, self, new_ref.pointer()) }.map_err(TableError::from)? { DuplicateError(old_ptr).into() } else { @@ -1073,11 +1098,11 @@ impl Table { // Do this before the actual deletion, as `index.delete` needs a `RowRef` // so it can extract the appropriate value. // SAFETY: We just inserted `new`, so it must be present. - unsafe { self.delete_from_indices_until(blob_store, new, index_id)? }; + unsafe { self.delete_from_indices_until(blobs, new, index_id)? }; // Cleanup, undo the row insertion of `new`s. // SAFETY: We just inserted `new`, so it must be present. - unsafe { self.delete_internal(blob_store, new)? }; + unsafe { self.delete_internal(blobs, new)? }; return Err(error); } @@ -1096,9 +1121,9 @@ impl Table { unsafe fn find_same_row_via_unique_index( target_table: &Table, needle_table: &Table, - needle_bs: &dyn BlobStore, + needle_bs: &BlobSet, needle_ptr: RowPointer, - ) -> Result, PageError> { + ) -> Result, TableError> { // Use some index (the one with the lowest `IndexId` currently). // TODO(centril): this isn't what we actually want. // Rather, we'd prefer the index with the simplest type, @@ -1146,9 +1171,9 @@ impl Table { /// Post-condition: If this method returns `Ok(_)`, the row still exists. unsafe fn insert_into_pointer_map<'a>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, ptr: RowPointer, - ) -> Result, DuplicateError>, PageError> { + ) -> Result, DuplicateError>, TableError> { if self.pointer_map.is_none() { // No pointer map? Set semantic constraint is checked by a unique index instead. return Ok(Ok(None)); @@ -1157,7 +1182,7 @@ impl Table { // SAFETY: // - `self` trivially has the same `row_layout` as `self`. // - Caller promised that `self.is_row_present(row)` holds. - let (hash, existing_row) = unsafe { Self::find_same_row_via_pointer_map(self, self, blob_store, ptr, None) }?; + let (hash, existing_row) = unsafe { Self::find_same_row_via_pointer_map(self, self, blobs, ptr, None) }?; if let Some(existing_row) = existing_row { // If an equal row was already present, @@ -1167,7 +1192,7 @@ impl Table { unsafe { self.inner .pages - .delete_row(&self.inner.visitor_prog, self.row_size(), ptr, blob_store)? + .delete_row(&self.inner.visitor_prog, self.row_size(), ptr, blobs)? }; return Ok(Err(DuplicateError(existing_row))); } @@ -1214,10 +1239,10 @@ impl Table { pub unsafe fn find_same_row_via_pointer_map( target_table: &Table, needle_table: &Table, - needle_bs: &dyn BlobStore, + needle_bs: &BlobSet, needle_ptr: RowPointer, row_hash: Option, - ) -> Result<(RowHash, Option), PageError> { + ) -> Result<(RowHash, Option), TableError> { let row_hash = match row_hash { Some(hash) => hash, None => { @@ -1292,10 +1317,10 @@ impl Table { pub unsafe fn find_same_row( target_table: &Table, needle_table: &Table, - needle_bs: &dyn BlobStore, + needle_bs: &BlobSet, needle_ptr: RowPointer, row_hash: Option, - ) -> Result<(Option, Option), PageError> { + ) -> Result<(Option, Option), TableError> { if target_table.pointer_map.is_some() { // SAFETY: Caller promised that `target_table` and `needle_table` have the same `row_layout`. // SAFETY: Caller promised that `needle_table.is_row_present(needle_ptr)`. @@ -1315,9 +1340,9 @@ impl Table { /// Returns a [`RowRef`] for `ptr` or `None` if the row isn't present. pub fn get_row_ref<'a>( &'a self, - blob_store: &'a dyn BlobStore, + blob_store: &'a BlobSet, ptr: RowPointer, - ) -> Result>, PageError> { + ) -> Result>, TableError> { if self.is_row_present(ptr)? { // SAFETY: We only call `get_row_ref_unchecked` when `is_row_present` holds. unsafe { self.get_row_ref_unchecked(blob_store, ptr) }.map(Some) @@ -1345,11 +1370,11 @@ impl Table { /// is sufficient to demonstrate all of these properties. pub unsafe fn get_row_ref_unchecked<'a>( &'a self, - blob_store: &'a dyn BlobStore, + blobs: &'a BlobSet, ptr: RowPointer, - ) -> Result, PageError> { + ) -> Result, TableError> { // SAFETY: Caller promised that ^-- holds. - unsafe { self.inner.get_row_ref_unchecked(blob_store, self.squashed_offset, ptr) } + unsafe { self.inner.get_row_ref_unchecked(blobs, self.squashed_offset, ptr) } } /// Deletes a row in the page manager @@ -1360,9 +1385,9 @@ impl Table { /// `ptr` must point to a valid, live row in this table. pub unsafe fn delete_internal_skip_pointer_map( &mut self, - blob_store: &mut dyn BlobStore, + blobs: &BlobSet, ptr: RowPointer, - ) -> Result { + ) -> Result { debug_assert!(self.is_row_present(ptr).unwrap()); // Delete the physical row. // @@ -1373,7 +1398,7 @@ impl Table { unsafe { self.inner .pages - .delete_row(&self.inner.visitor_prog, self.row_size(), ptr, blob_store) + .delete_row(&self.inner.visitor_prog, self.row_size(), ptr, blobs) } } @@ -1385,15 +1410,11 @@ impl Table { /// Use `delete_unchecked` or `delete` to delete a row with index updating. /// /// SAFETY: `self.is_row_present(row)` must hold. - unsafe fn delete_internal( - &mut self, - blob_store: &mut dyn BlobStore, - ptr: RowPointer, - ) -> Result { + unsafe fn delete_internal(&mut self, blobs: &BlobSet, ptr: RowPointer) -> Result { // Remove the set semantic association. if let Some(pointer_map) = &mut self.pointer_map { // SAFETY: `self.is_row_present(row)` holds. - let row = unsafe { self.inner.get_row_ref_unchecked(blob_store, self.squashed_offset, ptr) }?; + let row = unsafe { self.inner.get_row_ref_unchecked(blobs, self.squashed_offset, ptr) }?; let _remove_result = pointer_map.remove(row.row_hash(), ptr); debug_assert!(_remove_result); @@ -1402,7 +1423,7 @@ impl Table { // Delete the physical row. // SAFETY: `ptr` points to a valid row in this table as `self.is_row_present(row)` holds. - unsafe { self.delete_internal_skip_pointer_map(blob_store, ptr) } + unsafe { self.delete_internal_skip_pointer_map(blobs, ptr) }.map_err(Into::into) } /// Deletes the row identified by `ptr` from the table. @@ -1410,15 +1431,15 @@ impl Table { /// This method does update statistics. /// /// SAFETY: `self.is_row_present(row)` must hold. - unsafe fn delete_unchecked(&mut self, blob_store: &mut dyn BlobStore, ptr: RowPointer) -> Result<(), PageError> { + unsafe fn delete_unchecked(&mut self, blobs: &BlobSet, ptr: RowPointer) -> Result<(), TableError> { // Delete row from indices. // Do this before the actual deletion, as `index.delete` needs a `RowRef` // so it can extract the appropriate value. // SAFETY: Caller promised that `self.is_row_present(row)` holds. - unsafe { self.delete_from_indices(blob_store, ptr)? }; + unsafe { self.delete_from_indices(blobs, ptr)? }; // SAFETY: Caller promised that `self.is_row_present(row)` holds. - let blob_bytes_deleted = unsafe { self.delete_internal(blob_store, ptr)? }; + let blob_bytes_deleted = unsafe { self.delete_internal(blobs, ptr)? }; self.update_statistics_deleted_row(blob_bytes_deleted); @@ -1431,12 +1452,12 @@ impl Table { /// SAFETY: `self.is_row_present(row)` must hold. unsafe fn delete_from_indices_until( &mut self, - blob_store: &dyn BlobStore, + blobs: &BlobSet, ptr: RowPointer, index_id: IndexId, - ) -> Result<(), PageError> { + ) -> Result<(), TableError> { // SAFETY: Caller promised that `self.is_row_present(row)` holds. - let row_ref = unsafe { self.inner.get_row_ref_unchecked(blob_store, self.squashed_offset, ptr) }?; + let row_ref = unsafe { self.inner.get_row_ref_unchecked(blobs, self.squashed_offset, ptr) }?; for (_, index) in self.indexes.range_mut(..index_id) { // SAFETY: any index in this table was constructed with the same row type as this table. @@ -1449,9 +1470,9 @@ impl Table { /// Delete `row_ref` from all the indices of this table. /// /// SAFETY: `self.is_row_present(row)` must hold. - unsafe fn delete_from_indices(&mut self, blob_store: &dyn BlobStore, ptr: RowPointer) -> Result<(), PageError> { + unsafe fn delete_from_indices(&mut self, blobs: &BlobSet, ptr: RowPointer) -> Result<(), TableError> { // SAFETY: Caller promised that `self.is_row_present(row)` holds. - let row_ref = unsafe { self.inner.get_row_ref_unchecked(blob_store, self.squashed_offset, ptr) }?; + let row_ref = unsafe { self.inner.get_row_ref_unchecked(blobs, self.squashed_offset, ptr) }?; for index in self.indexes.values_mut() { // SAFETY: any index in this table was constructed with the same row type as this table. @@ -1471,21 +1492,21 @@ impl Table { /// so that the resulting `ProductValue`s can be passed to the subscription evaluator. pub fn delete<'a, R>( &'a mut self, - blob_store: &'a mut dyn BlobStore, + blobs: &'a BlobSet, ptr: RowPointer, before: impl for<'b> FnOnce(RowRef<'b>) -> R, - ) -> Result, PageError> { + ) -> Result, TableError> { if !self.is_row_present(ptr)? { return Ok(None); }; // SAFETY: We only call `get_row_ref_unchecked` when `is_row_present` holds. - let row_ref = unsafe { self.get_row_ref_unchecked(blob_store, ptr) }?; + let row_ref = unsafe { self.get_row_ref_unchecked(blobs, ptr) }?; let ret = before(row_ref); // SAFETY: We've checked above that `self.is_row_present(ptr)`. - unsafe { self.delete_unchecked(blob_store, ptr) }?; + unsafe { self.delete_unchecked(blobs, ptr) }?; Ok(Some(ret)) } @@ -1501,15 +1522,11 @@ impl Table { /// checking `find_same_row` on the newly-inserted row, /// deleting the matching row if it exists, /// then deleting the temporary insertion. - pub fn delete_equal_row( - &mut self, - blob_store: &mut dyn BlobStore, - row: &ProductValue, - ) -> Result, Error> { + pub fn delete_equal_row(&mut self, blobs: &BlobSet, row: &ProductValue) -> Result, Error> { // Insert `row` temporarily so `temp_ptr` and `hash` can be used to find the row. // This must avoid consulting and inserting to the pointer map, // as the row is already present, set-semantically. - let (temp_row, _) = self.insert_physically_pv(blob_store, row, None)?; + let (temp_row, _) = self.insert_physically_pv(blobs, row, None)?; let temp_ptr = temp_row.into_pointer(); // Find the row equal to the passed-in `row`. @@ -1520,31 +1537,32 @@ impl Table { // SAFETY: // - `self` trivially has the same `row_layout` as `self`. // - We just inserted `temp_ptr`, so it's valid. - let (_, existing_row_ptr) = unsafe { Self::find_same_row(self, self, blob_store, temp_ptr, None) }?; + let (_, existing_row_ptr) = unsafe { Self::find_same_row(self, self, blobs, temp_ptr, None) }?; // If an equal row was present, delete it. if let Some(existing_row_ptr) = existing_row_ptr { // SAFETY: `find_same_row` ensures that the pointer is valid. - unsafe { self.delete_unchecked(blob_store, existing_row_ptr)? }; + unsafe { self.delete_unchecked(blobs, existing_row_ptr)? }; } // Remove the temporary row we inserted in the beginning. // Avoid the pointer map, since we don't want to delete it twice. // SAFETY: `ptr` is valid as we just inserted it. unsafe { - self.delete_internal_skip_pointer_map(blob_store, temp_ptr)?; + self.delete_internal_skip_pointer_map(blobs, temp_ptr) + .map_err(TableError::from)?; } Ok(existing_row_ptr) } /// Clears this table, removing all present rows from it. - pub fn clear(&mut self, blob_store: &mut dyn BlobStore) -> Result { + pub fn clear(&mut self, blobs: &BlobSet) -> Result { let ptrs = self.scan_all_row_ptrs()?; let len = ptrs.len() as u64; for ptr in ptrs { // SAFETY: `ptr` came rom `self.scan_rows(...)`, so it's present. - unsafe { self.delete_unchecked(blob_store, ptr) }?; + unsafe { self.delete_unchecked(blobs, ptr) }?; } Ok(len) } @@ -1607,18 +1625,18 @@ impl Table { /// Caller must promise that `index` was constructed with the same row type/layout as this table. pub unsafe fn insert_index( &mut self, - blob_store: &dyn BlobStore, + blobs: &BlobSet, index_id: IndexId, mut index: TableIndex, - ) -> Result, PageError> { - let rows = self.scan_rows(blob_store); + ) -> Result, TableError> { + let rows = self.scan_rows(blobs); // SAFETY: Caller promised that table's row type/layout // matches that which `index` was constructed with. // It follows that this applies to any `rows`, as required. let violation = unsafe { index.build_from_rows(rows) }?; if let Err(ptr) = violation { // SAFETY: `ptr` just came out of `self.scan_rows`, so it is present. - let row = unsafe { self.get_row_ref_unchecked(blob_store, ptr) }?.to_product_value(); + let row = unsafe { self.get_row_ref_unchecked(blobs, ptr) }?.to_product_value(); let errmsg = if let Some(index_schema) = self .schema @@ -1690,17 +1708,17 @@ impl Table { /// Returns whether an index existed with `index_id`. pub fn delete_index( &mut self, - blob_store: &dyn BlobStore, + blobs: &BlobSet, index_id: IndexId, pointer_map: Option, - ) -> Result, PageError> { + ) -> Result, TableError> { let Some(index) = self.indexes.remove(&index_id) else { return Ok(None); }; // If we removed the last unique index, add a pointer map. if index.is_unique() && !self.indexes.values().any(|idx| idx.is_unique()) { - self.pointer_map = Some(pointer_map.map_or_else(|| self.rebuild_pointer_map(blob_store), Ok)?); + self.pointer_map = Some(pointer_map.map_or_else(|| self.rebuild_pointer_map(blobs), Ok)?); } Ok(Some(index)) @@ -1745,19 +1763,19 @@ impl Table { } /// Returns an iterator over all the rows of `self`, yielded as [`RowRef`]s. - pub fn scan_rows<'a>(&'a self, blob_store: &'a dyn BlobStore) -> TableScanIter<'a> { + pub fn scan_rows<'a>(&'a self, blobs: &'a BlobSet) -> TableScanIter<'a> { TableScanIter { current_page: None, // Will be filled by the iterator. current_page_idx: PageIndex(0), table: self, - blob_store, + blobs, } } /// Returns a list of all present row pointers. - pub fn scan_all_row_ptrs(&self) -> Result, PageError> { + pub fn scan_all_row_ptrs(&self) -> Result, TableError> { let mut ptrs = Vec::with_capacity(self.row_count as usize); - for row in self.scan_rows(&NullBlobStore) { + for row in self.scan_rows(&BlobSet::new_for_test()) { let row = row?; ptrs.push(row.pointer); } @@ -1767,12 +1785,12 @@ impl Table { /// Returns this table combined with the index for [`IndexId`], if any. pub fn get_index_by_id_with_table<'a>( &'a self, - blob_store: &'a dyn BlobStore, + blobs: &'a BlobSet, index_id: IndexId, ) -> Option> { Some(TableAndIndex { table: self, - blob_store, + blobs, index: self.get_index_by_id(index_id)?, }) } @@ -1783,15 +1801,11 @@ impl Table { } /// Returns this table combined with the first index with `cols`, if any. - pub fn get_index_by_cols_with_table<'a>( - &'a self, - blob_store: &'a dyn BlobStore, - cols: &ColList, - ) -> Option> { + pub fn get_index_by_cols_with_table<'a>(&'a self, blobs: &'a BlobSet, cols: &ColList) -> Option> { let (_, index) = self.get_index_by_cols(cols)?; Some(TableAndIndex { table: self, - blob_store, + blobs, index, }) } @@ -1816,12 +1830,12 @@ impl Table { /// Makes the index at `index_id` non-unique. /// /// If no unique indices remain after this, rebuilds and restores the pointer map. - pub fn make_index_non_unique(&mut self, index_id: IndexId, blob_store: &dyn BlobStore) -> Result<(), PageError> { + pub fn make_index_non_unique(&mut self, index_id: IndexId, blobs: &BlobSet) -> Result<(), TableError> { if let Some(idx) = self.indexes.get_mut(&index_id) { idx.make_non_unique(); } if !self.has_unique_index() && self.pointer_map.is_none() { - let pm = self.rebuild_pointer_map(blob_store)?; + let pm = self.rebuild_pointer_map(blobs)?; self.restore_pointer_map(pm); } @@ -1888,17 +1902,13 @@ impl Table { /// # Safety /// /// The schema of rows stored in the `pages` must exactly match `self.schema` and `self.inner.row_layout`. - pub unsafe fn set_pages( - &mut self, - pages: Vec>>, - blob_store: &dyn BlobStore, - ) -> Result<(), PageError> { + pub unsafe fn set_pages(&mut self, pages: Vec>>, blobs: &BlobSet) -> Result<(), TableError> { self.inner.pages.set_contents(pages, self.inner.row_layout.size()); // Recompute table metadata based on the new pages. // Compute the row count first, in case later computations want to use it as a capacity to pre-allocate. - self.compute_row_count(blob_store)?; - self.pointer_map = Some(self.rebuild_pointer_map(blob_store)?); + self.compute_row_count(blobs)?; + self.pointer_map = Some(self.rebuild_pointer_map(blobs)?); Ok(()) } @@ -1928,7 +1938,7 @@ impl Table { /// - Per-page overhead (e.g. page headers). /// - Table overhead (e.g. the [`RowTypeLayout`], [`PointerMap`], [`Schema`] &c). /// - Indexes. - /// - Large blobs in the [`BlobStore`]. + /// - Large blobs in the [`BlobSet`]. /// /// Of these, the caller should inspect the blob store in order to account for memory usage by large blobs, /// and call [`Self::bytes_used_by_index_keys`] to account for indexes, @@ -1985,23 +1995,35 @@ impl Table { pub fn prepare_commit( &self, + blobs: &BlobSet, deletes: impl IntoIterator, - inserts: impl IntoIterator>, - ) -> Result { - self.inner.pages.prepare_commit( + inserts: impl IntoIterator>, + ) -> Result { + let deletes: Vec<_> = deletes.into_iter().collect(); + let mut pinned_blobs = Vec::new(); + for &ptr in &deletes { + // Constructing the row reference faults all of its blobs. Transfer + // clones of those handles to the prepared commit so apply cannot + // encounter a blob fault. + let row = unsafe { self.get_row_ref_unchecked(blobs, ptr) }?; + pinned_blobs.extend(row.blobs.iter().map(|(_, handle)| handle.clone())); + } + + let mut prepared = self.inner.pages.prepare_commit( self.row_size(), &self.inner.visitor_prog, deletes, - inserts.into_iter().map(|res| { - res.map(|insert| { - let num_granules = match &insert.inner { - PreparedInsertInner::Product(row) => self.required_var_len_granules_for_insert(row), - PreparedInsertInner::CopyFixed { .. } => 0, - }; - (insert, num_granules) - }) + inserts.into_iter().map_ok(|insert| { + let num_granules = match &insert.inner { + PreparedInsertInner::Product(row) => self.required_var_len_granules_for_insert(row), + PreparedInsertInner::CopyFixed { .. } => 0, + }; + (insert, num_granules) }), - ) + )?; + prepared.pin_blobs(pinned_blobs); + + Ok(prepared) } pub fn prepare_insert(&self, row: RowRef<'_>) -> PreparedInsert { @@ -2038,12 +2060,12 @@ impl Table { pub struct RowRef<'a> { /// The table that has the row at `self.pointer`. table: &'a TableInner, - /// The blob store used in case there are blob hashes to resolve. - blob_store: &'a dyn BlobStore, /// The pointer to the row in `self.table`. pointer: RowPointer, /// Guard that ensures the page stays resident while this `RowRef` is live. page: PageHandle, + /// Blobs referenced by this row. + blobs: Arc, } impl fmt::Debug for RowRef<'_> { @@ -2074,21 +2096,31 @@ impl<'a> RowRef<'a> { /// is sufficient to demonstrate all of these properties. unsafe fn new( table: &'a TableInner, - blob_store: &'a dyn BlobStore, + blobs: &BlobSet, _squashed_offset: SquashedOffset, pointer: RowPointer, page: PageHandle, - ) -> Self { + ) -> Result { debug_assert_eq!(_squashed_offset, pointer.squashed_offset()); debug_assert!(page .read() .has_row_offset(table.row_layout.size(), pointer.page_offset())); - Self { + + let blobs = BlobReadSet::new( + &table.visitor_prog, + &page.read(), + blobs, + &table.row_layout, + pointer.page_offset(), + )? + .into(); + + Ok(Self { table, - blob_store, pointer, page, - } + blobs, + }) } /// Extract a `ProductValue` from the table. @@ -2126,7 +2158,7 @@ impl<'a> RowRef<'a> { // - We have a `RowRef`, so `ptr` points to a valid row in this table // so safety requirements 1-3 flow from that. // - Caller promised that any `col` in `cols` is in-bounds of `self`'s layout. - unsafe { serialize_columns_from_page(ser, &page.read(), self.blob_store, offset, &table.row_layout, cols) } + unsafe { serialize_columns_from_page(ser, &page.read(), &self.blobs, offset, &table.row_layout, cols) } } /// Construct a projection of the row at `self` by extracting the `cols`. @@ -2213,9 +2245,8 @@ impl<'a> RowRef<'a> { self.pointer } - /// Returns the blob store that any [`crate::blob_store::BlobHash`]es within the row refer to. - pub(crate) fn blob_store(&self) -> &dyn BlobStore { - self.blob_store + pub(crate) fn blob_store(&self) -> &BlobReadSet { + &self.blobs } /// Return the layout of the row. @@ -2264,24 +2295,7 @@ impl<'a> RowRef<'a> { /// as a row may contain multiple references to the same large blob. /// This seems unlikely to occur in practice. fn blob_store_bytes(&self) -> usize { - let (page, offset) = self.page_and_offset(); - let page = page.read(); - let row_data = page.get_row_data(offset, self.table.row_layout.size()); - // SAFETY: - // - Existence of a `RowRef` treated as proof - // of the row's validity and type information's correctness. - unsafe { self.table.visitor_prog.visit_var_len(row_data) } - .filter(|vlr| vlr.is_large_blob()) - .map(|vlr| { - // SAFETY: - // - Because `vlr.is_large_blob`, it points to exactly one granule. - let granule = unsafe { page.iter_var_len_object(vlr.first_granule) }.next().unwrap(); - let blob_hash = granule.blob_hash(); - let blob = self.blob_store.retrieve_blob(&blob_hash).unwrap(); - - blob.len() - }) - .sum() + self.blobs.iter().map(|(_, blob)| blob.size() as usize).sum() } pub fn into_page(self) -> PageHandle { @@ -2301,7 +2315,7 @@ impl Serialize for RowRef<'_> { .map_err(::custom)?; // SAFETY: We have a `RowRef`, so `ptr` points to a valid row in this table // so safety requirements 1-3 flow from that. - unsafe { serialize_row_from_page(ser, &page.read(), self.blob_store, offset, &table.row_layout) } + unsafe { serialize_row_from_page(ser, &page.read(), &self.blobs, offset, &table.row_layout) } } } @@ -2380,7 +2394,7 @@ impl PartialEq for RowRef<'_> { let (page, offset) = self.page_and_offset(); // SAFETY: By having `RowRef`, // we know that `offset` is a valid offset for a row in `page` typed at `ty`. - unsafe { eq_row_in_page_to_pv(self.blob_store, &page.read(), offset, rhs, ty) } + unsafe { eq_row_in_page_to_pv(&self.blobs, &page.read(), offset, rhs, ty) } } } @@ -2393,7 +2407,7 @@ impl Hash for RowRef<'_> { // 2. the row is valid for `ty`. // 3. for any `vlr: VarLenRef` stored in the row, // `vlr.first_offset` is either `NULL` or points to a valid granule in `page`. - unsafe { hash_row_in_page(state, &page.read(), self.blob_store, offset, ty) }; + unsafe { hash_row_in_page(state, &page.read(), &self.blobs, offset, ty) }; } } @@ -2407,8 +2421,8 @@ pub struct TableScanIter<'table> { /// The table the iterator is yielding rows from. pub(crate) table: &'table Table, - /// The `BlobStore` that row references may refer into. - pub(crate) blob_store: &'table dyn BlobStore, + /// The [`BlobSet`] that row references may refer into. + pub(crate) blobs: &'table BlobSet, } struct CurrentPage { @@ -2428,7 +2442,7 @@ impl Iterator for CurrentPage { } impl<'a> Iterator for TableScanIter<'a> { - type Item = Result, PageError>; + type Item = Result, TableError>; fn next(&mut self) -> Option { // This could have been written using `.flat_map`, @@ -2447,13 +2461,13 @@ impl<'a> Iterator for TableScanIter<'a> { let row_ref = unsafe { RowRef::new( &self.table.inner, - self.blob_store, + self.blobs, self.table.squashed_offset, ptr, page_handle, ) }; - return Some(Ok(row_ref)); + return Some(row_ref.map_err(Into::into)); } else { // We've finished visiting that page, so set `current_page` to `None`, // increment `self.current_page_idx` to the index of the next page, @@ -2501,7 +2515,7 @@ impl<'a> Iterator for TableScanIter<'a> { // so continue past it to the next page slot. self.current_page_idx.0 += 1; } - Err(e) => return Some(Err(e)), + Err(e) => return Some(Err(e.into())), } } } @@ -2515,7 +2529,7 @@ impl<'a> Iterator for TableScanIter<'a> { #[derive(Copy, Clone)] pub struct TableAndIndex<'a> { table: &'a Table, - blob_store: &'a dyn BlobStore, + blobs: &'a BlobSet, index: &'a TableIndex, } @@ -2533,9 +2547,9 @@ impl<'a> TableAndIndex<'a> { /// # Safety /// /// The `self.table().is_row_present(ptr)` must hold. - pub unsafe fn combine_with_ptr(&self, ptr: RowPointer) -> Result, PageError> { + pub unsafe fn combine_with_ptr(&self, ptr: RowPointer) -> Result, TableError> { // SAFETY: forward caller requirement. - unsafe { self.table.get_row_ref_unchecked(self.blob_store, ptr) } + unsafe { self.table.get_row_ref_unchecked(self.blobs, ptr) } } /// Returns an iterator yielding all rows in this index for `key`. @@ -2544,7 +2558,7 @@ impl<'a> TableAndIndex<'a> { pub fn seek_point(&self, key: &IndexKey<'_>) -> IndexScanPointIter<'a> { IndexScanPointIter { table: self.table, - blob_store: self.blob_store, + blobs: self.blobs, btree_index_iter: self.index.seek_point(key), } } @@ -2559,7 +2573,7 @@ impl<'a> TableAndIndex<'a> { ) -> Result, IndexCannotSeekRange> { Ok(IndexScanRangeIter { table: self.table, - blob_store: self.blob_store, + blobs: self.blobs, btree_index_iter: self.index.seek_range(range)?, }) } @@ -2586,7 +2600,7 @@ impl<'a> TableAndIndex<'a> { Ok(IndexScanRangeIter { table: self.table, - blob_store: self.blob_store, + blobs: self.blobs, btree_index_iter, }) } @@ -2600,7 +2614,7 @@ pub struct IndexScanPointIter<'a> { /// The table being scanned for rows. table: &'a Table, /// The blob store; passed on to the [`RowRef`]s in case they need it. - blob_store: &'a dyn BlobStore, + blobs: &'a BlobSet, /// The iterator performing the index scan yielding row pointers. btree_index_iter: TableIndexPointIter<'a>, } @@ -2613,12 +2627,12 @@ impl<'a> IndexScanPointIter<'a> { } impl<'a> Iterator for IndexScanPointIter<'a> { - type Item = Result, PageError>; + type Item = Result, TableError>; fn next(&mut self) -> Option { self.btree_index_iter.next().map(|ptr| { // SAFETY: `ptr` came from the index, which always holds pointers to valid rows for its table. - unsafe { self.table.get_row_ref_unchecked(self.blob_store, ptr) } + unsafe { self.table.get_row_ref_unchecked(self.blobs, ptr) } }) } } @@ -2631,18 +2645,18 @@ pub struct IndexScanRangeIter<'a> { /// The table being scanned for rows. table: &'a Table, /// The blob store; passed on to the [`RowRef`]s in case they need it. - blob_store: &'a dyn BlobStore, + blobs: &'a BlobSet, /// The iterator performing the index scan yielding row pointers. btree_index_iter: TableIndexRangeIter<'a>, } impl<'a> Iterator for IndexScanRangeIter<'a> { - type Item = Result, PageError>; + type Item = Result, TableError>; fn next(&mut self) -> Option { self.btree_index_iter.next().map(|ptr| { // SAFETY: `ptr` came from the index, which always holds pointers to valid rows for its table. - unsafe { self.table.get_row_ref_unchecked(self.blob_store, ptr) } + unsafe { self.table.get_row_ref_unchecked(self.blobs, ptr) } }) } } @@ -2810,10 +2824,10 @@ impl Table { /// Called when restoring from a snapshot after installing the pages, /// but after computing the row count, /// since snapshots do not save the pointer map.. - fn rebuild_pointer_map(&mut self, blob_store: &dyn BlobStore) -> Result { + fn rebuild_pointer_map(&mut self, blobs: &BlobSet) -> Result { // TODO(perf): Pre-allocate `PointerMap.map` with capacity `self.row_count`. // Alternatively, do this at the same time as `compute_row_count`. - self.scan_rows(blob_store) + self.scan_rows(blobs) .map(|row_ref| row_ref.map(|row_ref| (row_ref.row_hash(), row_ref.pointer()))) .collect() } @@ -2823,10 +2837,10 @@ impl Table { /// /// Called when restoring from a snapshot after installing the pages, /// since snapshots do not save this metadata. - fn compute_row_count(&mut self, blob_store: &dyn BlobStore) -> Result<(), PageError> { + fn compute_row_count(&mut self, blobs: &BlobSet) -> Result<(), TableError> { let mut row_count = 0; let mut blob_store_bytes = 0; - for row in self.scan_rows(blob_store) { + for row in self.scan_rows(blobs) { let row = row?; row_count += 1; blob_store_bytes += row.blob_store_bytes(); @@ -2874,7 +2888,6 @@ enum PreparedInsertInner { #[cfg(test)] pub(crate) mod test { use super::*; - use crate::blob_store::{HashMapBlobStore, NullBlobStore}; use crate::page::tests::hash_unmodified_save_get; use crate::table_index::KeySize; use crate::var_len::VarLenGranule; @@ -2931,7 +2944,7 @@ pub(crate) mod test { PageManager::new_for_test().into(), PageEvictionPolicy::NeverEvict, ); - let blob_store = &mut NullBlobStore; + let blob_store = &mut BlobSet::new_for_test(); let cols = ColList::new(0.into()); let algo = BTreeAlgorithm { columns: cols.clone() }.into(); @@ -2997,9 +3010,9 @@ pub(crate) mod test { fn insert_retrieve_body(ty: impl Into, val: impl Into) -> TestCaseResult { let val = val.into(); - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty.into()); - let (hash, row) = table.insert(&mut blob_store, &val).unwrap(); + let (hash, row) = table.insert(&blob_store, &val).unwrap(); let hash = hash.unwrap(); prop_assert_eq!(row.row_hash(), hash); let ptr = row.pointer(); @@ -3050,14 +3063,14 @@ pub(crate) mod test { insert_retrieve_body(ty, AlgebraicValue::from(arr)).unwrap(); } - fn reconstruct_index_num_key_bytes(table: &Table, blob_store: &dyn BlobStore, index_id: IndexId) -> u64 { + fn reconstruct_index_num_key_bytes(table: &Table, blobs: &BlobSet, index_id: IndexId) -> u64 { let index = table.get_index_by_id(index_id).unwrap(); index .iter() .map(|row_ptr| { let row_ref = table - .get_row_ref(blob_store, row_ptr) + .get_row_ref(blobs, row_ptr) .expect("no page fault") .expect("row to be present"); index.project_row(&row_ref).key_size_in_bytes() as u64 @@ -3076,11 +3089,11 @@ pub(crate) mod test { index_kind: IndexKind, is_unique: bool, ) -> Result<(), TestCaseError> { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty.clone()); for row in &vals { - prop_assume!(table.insert(&mut blob_store, row).is_ok()); + prop_assume!(table.insert(&blob_store, row).is_ok()); } // We haven't added any indexes yet, so there should be 0 rows in indexes. @@ -3145,9 +3158,9 @@ pub(crate) mod test { #[test] fn insert_delete_removed_from_pointer_map((ty, val) in generate_typed_row()) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty); - let (hash, row) = table.insert(&mut blob_store, &val).unwrap(); + let (hash, row) = table.insert(&blob_store, &val).unwrap(); let hash = hash.unwrap(); prop_assert_eq!(row.row_hash(), hash); let ptr = row.into_pointer(); @@ -3160,7 +3173,7 @@ pub(crate) mod test { let hash_pre_del = table.inner.pages.with_page_mut(ptr.page_index(), table.row_size(), hash_unmodified_save_get).unwrap(); - table.delete(&mut blob_store, ptr, |_| ()).unwrap(); + table.delete(&blob_store, ptr, |_| ()).unwrap(); // Deleting the only row frees the tiered page slot. prop_assert!(table.inner.pages.get_page(ptr.page_index()).unwrap().is_none()); @@ -3177,10 +3190,10 @@ pub(crate) mod test { #[test] fn insert_duplicate_set_semantic((ty, val) in generate_typed_row()) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty); - let (hash, row) = table.insert(&mut blob_store, &val).unwrap(); + let (hash, row) = table.insert(&blob_store, &val).unwrap(); let hash = hash.unwrap(); prop_assert_eq!(row.row_hash(), hash); let ptr = row.into_pointer(); @@ -3193,7 +3206,7 @@ pub(crate) mod test { let hash_pre_ins = table.inner.pages.with_page_mut(ptr.page_index(), table.row_size(), hash_unmodified_save_get).unwrap(); - prop_assert!(table.insert(&mut blob_store, &val).is_err()); + prop_assert!(table.insert(&blob_store, &val).is_err()); // Hash was cleared and is different despite failure to insert. let hash_post_ins = table.inner.pages.with_page_mut(ptr.page_index(), table.row_size(), hash_unmodified_save_get).unwrap(); @@ -3213,26 +3226,27 @@ pub(crate) mod test { #[test] fn insert_bsatn_same_as_pv((ty, val) in generate_typed_row()) { let page_manager = Arc::new(PageManager::new_for_test()); - let mut bs_pv = HashMapBlobStore::default(); + let bs_pv = BlobSet::new_for_test(); let mut table_pv = table_with_manager(page_manager.clone(), ty.clone()); - let _res_pv = table_pv.insert(&mut bs_pv, &val); + let _res_pv = table_pv.insert(&bs_pv, &val); - let mut bs_bsatn = HashMapBlobStore::default(); + let mut bs_bsatn = BlobSet::new_for_test(); let mut table_bsatn = table_with_manager(page_manager.clone(), ty); let _res_bsatn = insert_bsatn(&mut table_bsatn, &mut bs_bsatn, &val); //prop_assert_eq!(res_pv, res_bsatn); - prop_assert_eq!(bs_pv, bs_bsatn); + //prop_assert_eq!(bs_pv.entries(), bs_bsatn.entries()); + prop_assert_eq!(bs_pv.keys(), bs_bsatn.keys()); //prop_assert_eq!(table_pv, table_bsatn); } #[test] fn row_size_reporting_matches_slow_implementations((ty, vals) in generate_typed_row_vec(0..SIZE, 128, 2048)) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty.clone()); for row in &vals { - prop_assume!(table.insert(&mut blob_store, row).is_ok()); + prop_assume!(table.insert(&blob_store, row).is_ok()); } prop_assert_eq!(table.bytes_used_by_rows(), table.reconstruct_bytes_used_by_rows()); @@ -3250,7 +3264,7 @@ pub(crate) mod test { /// Tested here rather than in pages.rs because it's easier to test with typed rows than raw byte buffers. #[test] fn non_full_pages_consistent((ty, vals) in generate_typed_row_vec(0..SIZE, 128, 2048)) { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ty); let mut inserted_row_ptrs = Vec::new(); @@ -3260,7 +3274,7 @@ pub(crate) mod test { // This keeps the page usage growing towards fullness, but also includes some deletes. for rows in vals.chunks(3) { for row in rows { - let row_ptr = match table.insert(&mut blob_store, row) { + let row_ptr = match table.insert(&blob_store, row) { Ok((_, row_ref)) => row_ref.pointer(), Err(InsertError::Duplicate(_)) => continue, Err(e) => return Err(TestCaseError::fail(format!("unexpected insert error: {e:?}"))), @@ -3270,7 +3284,7 @@ pub(crate) mod test { } if let Some(row_ptr) = inserted_row_ptrs.pop() { - table.delete(&mut blob_store, row_ptr, |_| ()).unwrap(); + table.delete(&blob_store, row_ptr, |_| ()).unwrap(); table.inner.pages.assert_non_full_pages_consistent(table.row_size()); } } @@ -3297,31 +3311,31 @@ pub(crate) mod test { fn insert_bsatn<'a>( table: &'a mut Table, - blob_store: &'a mut dyn BlobStore, + blobs: &'a mut BlobSet, val: &ProductValue, ) -> Result<(Option, RowRef<'a>), InsertError> { let row = &to_vec(&val).unwrap(); // Optimistically insert the `row` before checking any constraints // under the assumption that errors (unique constraint & set semantic violations) are rare. - let (row_ref, blob_bytes) = table.insert_physically_bsatn(blob_store, row)?; + let (row_ref, blob_bytes) = table.insert_physically_bsatn(blobs, row)?; let row_ptr = row_ref.into_pointer(); // Confirm the insertion, checking any constraints, removing the physical row on error. // SAFETY: We just inserted `ptr`, so it must be present. - let (hash, row_ptr) = unsafe { table.confirm_insertion::(blob_store, row_ptr, blob_bytes) }?; + let (hash, row_ptr) = unsafe { table.confirm_insertion::(blobs, row_ptr, blob_bytes) }?; // SAFETY: Per post-condition of `confirm_insertion`, `row_ptr` refers to a valid row. - let row_ref = unsafe { table.get_row_ref_unchecked(blob_store, row_ptr) }?; + let row_ref = unsafe { table.get_row_ref_unchecked(blobs, row_ptr) }?; Ok((hash, row_ref)) } // Compare `scan_rows` against a simpler implementation. #[test] fn table_scan_iter_eq_flatmap() { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(AlgebraicType::U64.into()); for v in 0..2u64.pow(14) { - table.insert(&mut blob_store, &product![v]).unwrap(); + table.insert(&blob_store, &product![v]).unwrap(); } let complex = table.scan_rows(&blob_store).map(|r| r.unwrap().pointer()); @@ -3353,13 +3367,13 @@ pub(crate) mod test { /// assuming no other reason to prefer a different page. #[test] fn prefer_earlier_non_full_page() { - let mut blob_store = HashMapBlobStore::default(); + let blob_store = BlobSet::new_for_test(); let mut table = table(ProductType::from([AlgebraicType::I32])); let mut inserted_ptrs = Vec::new(); let mut next_value = 0i32; while table.num_pages() < 2 { - let (_, row_ref) = table.insert(&mut blob_store, &product![next_value]).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &product![next_value]).unwrap(); inserted_ptrs.push(row_ref.pointer()); next_value += 1; } @@ -3387,7 +3401,7 @@ pub(crate) mod test { } let first_ptr = inserted_ptrs[0]; - table.delete(&mut blob_store, first_ptr, |_| ()).unwrap(); + table.delete(&blob_store, first_ptr, |_| ()).unwrap(); { let first_page = table @@ -3401,7 +3415,7 @@ pub(crate) mod test { assert_eq!(first_page.available_var_len_granules(), 0); } - let (_, row_ref) = table.insert(&mut blob_store, &product![next_value]).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &product![next_value]).unwrap(); let new_ptr = row_ref.pointer(); assert_eq!(new_ptr.page_index(), first_ptr.page_index()); assert_eq!(new_ptr.page_offset(), first_ptr.page_offset()); @@ -3424,7 +3438,7 @@ pub(crate) mod test { let pt = AlgebraicType::U64.into(); let pv = product![42u64]; let mut table = table(pt); - let blob_store = &mut NullBlobStore; + let blob_store = &mut BlobSet::new_for_test(); let (_, row_ref) = table.insert(blob_store, &pv).unwrap(); // Manipulate the page offset to 1 instead of 0. @@ -3434,7 +3448,7 @@ pub(crate) mod test { // We expect this to panic. // Miri should not have any issue with this call either. table - .get_row_ref(&NullBlobStore, ptr) + .get_row_ref(&BlobSet::new_for_test(), ptr) .expect("no page faults") .unwrap() .to_product_value(); @@ -3444,8 +3458,8 @@ pub(crate) mod test { fn test_blob_store_bytes() { let page_manager = Arc::new(PageManager::new_for_test()); let pt: ProductType = [AlgebraicType::String, AlgebraicType::I32].into(); - let blob_store = &mut HashMapBlobStore::default(); - let mut insert = + let blob_store = &mut BlobSet::new_for_test(); + let insert = |table: &mut Table, string, num| table.insert(blob_store, &product![string, num]).unwrap().1.pointer(); let mut table1 = table_with_manager(page_manager.clone(), pt.clone()); @@ -3462,14 +3476,14 @@ pub(crate) mod test { // Insert previous long string in the same table, // `blob_store_bytes` should count the length twice, - // even though `HashMapBlobStore` deduplicates it. + // even though `BlobSet` deduplicates it. let long_row_ptr2 = insert(&mut table1, long_str, 1); const BLOB_OBJ_LEN_2X: BlobNumBytes = BlobNumBytes(BLOB_OBJ_LEN.0 * 2); assert_eq!(table1.blob_store_bytes, BLOB_OBJ_LEN_2X); // Insert previous long string in a new table, // `blob_store_bytes` should show the length, - // even though `HashMapBlobStore` deduplicates it. + // even though `BlobSet` deduplicates it. let mut table2 = table_with_manager(page_manager, pt); let _ = insert(&mut table2, long_str, 0); assert_eq!(table2.blob_store_bytes, BLOB_OBJ_LEN); @@ -3484,14 +3498,14 @@ pub(crate) mod test { // Delete the first long string row. This gets us down to 0 (we've now deleted 2x). table1.delete(blob_store, long_row_ptr2, |_| ()).unwrap(); - assert_eq!(table1.blob_store_bytes, 0.into()); + assert_eq!(table1.blob_store_bytes, 0usize.into()); } /// Assert that calling `get_row_ref` to get a row ref to a non-existent `RowPointer` /// does not panic. #[test] fn get_row_ref_no_panic() { - let blob_store = &mut HashMapBlobStore::default(); + let blob_store = &mut BlobSet::new_for_test(); let table = table([AlgebraicType::String, AlgebraicType::I32].into()); // This row pointer has an incorrect `SquashedOffset`, and so does not point into `table`. @@ -3515,7 +3529,7 @@ pub(crate) mod test { #[test] fn table_iter_skips_absent_pages() { - let blob_store = &mut NullBlobStore; + let blob_store = &mut BlobSet::new_for_test(); let mut table = table([AlgebraicType::I32].into()); let mut next_value = 0i32; @@ -3557,7 +3571,7 @@ pub(crate) mod test { #[test] fn alloc_new_page_fills_hole() { - let blob_store = &mut NullBlobStore; + let blob_store = &mut BlobSet::new_for_test(); let mut table = table([AlgebraicType::I32].into()); fn has_two_full_pages(table: &Table) -> bool { diff --git a/crates/table/src/table_index/mod.rs b/crates/table/src/table_index/mod.rs index 67a93ed7878..b3217056e90 100644 --- a/crates/table/src/table_index/mod.rs +++ b/crates/table/src/table_index/mod.rs @@ -38,7 +38,7 @@ use self::unique_direct_index::{ToFromUsize, UniqueDirectIndex, UniqueDirectInde use self::unique_hash_index::UniqueHashIndex; use super::indexes::RowPointer; use super::table::RowRef; -use crate::tiered::PageError; +use crate::table::TableError; use crate::{read_column::ReadColumn, static_assert_size}; use core::cmp::Ordering; use core::ops::{Bound, Deref, RangeBounds}; @@ -2590,8 +2590,8 @@ impl TableIndex { /// being the same as passed in on `self`'s construction. pub unsafe fn build_from_rows<'table>( &mut self, - rows: impl IntoIterator, PageError>>, - ) -> Result, PageError> { + rows: impl IntoIterator, TableError>>, + ) -> Result, TableError> { for row_ref in rows { let row_ref = row_ref?; // SAFETY: Forward caller proof obligation. @@ -2772,8 +2772,9 @@ impl TableIndex { #[cfg(test)] mod test { use super::*; + use crate::table::test::table; use crate::table::Table; - use crate::{blob_store::HashMapBlobStore, table::test::table}; + use crate::tiered::BlobSet; use core::cmp::Ordering; use core::ops::Bound::*; use core::slice; @@ -2821,8 +2822,8 @@ mod test { } } - fn setup(ty: ProductType) -> (Table, HashMapBlobStore) { - (table(ty), HashMapBlobStore::default()) + fn setup(ty: ProductType) -> (Table, BlobSet) { + (table(ty), BlobSet::new_for_test()) } fn new_index(row_type: &ProductType, cols: &ColList, is_unique: bool, kind: IndexKind) -> TableIndex { @@ -2952,8 +2953,8 @@ mod test { #[test] fn remove_nonexistent_noop((ty, cols, pv) in gen_row_and_cols(), kind: IndexKind, is_unique: bool) { let mut index = new_index(&ty, &cols, is_unique, kind); - let (mut table, mut blob_store) = setup(ty); - let row_ref = table.insert(&mut blob_store, &pv).unwrap().1; + let (mut table, blob_store) = setup(ty); + let row_ref = table.insert(&blob_store, &pv).unwrap().1; prop_assert_eq!(unsafe { index.delete(&row_ref) }, false); prop_assert!(index.idx.is_empty()); prop_assert_eq!(index.num_keys(), 0); @@ -2964,8 +2965,8 @@ mod test { #[test] fn insert_delete_noop((ty, cols, pv) in gen_row_and_cols(), kind: IndexKind, is_unique: bool) { let mut index = new_index(&ty, &cols, is_unique, kind); - let (mut table, mut blob_store) = setup(ty); - let row_ref = table.insert(&mut blob_store, &pv).unwrap().1; + let (mut table, blob_store) = setup(ty); + let row_ref = table.insert(&blob_store, &pv).unwrap().1; let value = get_fields(&cols, &pv); prop_assert_eq!(index.num_keys(), 0); @@ -2995,7 +2996,7 @@ mod test { let ty = ProductType::from(ty.into_boxed_slice()); let mut index = new_index(&ty, &cols, false, kind); - let (mut table, mut blob_store) = setup(ty); + let (mut table, blob_store) = setup(ty); let num_vals = vals.len(); for val in vals { @@ -3003,7 +3004,7 @@ mod test { key.push(val.into()); let key = ProductValue::from(key); - let row_ref = table.insert(&mut blob_store, &key).unwrap().1; + let row_ref = table.insert(&blob_store, &key).unwrap().1; // SAFETY: `row_ref` has the same type as was passed in when constructing `index`. prop_assert_eq!(unsafe { index.check_and_insert(&row_ref) }, Ok(())); @@ -3016,8 +3017,8 @@ mod test { #[test] fn insert_again_violates_unique_constraint((ty, cols, pv) in gen_row_and_cols(), kind: IndexKind) { let mut index = new_index(&ty, &cols, true, kind); - let (mut table, mut blob_store) = setup(ty); - let row_ref = table.insert(&mut blob_store, &pv).unwrap().1; + let (mut table, blob_store) = setup(ty); + let row_ref = table.insert(&blob_store, &pv).unwrap().1; let value = get_fields(&cols, &pv); // Nothing in the index yet. @@ -3053,7 +3054,7 @@ mod test { let cols = 0.into(); let ty = ProductType::from_iter([AlgebraicType::U64]); let mut index = new_index(&ty, &cols, is_unique, kind); - let (mut table, mut blob_store) = setup(ty); + let (mut table, blob_store) = setup(ty); let prev = needle - 1; let next = needle + 1; @@ -3064,7 +3065,7 @@ mod test { // Insert `prev`, `needle`, and `next`. for x in range.clone() { let row = product![x]; - let row_ref = table.insert(&mut blob_store, &row).unwrap().1; + let row_ref = table.insert(&blob_store, &row).unwrap().1; val_to_ptr.insert(x, row_ref.pointer()); // SAFETY: `row_ref` has the same type as was passed in when constructing `index`. prop_assert_eq!(unsafe { index.check_and_insert(&row_ref) }, Ok(())); @@ -3157,9 +3158,9 @@ mod test { let mut index = new_index(&row_ty, &[0].into(), is_unique, kind); // Construct the table and add `val` as a row. - let (mut table, mut blob_store) = setup(row_ty); + let (mut table, blob_store) = setup(row_ty); let pv = product![val.clone()]; - let row_ref = table.insert(&mut blob_store, &pv).unwrap().1; + let row_ref = table.insert(&blob_store, &pv).unwrap().1; // Add the row to the index. assert_eq!(index.num_keys(), 0); @@ -3199,9 +3200,9 @@ mod test { let row = product![prefix_val.clone(), middle.clone(), suffix_val.clone()]; // Make a table, add the index, and insert the row. - let (mut table, mut blob_store) = setup(ty); + let (mut table, blob_store) = setup(ty); unsafe { table.add_index(IndexId::SENTINEL, index) }; - let (_, row_ref) = table.insert(&mut blob_store, &row).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &row).unwrap(); let row_ptr = row_ref.pointer(); let index = table.get_index_by_id(IndexId::SENTINEL).unwrap(); @@ -3259,9 +3260,9 @@ mod test { let other_row = product![prefix_val, excluded, suffix_val]; // Make a table, add the index, and insert the row. - let (mut table, mut blob_store) = setup(ty); + let (mut table, blob_store) = setup(ty); unsafe { table.add_index(IndexId::SENTINEL, index) }; - let (_, row_ref) = table.insert(&mut blob_store, &row).unwrap(); + let (_, row_ref) = table.insert(&blob_store, &row).unwrap(); let row_ptr = row_ref.pointer(); let index = table.get_index_by_id(IndexId::SENTINEL).unwrap(); diff --git a/crates/table/src/tiered/blob_set.rs b/crates/table/src/tiered/blob_set.rs new file mode 100644 index 00000000000..64c78ca6ffb --- /dev/null +++ b/crates/table/src/tiered/blob_set.rs @@ -0,0 +1,509 @@ +use std::{ + num::NonZeroUsize, + ops::Deref, + sync::{Arc, Mutex}, +}; + +use parking_lot::RwLock; +use spacetimedb_data_structures::small_map::SmallHashMap; +use spacetimedb_memory_usage::MemoryUsage; +use spacetimedb_sats::layout::RowTypeLayout; + +use crate::{ + blob_store::BlobHash, + indexes::PageOffset, + page::Page, + row_type_visitor::VarLenVisitorProgram, + tiered::{ + error::{BlobError, ObjectStoreError}, + map::{self, ObjectKey, ObjectMap}, + mem::{self, BlobEvictionCandidate, BlobFrame, ReservedBlob}, + ByteBudget, ByteBudgetUsage, TieredStorage, + }, + var_len::VarLenMembers, +}; + +/// A blob set contains the content-addressed blobs, shared across all tables in +/// a database. +/// +/// Residency of blobs is controlled by the shared [mem::Cache]'s budget. Blobs +/// that get evicted from memory are handed off to the shared [TieredStorage]. +#[derive(Debug)] +pub struct BlobSet { + resident: Arc, + storage: Arc, + entries: Arc>>, + evictable: Mutex>, +} + +impl BlobSet { + /// Create a new, empty [BlobSet]. + pub fn new(resident: Arc, storage: Arc) -> Self { + Self { + resident, + storage, + entries: <_>::default(), + evictable: <_>::default(), + } + } + + /// Create a new [BlobSet] from a catalog of entries. + /// + /// The blobs in the catalog will be faulted in lazily as they are accessed. + pub fn from_catalog( + resident: Arc, + storage: Arc, + catalog: impl IntoIterator, + ) -> Self { + let entries = catalog + .into_iter() + .map(|CatalogEntry { hash, uses, size }| (ObjectKey::from(hash), BlobEntry::NonResident { uses, size })) + .collect(); + + Self { + resident, + storage, + entries: Arc::new(RwLock::new(entries)), + evictable: <_>::default(), + } + } + + /// Create a new [BlobSet] from externally loaded blobs. + /// + /// This is used when restoring from a snapshot. + /// + /// All blobs in the map will initially be resident, bypassing the memory budget. + pub fn from_blob_map(resident: Arc, storage: Arc, map: BlobMap) -> Self { + let mut entries = ObjectMap::default(); + for (key, BlobMapEntry { uses, blob }) in map.inner { + let frame = resident.register_blob_unchecked(blob); + entries.insert( + key, + BlobEntry::Resident { + uses: uses.get(), + blob: BlobHandle { frame }, + }, + ); + } + + Self { + resident, + storage, + entries: Arc::new(RwLock::new(entries)), + evictable: <_>::default(), + } + } + + pub fn new_for_test() -> Self { + Self::new( + mem::ResidentObjects::new(ByteBudget::unlimited()).into(), + TieredStorage::new_for_test().into(), + ) + } + + /// Attempt to evict resident blobs in this set that have no external + /// references. + pub fn evict(&self) -> Result { + let mut candidates = self.evictable.lock().unwrap(); + self.resident.evictable_blobs(&mut candidates); + for candidate in candidates.drain(..) { + let maybe_resident = self.entries.read().get(&candidate.key).and_then(|entry| { + let BlobEntry::Resident { blob, .. } = entry else { + return None; + }; + Some(blob.clone()) + }); + + if let Some(blob) = maybe_resident { + self.storage.write(&blob)?; + self.entries.write().remove(candidate.key); + self.resident.evict_blob(candidate); + } + } + + Ok(self.resident.usage()) + } + + fn reserve(&self, bytes: &[u8]) -> Result { + let size = bytes.len() as u64; + self.resident.reserve_blob(size, || bytes.into()).or_else(|_| { + self.evict()?; + self.resident + .reserve_blob(size, || bytes.into()) + .map_err(BlobError::from) + }) + } + + /// Insert `bytes` into this set, making it a resident blob. + /// + /// Subject to the memory budget. + pub fn insert(&self, bytes: &[u8]) -> Result { + self.insert_with_uses(NonZeroUsize::MIN, bytes) + } + + fn insert_with_uses(&self, uses: NonZeroUsize, bytes: &[u8]) -> Result { + let hash = BlobHash::hash_from_bytes(bytes); + let blob_or_reservation = { + let mut entries = self.entries.write(); + match entries + .entry(hash.into()) + .and_modify(|entry| entry.mod_uses(|u| *u += uses.get())) + { + map::Entry::Vacant(_) => { + drop(entries); + Err(self.reserve(bytes)?) + } + map::Entry::Occupied(entry) => match entry.get() { + BlobEntry::Resident { blob, .. } => Ok(blob.clone()), + BlobEntry::NonResident { .. } => { + drop(entries); + Err(self.reserve(bytes)?) + } + }, + } + }; + match blob_or_reservation { + Ok(handle) => Ok(handle), + Err(reservation) => { + let frame = self.resident.redeem_blob(reservation); + let handle = BlobHandle { frame }; + self.entries.write().insert( + hash.into(), + BlobEntry::Resident { + uses: uses.get(), + blob: handle.clone(), + }, + ); + Ok(handle) + } + } + } + + /// Increment the reference count of the blob with content address `hash`. + /// + /// This does not fault the blob if it is non-resident. + /// + /// Returns [BlobError::MissingBlob] if no such blob is in this blob set's + /// entry catalog. + pub fn clone_ref(&mut self, hash: BlobHash) -> Result<(), BlobError> { + self.entries + .write() + .entry(hash.into()) + .and_modify(BlobEntry::inc_uses) + .or_error(|| BlobError::MissingBlob(hash)) + .map(drop) + } + + /// Decrement the reference count of the blob with content address `hash`. + /// + /// If the refcount is zero of decrementing, the entry is removed from the + /// entry catalog. + /// + /// Returns [BlobError::MissingBlob] if no such blob is in this blob set's + /// entry catalog. + /// + /// On success returns the size in bytes of the blob. + pub fn free_ref(&self, hash: BlobHash) -> Result { + let mut entries = self.entries.write(); + match entries.entry(hash.into()) { + map::Entry::Vacant(_) => Err(BlobError::MissingBlob(hash)), + map::Entry::Occupied(mut entry) => { + let size = entry.get().size(); + entry.get_mut().dec_uses(); + if entry.get().uses() == 0 { + entry.remove(); + } + + Ok(size) + } + } + } + + /// Get a handle to the blob with content address `hash`, faulting it in if + /// it is not resident. + /// + /// Subject to the memory budget. Note that the blob will remain resident as + /// long as the returned [BlobHandle] is live. + pub fn get(&self, hash: BlobHash) -> Result { + let blob_or_bytes = { + let entries = self.entries.read(); + let entry = entries.get(&hash.into()).ok_or(BlobError::MissingBlob(hash))?; + match entry { + BlobEntry::Resident { blob, .. } => Ok::<_, BlobError>(Ok(blob.clone())), + BlobEntry::NonResident { uses, .. } => { + let bytes = self.storage.read(hash.into())?; + Ok(Err((*uses, bytes))) + } + } + }?; + let handle = match blob_or_bytes { + Ok(blob) => Ok(blob), + Err((uses, bytes)) => self.insert_with_uses(n_or_one(uses), &bytes), + }?; + assert_eq!(hash, handle.hash()); + self.resident.touch_blob(&handle.frame); + + Ok(handle) + } + + /// Merge the blobs in `other` with this blob set. + /// + /// Subject to the memory budget, all blobs in `other` will be made resident + /// and a [BlobHandle] will be returned for each. + /// + /// This is used during transaction planning to insert and pin the blobs + /// from the transaction- into the committed-state. + pub fn extend_from(&self, other: &Self) -> Result, BlobError> { + let mut handles = Vec::new(); + let entries = other.entries.read(); + for (&key, entry) in entries.iter() { + match entry { + BlobEntry::Resident { uses, blob, .. } => { + let handle = self.insert_with_uses(n_or_one(*uses), blob)?; + assert_eq!(BlobHash::from(key), handle.hash()); + handles.push(handle); + } + BlobEntry::NonResident { uses, .. } => { + let bytes = other.storage.read(key)?; + let handle = self.insert_with_uses(n_or_one(*uses), &bytes)?; + assert_eq!(BlobHash::from(key), handle.hash()); + handles.push(handle); + } + } + } + + Ok(handles.into()) + } + + /// Apply a fallible operation to each blob in this set. + /// + /// Note that non-resident blobs will not be faulted in, but read directly + /// from tiered storage and passed to the closure. + // + // TODO(disk-tables): This is a temporary measure to satisfy the legacy bulk + // snapshotting. In particular, it will read pages from tiered storage, + // without budget accounting, nor an opportunity for optimizations such as + // hardlinking. + pub fn try_for_each>( + &self, + mut f: impl FnMut(BlobHash, NonZeroUsize, &[u8]) -> Result<(), E>, + ) -> Result<(), E> { + let entries = self.entries.read(); + for (&key, entry) in entries.iter().filter(|(_, entry)| entry.uses() > 0) { + match entry { + BlobEntry::Resident { uses, blob, .. } => { + f(key.into(), NonZeroUsize::new(*uses).unwrap(), blob)?; + } + BlobEntry::NonResident { uses, .. } => { + let bytes = self.storage.read(key)?; + f(key.into(), NonZeroUsize::new(*uses).unwrap(), &bytes)?; + } + } + } + + Ok(()) + } + + /// Sums size + hash bytes of each blob in the set. + pub fn physical_bytes_used_by_blobs(&self) -> u64 { + self.entries + .read() + .iter() + .map(|(_, entry)| entry.size() + std::mem::size_of::() as u64) + .sum() + } + + /// Sums size * uses of each blob in the set. + pub fn bytes_used_by_blobs(&self) -> u64 { + self.entries + .read() + .iter() + .map(|(_, entry)| entry.size() * entry.uses() as u64) + .sum() + } + + /// The number of blobs in the set's entry catalog. + pub fn num_blobs(&self) -> usize { + self.entries.read().len() + } +} + +#[cfg(test)] +impl BlobSet { + pub fn usage_counter(&self) -> spacetimedb_data_structures::map::HashMap { + self.entries + .read() + .iter() + .map(|(hash, entry)| (BlobHash::from(*hash), entry.uses())) + .collect() + } + + pub(crate) fn keys(&self) -> Vec { + self.entries.read().keys().copied().collect() + } +} + +impl MemoryUsage for BlobSet { + fn heap_usage(&self) -> usize { + let arc = std::mem::size_of::() * 2; + (3 * arc) + + std::mem::size_of::() + + std::mem::size_of::() + + std::mem::size_of::>>() + + std::mem::size_of::>>() + + (self.physical_bytes_used_by_blobs() as usize) + } +} + +/// A catalog entry is metadata about a blob, without the blob's content. +/// +/// Can be used to instantiate a [BlobSet] without any resident blobs (see: +/// [BlobSet::from_catalog]). +pub struct CatalogEntry { + pub hash: BlobHash, + pub uses: usize, + pub size: u64, +} + +#[derive(Debug)] +enum BlobEntry { + Resident { + uses: usize, + blob: BlobHandle, + }, + #[allow(unused)] + NonResident { + uses: usize, + size: u64, + }, +} + +impl BlobEntry { + fn inc_uses(&mut self) { + self.mod_uses(|uses| *uses += 1); + } + + fn dec_uses(&mut self) { + self.mod_uses(|uses| *uses = uses.saturating_sub(1)); + } + + fn mod_uses(&mut self, f: impl FnOnce(&mut usize)) { + match self { + Self::Resident { uses, .. } | Self::NonResident { uses, .. } => f(uses), + } + } + + fn uses(&self) -> usize { + match self { + Self::Resident { uses, .. } | Self::NonResident { uses, .. } => *uses, + } + } + + fn size(&self) -> u64 { + match self { + Self::NonResident { size, .. } => *size, + Self::Resident { blob, .. } => blob.size(), + } + } +} + +/// A handle to a blob. +/// +/// As long as a handle to a blob is live, the blob is guaranteed to be +/// resident in main memory. +#[derive(Clone, Debug)] +pub struct BlobHandle { + frame: Arc, +} + +impl BlobHandle { + pub fn hash(&self) -> BlobHash { + self.frame.hash() + } + + pub fn size(&self) -> u64 { + self.frame.size() + } +} + +impl Deref for BlobHandle { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.frame + } +} + +/// A set of resident blobs for a specific row. +/// +/// Essentially just a set of [BlobHandle]s that guarantees that those blobs are +/// resident. Used to perform operations such as equality checking that cannot +/// be fallible due to faults. +#[derive(Default)] +pub struct BlobReadSet { + inner: SmallHashMap, +} + +impl BlobReadSet { + /// Create a read set that contains all blobs referenced by `page`. + pub fn new( + visitor: &VarLenVisitorProgram, + page: &Page, + blobs: &BlobSet, + layout: &RowTypeLayout, + offset: PageOffset, + ) -> Result { + let row_data = page.get_row_data(offset, layout.size()); + let mut inner = SmallHashMap::default(); + for vlr in unsafe { visitor.visit_var_len(row_data) }.filter(|vlr| vlr.is_large_blob()) { + let granule = unsafe { page.iter_var_len_object(vlr.first_granule) }.next().unwrap(); + let blob_hash = granule.blob_hash(); + let blob = blobs.get(blob_hash)?; + inner.insert(blob_hash, blob); + } + + Ok(Self { inner }) + } + + /// Create an empty read set. + pub fn empty() -> Self { + Self::default() + } + + /// Get the blob with content address `hash`, or `None` if it is not in the + /// read set. + pub fn get(&self, hash: &BlobHash) -> Option<&BlobHandle> { + self.inner.get(hash) + } + + /// Iterator over all entries in the read set. + pub fn iter(&self) -> impl Iterator { + self.inner.iter() + } +} + +/// A map of resident blobs. +/// +/// Used when restoring from a snapshot (see: [BlobSet::from_blob_map]). +#[derive(Default)] +pub struct BlobMap { + inner: ObjectMap, +} + +impl BlobMap { + /// Insert a blob into the map. The caller must guarantee that the `hash` is + /// equal to `BlobHash::hash_from_bytes(blob)`. + pub fn insert_with_uses(&mut self, hash: &BlobHash, uses: NonZeroUsize, blob: Box<[u8]>) { + debug_assert_eq!(hash, &BlobHash::hash_from_bytes(&blob)); + self.inner.insert(ObjectKey::from(*hash), BlobMapEntry { uses, blob }); + } +} + +struct BlobMapEntry { + uses: NonZeroUsize, + blob: Box<[u8]>, +} + +fn n_or_one(x: usize) -> NonZeroUsize { + NonZeroUsize::new(x).unwrap_or(NonZeroUsize::MIN) +} diff --git a/crates/table/src/tiered/budget.rs b/crates/table/src/tiered/budget.rs index 28355c70929..47fd8cc006e 100644 --- a/crates/table/src/tiered/budget.rs +++ b/crates/table/src/tiered/budget.rs @@ -20,6 +20,59 @@ pub struct ByteBudget { config: ByteBudgetConfig, } +impl ByteBudget { + pub fn new(config: ByteBudgetConfig) -> Result { + config.validate_then(|config| Self { + state: Arc::new(Mutex::new(ByteBudgetState { accounted_bytes: 0 })), + config, + }) + } + + pub fn unlimited() -> Self { + Self::try_from(ByteBudgetConfig::unlimited()).unwrap() + } + + pub fn usage(&self) -> ByteBudgetUsage { + ByteBudgetUsage { + accounted_bytes: self.state.lock().unwrap().accounted_bytes, + } + } + + pub fn acquire(&self, bytes: u64) -> Result { + let mut state = self.state.lock().unwrap(); + if state.accounted_bytes + bytes <= self.config.hard_limit_bytes { + state.accounted_bytes += bytes; + Ok(BudgetPermit { + state: self.state.clone(), + bytes, + }) + } else { + Err(BudgetExceeded { + requested_bytes: bytes, + accounted_bytes: state.accounted_bytes, + hard_limit_bytes: self.config.hard_limit_bytes, + }) + } + } + + pub(super) fn force_acquire(&self, bytes: u64) -> BudgetPermit { + let mut state = self.state.lock().unwrap(); + state.accounted_bytes += bytes; + BudgetPermit { + state: self.state.clone(), + bytes, + } + } +} + +impl TryFrom for ByteBudget { + type Error = ConfigError; + + fn try_from(config: ByteBudgetConfig) -> Result { + ByteBudget::new(config) + } +} + #[derive(Clone, Copy, Debug)] pub struct ByteBudgetConfig { pub low_water_bytes: u64, @@ -76,47 +129,6 @@ pub struct ByteBudgetUsage { pub accounted_bytes: u64, } -impl ByteBudget { - pub fn new(config: ByteBudgetConfig) -> Result { - config.validate_then(|config| Self { - state: Arc::new(Mutex::new(ByteBudgetState { accounted_bytes: 0 })), - config, - }) - } - - pub fn usage(&self) -> ByteBudgetUsage { - ByteBudgetUsage { - accounted_bytes: self.state.lock().unwrap().accounted_bytes, - } - } - - pub fn acquire(&self, bytes: u64) -> Result { - let mut state = self.state.lock().unwrap(); - if state.accounted_bytes + bytes <= self.config.hard_limit_bytes { - state.accounted_bytes += bytes; - Ok(BudgetPermit { - state: self.state.clone(), - bytes, - }) - } else { - Err(BudgetExceeded { - requested_bytes: bytes, - accounted_bytes: state.accounted_bytes, - hard_limit_bytes: self.config.hard_limit_bytes, - }) - } - } - - pub(super) fn force_acquire(&self, bytes: u64) -> BudgetPermit { - let mut state = self.state.lock().unwrap(); - state.accounted_bytes += bytes; - BudgetPermit { - state: self.state.clone(), - bytes, - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/table/src/tiered/error.rs b/crates/table/src/tiered/error.rs new file mode 100644 index 00000000000..2ad8f0c6fad --- /dev/null +++ b/crates/table/src/tiered/error.rs @@ -0,0 +1,88 @@ +use std::io; + +use spacetimedb_lib::bsatn::DecodeError; + +use crate::{ + blob_store::BlobHash, + indexes::PageIndex, + page, + tiered::{map::ObjectKey, BudgetExceeded}, +}; + +#[derive(Debug, thiserror::Error)] +pub enum PageError { + #[error("maximum number of pages exceeded")] + TooManyPages, + #[error(transparent)] + MemoryLimitExceeded(#[from] BudgetExceeded), + #[error(transparent)] + Page(page::Error), + #[error("page at index {0:?} is missing")] + MissingPage(PageIndex), + #[error("object {0} is missing")] + MissingObject(blake3::Hash), + #[error("error decoding page from bsatn")] + Deserialize(#[from] DecodeError), + #[error(transparent)] + Storage(#[from] ObjectStoreError), +} + +impl PageError { + pub fn is_transient(&self) -> bool { + match self { + PageError::TooManyPages + | PageError::MemoryLimitExceeded(_) + | PageError::Page(_) + | PageError::MissingPage(_) + | PageError::MissingObject(_) + | PageError::Deserialize(_) => false, + + PageError::Storage(storage) => storage.is_transient(), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum BlobError { + #[error(transparent)] + MemoryLimitExceeded(#[from] BudgetExceeded), + #[error("blob {0} is missing")] + MissingBlob(BlobHash), + #[error(transparent)] + Storage(#[from] ObjectStoreError), +} + +impl BlobError { + pub fn is_transient(&self) -> bool { + match self { + BlobError::MemoryLimitExceeded(_) | BlobError::MissingBlob(_) => false, + BlobError::Storage(storage) => storage.is_transient(), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ObjectStoreError { + #[error("object {0} is unknown")] + UnknownObject(ObjectKey), + #[error("object {0} is missing")] + MissingObject(ObjectKey), + #[error("expected object hash {expected} doesn't match computed hash {computed}")] + HashMismatch { + expected: blake3::Hash, + computed: blake3::Hash, + }, + #[error(transparent)] + Io(#[from] io::Error), +} + +impl ObjectStoreError { + pub fn is_transient(&self) -> bool { + match self { + ObjectStoreError::UnknownObject(_) + | ObjectStoreError::MissingObject(_) + | ObjectStoreError::HashMismatch { .. } => false, + ObjectStoreError::Io(_) => true, + } + } +} diff --git a/crates/table/src/tiered/map.rs b/crates/table/src/tiered/map.rs new file mode 100644 index 00000000000..c0fe7ac64ba --- /dev/null +++ b/crates/table/src/tiered/map.rs @@ -0,0 +1,191 @@ +use std::{ + fmt::{self, Write as _}, + hash::{BuildHasherDefault, Hash, Hasher}, +}; + +use hashbrown::{ + hash_map::{self, OccupiedEntry, VacantEntry}, + HashMap, +}; + +use crate::{blob_store::BlobHash, page::Page}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ObjectKey([u8; 32]); + +impl ObjectKey { + fn hash_prefix(&self) -> &[u8] { + &self.0[..N] + } +} + +impl Hash for ObjectKey { + fn hash(&self, state: &mut H) { + const PREFIX_LEN: usize = (u64::BITS / 8) as usize; + let prefix = u64::from_le_bytes(self.hash_prefix::().try_into().unwrap()); + state.write_u64(prefix); + } +} + +impl fmt::Display for ObjectKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + const ALPHABET: &[u8; 16] = b"0123456789abcdef"; + for &b in self.0.iter() { + f.write_char(ALPHABET[(b >> 4) as usize] as char)?; + f.write_char(ALPHABET[(b & 0xf) as usize] as char)?; + } + + Ok(()) + } +} + +impl From for ObjectKey { + fn from(blob_hash: BlobHash) -> Self { + Self(blob_hash.data) + } +} + +impl From for BlobHash { + fn from(key: ObjectKey) -> Self { + BlobHash { data: key.0 } + } +} + +impl From for ObjectKey { + fn from(hash: blake3::Hash) -> Self { + Self(*hash.as_bytes()) + } +} + +impl From<&mut Page> for ObjectKey { + fn from(page: &mut Page) -> Self { + Self::from(page.save_or_get_content_hash()) + } +} + +#[derive(Default)] +pub struct IdentityHasher(u64); + +impl Hasher for IdentityHasher { + fn finish(&self) -> u64 { + self.0 + } + + fn write(&mut self, bytes: &[u8]) { + self.0 = u64::from_ne_bytes(bytes.try_into().unwrap()); + } + + fn write_u64(&mut self, value: u64) { + self.0 = value; + } +} + +#[derive(Debug, PartialEq)] +pub struct ObjectMap { + inner: HashMap>, +} + +impl ObjectMap { + pub fn contains_key(&self, k: &ObjectKey) -> bool { + self.inner.contains_key(k) + } + + pub fn entry(&mut self, k: ObjectKey) -> Entry<'_, V> { + self.inner.entry(k).into() + } + + pub fn get(&self, k: &ObjectKey) -> Option<&V> { + self.inner.get(k) + } + + #[allow(unused)] + pub fn get_mut(&mut self, k: &ObjectKey) -> Option<&mut V> { + self.inner.get_mut(k) + } + + pub fn insert(&mut self, k: ObjectKey, v: V) -> Option { + self.inner.insert(k, v) + } + + #[allow(unused)] + pub fn remove(&mut self, k: ObjectKey) -> Option { + self.inner.remove(&k) + } + + pub fn iter(&self) -> impl Iterator { + self.inner.iter() + } + + #[allow(unused)] + pub fn keys(&self) -> impl Iterator { + self.inner.keys() + } + + pub fn len(&self) -> usize { + self.inner.len() + } + + #[allow(unused)] + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub fn retain(&mut self, f: impl FnMut(&ObjectKey, &mut V) -> bool) { + self.inner.retain(f); + } +} + +impl Default for ObjectMap { + fn default() -> Self { + Self { inner: <_>::default() } + } +} + +impl IntoIterator for ObjectMap { + type Item = (ObjectKey, V); + type IntoIter = hash_map::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.inner.into_iter() + } +} + +impl FromIterator<(ObjectKey, V)> for ObjectMap { + fn from_iter>(iter: T) -> Self { + Self { + inner: HashMap::from_iter(iter), + } + } +} + +pub enum Entry<'a, V> { + Vacant(VacantEntry<'a, ObjectKey, V, BuildHasherDefault>), + Occupied(OccupiedEntry<'a, ObjectKey, V, BuildHasherDefault>), +} + +impl<'a, V> Entry<'a, V> { + pub fn and_modify(self, f: impl FnOnce(&mut V)) -> Self { + match self { + Self::Occupied(mut entry) => { + f(entry.get_mut()); + Self::Occupied(entry) + } + Self::Vacant(entry) => Self::Vacant(entry), + } + } + + pub fn or_error(self, f: impl FnOnce() -> E) -> Result<&'a mut V, E> { + let Self::Occupied(entry) = self else { return Err(f()) }; + + Ok(entry.into_mut()) + } +} + +impl<'a, V> From>> for Entry<'a, V> { + fn from(entry: hash_map::Entry<'a, ObjectKey, V, BuildHasherDefault>) -> Self { + match entry { + hash_map::Entry::Occupied(entry) => Self::Occupied(entry), + hash_map::Entry::Vacant(entry) => Self::Vacant(entry), + } + } +} diff --git a/crates/table/src/tiered/mem.rs b/crates/table/src/tiered/mem.rs new file mode 100644 index 00000000000..63bd1ad94ae --- /dev/null +++ b/crates/table/src/tiered/mem.rs @@ -0,0 +1,445 @@ +use std::{ + cmp, fmt, + mem::ManuallyDrop, + ops::{Deref, DerefMut}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, Weak, + }, +}; + +use parking_lot::{ArcRwLockReadGuard, ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockWriteGuard}; +use spacetimedb_data_structures::map::{hash_map, IntMap, ValidAsIdentityHash}; +use spacetimedb_memory_usage::MemoryUsage; + +use crate::{ + blob_store::BlobHash, + indexes::PAGE_SIZE, + page::Page, + page_pool::PagePool, + tiered::{ + map::{self, ObjectKey, ObjectMap}, + BudgetExceeded, BudgetPermit, ByteBudget, ByteBudgetUsage, + }, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PageId(u64); + +impl std::hash::Hash for PageId { + fn hash(&self, state: &mut H) { + state.write_u64(self.0); + } +} + +impl ValidAsIdentityHash for PageId {} + +pub struct ReservedBlob { + frame: BlobFrame, +} + +#[derive(Debug)] +pub struct BlobFrame { + hash: BlobHash, + bytes: Box<[u8]>, + access: Access, + #[allow(unused)] + permit: BudgetPermit, +} + +impl BlobFrame { + pub fn hash(&self) -> BlobHash { + self.hash + } + + pub fn size(&self) -> u64 { + self.bytes.len() as _ + } + + fn access(&self) -> AccessSnapshot { + self.access.snapshot() + } + + fn touch(&self, epoch: u64) { + self.access.touch(epoch); + } +} + +impl Deref for BlobFrame { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.bytes + } +} + +pub struct ReservedPage { + id: PageId, + frame: PageFrame, +} + +#[derive(Debug)] +pub struct PageFrame { + pub(super) id: PageId, + page: Arc>, + access: Access, + #[allow(unused)] + permit: BudgetPermit, +} + +impl PageFrame { + pub fn read(&self) -> ArcRwLockReadGuard { + RwLock::read_arc(&self.page) + } + + pub fn write(&self) -> ArcRwLockWriteGuard { + RwLock::write_arc(&self.page) + } + + pub fn write_ref(&self) -> RwLockWriteGuard<'_, PooledPage> { + self.page.write() + } + + fn access(&self) -> AccessSnapshot { + self.access.snapshot() + } + + fn touch(&self, epoch: u64) { + self.access.touch(epoch); + } +} + +impl MemoryUsage for PageFrame { + fn heap_usage(&self) -> usize { + let refcount = std::mem::size_of::() * 2; + + refcount + + std::mem::size_of::>() + + std::mem::size_of::() + + std::mem::size_of::() + + self.page.read().heap_usage() + } +} + +/// Tracks resident objects and controls the residency budget. +/// +/// The objects, [PageFrame]s and [BlobFrame]s, are owned by [super::PageSet] +/// respectively [super::BlobSet]. The [ResidentObjects] only tracks [Weak] +/// references to them. +/// +/// To make an object resident, a reservation must be acquired via +/// [ResidentObjects::reserve_blob] respectively [ResidentObjects::reserve_page]. +/// This may fail because the available [ByteBudget] is already used. +/// +/// Once a reservation has been successfully acquired, it may be exchanged for +/// its actual "frame" (allocation), see [ResidentObjects::redeem_blob], +/// [ResidentObjects::redeem_page]. +/// +/// NOTE: This two-step process is deliberate, as it reduces the surface for +/// lock-ordering bugs in the owning datastructures. +/// +/// [PageFrame]s and [BlobFrame]s consume permits from the [ByteBudget] until +/// they're dropped. [ReservedPage]s and [ReservedBlob]s contain frames, so +/// count towards the budget even while not redeemed. +/// +/// If a reservation attempt fails, callers may request eviction candidates +/// (see: [ResidentObjects::evictable_blobs], [ResidentObjects::evictable_pages]). +/// +/// Eviction candidates are objects that have only a single strong reference, +/// and are sorted according to an LRU-2 algorithm. Note that, for the latter to +/// work, the owners must call [ResidentObjects::touch_blob] respectively +/// [ResidentObjects::touch_page] whenever they hand out an object reference. +#[derive(Debug)] +pub struct ResidentObjects { + budget: ByteBudget, + access_epoch: AtomicU64, + page_id: AtomicU64, + pages: Mutex>>, + blobs: Mutex>>, +} + +impl ResidentObjects { + pub fn new(budget: ByteBudget) -> Self { + Self { + budget, + access_epoch: AtomicU64::new(1), + page_id: <_>::default(), + pages: <_>::default(), + blobs: <_>::default(), + } + } + + pub(super) fn reserve_blob( + &self, + size: u64, + alloc: impl FnOnce() -> Box<[u8]>, + ) -> Result { + let permit = self.budget.acquire(size)?; + let bytes = alloc(); + assert_eq!(size, bytes.len() as u64); + let hash = BlobHash::hash_from_bytes(&bytes); + let frame = BlobFrame { + hash, + bytes, + access: Access::new(self.access_epoch.fetch_add(1, Ordering::Relaxed)), + permit, + }; + + Ok(ReservedBlob { frame }) + } + + pub(super) fn redeem_blob(&self, ReservedBlob { frame }: ReservedBlob) -> Arc { + let key = frame.hash.into(); + let frame = Arc::new(frame); + assert!(self.blobs.lock().unwrap().insert(key, Arc::downgrade(&frame)).is_none()); + frame + } + + pub(super) fn evictable_blobs(&self, candidates: &mut Vec) { + { + let mut blobs = self.blobs.lock().unwrap(); + blobs.retain(|&key, weak| { + let Some(frame) = weak.upgrade() else { + return false; + }; + + if Arc::strong_count(&frame) <= 2 { + candidates.push(BlobEvictionCandidate { + key, + access: frame.access(), + }); + } + + true + }); + } + candidates.sort_unstable_by_key(|candidate| candidate.access); + } + + pub(super) fn evict_blob(&self, victim: BlobEvictionCandidate) { + let mut blobs = self.blobs.lock().unwrap(); + if let map::Entry::Occupied(entry) = blobs.entry(victim.key) + && entry.get().strong_count() == 0 + { + entry.remove(); + } + } + + pub(super) fn touch_blob(&self, frame: &BlobFrame) { + frame.touch(self.access_epoch.fetch_add(1, Ordering::Relaxed)); + } + + pub(super) fn reserve_page>( + &self, + alloc: impl FnOnce() -> Result, + ) -> Result { + let permit = self.budget.acquire(PAGE_SIZE as _)?; + let page = alloc()?; + Ok(self.reserve_page_with_permit(permit, page)) + } + + pub(super) fn redeem_page(&self, ReservedPage { id, frame }: ReservedPage) -> Arc { + let frame = Arc::new(frame); + assert!(self.pages.lock().unwrap().insert(id, Arc::downgrade(&frame)).is_none()); + frame + } + + pub(super) fn evictable_pages(&self, candidates: &mut Vec) { + { + let mut pages = self.pages.lock().unwrap(); + pages.retain(|&id, weak| { + let Some(frame) = weak.upgrade() else { + return false; + }; + + if Arc::strong_count(&frame) <= 2 { + candidates.push(PageEvictionCandidate { + id, + access: frame.access(), + }); + } + + true + }); + } + + candidates.sort_unstable_by_key(|candidate| candidate.access); + } + + pub(super) fn evict_page(&self, victim: PageEvictionCandidate) { + let mut pages = self.pages.lock().unwrap(); + if let hash_map::Entry::Occupied(entry) = pages.entry(victim.id) + && entry.get().strong_count() == 0 + { + entry.remove(); + } + } + + pub(super) fn touch_page(&self, frame: &PageFrame) { + frame.touch(self.access_epoch.fetch_add(1, Ordering::Relaxed)); + } + + pub fn usage(&self) -> ByteBudgetUsage { + self.budget.usage() + } + + pub(super) fn register_page_unchecked(&self, pool: PagePool, page: Box) -> Arc { + let permit = self.budget.force_acquire(PAGE_SIZE as u64); + let reservation = self.reserve_page_with_permit(permit, PooledPage::new(page, pool)); + self.redeem_page(reservation) + } + + pub(super) fn register_blob_unchecked(&self, blob: Box<[u8]>) -> Arc { + let permit = self.budget.force_acquire(blob.len() as u64); + let reservation = self.reserve_blob_with_permit(permit, blob); + self.redeem_blob(reservation) + } + + fn reserve_page_with_permit(&self, permit: BudgetPermit, page: PooledPage) -> ReservedPage { + let id = PageId(self.page_id.fetch_add(1, Ordering::Relaxed)); + let frame = PageFrame { + id, + page: Arc::new(RwLock::new(page)), + access: Access::new(self.access_epoch.fetch_add(1, Ordering::Relaxed)), + permit, + }; + + ReservedPage { id, frame } + } + + fn reserve_blob_with_permit(&self, permit: BudgetPermit, bytes: Box<[u8]>) -> ReservedBlob { + let hash = BlobHash::hash_from_bytes(&bytes); + let frame = BlobFrame { + hash, + bytes, + access: Access::new(self.access_epoch.fetch_add(1, Ordering::Relaxed)), + permit, + }; + + ReservedBlob { frame } + } +} + +#[derive(Debug)] +pub struct BlobEvictionCandidate { + pub key: ObjectKey, + access: AccessSnapshot, +} + +#[derive(Debug)] +pub struct PageEvictionCandidate { + pub id: PageId, + access: AccessSnapshot, +} + +#[derive(Debug)] +struct Access { + last_access_epoch: AtomicU64, + prev_access_epoch: AtomicU64, +} + +impl Access { + pub fn new(epoch: u64) -> Self { + Self { + last_access_epoch: AtomicU64::new(epoch), + prev_access_epoch: <_>::default(), + } + } + + pub fn snapshot(&self) -> AccessSnapshot { + AccessSnapshot { + last_access_epoch: self.last_access_epoch.load(Ordering::Relaxed), + prev_access_epoch: self.prev_access_epoch.load(Ordering::Relaxed), + } + } + + pub fn touch(&self, epoch: u64) { + let prev = self.last_access_epoch.swap(epoch, Ordering::Relaxed); + self.prev_access_epoch.store(prev, Ordering::Relaxed); + } +} + +#[derive(Clone, Copy, Debug)] +struct AccessSnapshot { + last_access_epoch: u64, + prev_access_epoch: u64, +} + +impl Ord for AccessSnapshot { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn key( + &AccessSnapshot { + last_access_epoch, + prev_access_epoch, + }: &AccessSnapshot, + ) -> (bool, u64, u64) { + ( + prev_access_epoch != 0, + if prev_access_epoch == 0 { + last_access_epoch + } else { + prev_access_epoch + }, + last_access_epoch, + ) + } + + key(self).cmp(&key(other)) + } +} + +impl PartialOrd for AccessSnapshot { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for AccessSnapshot { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == cmp::Ordering::Equal + } +} + +impl Eq for AccessSnapshot {} + +pub(super) struct PooledPage { + page: ManuallyDrop>, + pool: PagePool, +} + +impl PooledPage { + pub fn new(page: Box, pool: PagePool) -> Self { + Self { + page: ManuallyDrop::new(page), + pool, + } + } +} + +impl Drop for PooledPage { + fn drop(&mut self) { + // SAFETY: We're in `drop`, so won't be using `ManuallyDrop` again. + let page = unsafe { ManuallyDrop::take(&mut self.page) }; + self.pool.put(page); + } +} + +impl Deref for PooledPage { + type Target = Page; + + fn deref(&self) -> &Self::Target { + &self.page + } +} + +impl DerefMut for PooledPage { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.page + } +} + +impl fmt::Debug for PooledPage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PooledPage").field("page", &self.page).finish() + } +} diff --git a/crates/table/src/tiered/mod.rs b/crates/table/src/tiered/mod.rs index dbec1f3cc26..7345bc69522 100644 --- a/crates/table/src/tiered/mod.rs +++ b/crates/table/src/tiered/mod.rs @@ -1,8 +1,118 @@ -mod budget; -pub use budget::{BudgetExceeded, BudgetPermit, ByteBudget, ByteBudgetConfig, ByteBudgetUsage}; +use std::sync::Arc; -mod page_manager; -pub use page_manager::{PageError, PageEvictionPolicy, PageHandle, PageManager, ReservedPage}; +use spacetimedb_memory_usage::MemoryUsage; + +use crate::page_pool::PagePool; +mod blob_set; +mod budget; +mod error; +mod map; +mod mem; +mod page_manager; mod page_set; -pub use page_set::{PageSet, PreparedCommit}; +mod storage; + +pub use self::{ + blob_set::{BlobHandle, BlobMap, BlobReadSet, BlobSet}, + budget::{BudgetExceeded, BudgetPermit, ByteBudget, ByteBudgetConfig, ByteBudgetUsage}, + error::{BlobError, ObjectStoreError, PageError}, + mem::ResidentObjects, + page_manager::{PageEvictionPolicy, PageHandle, PageManager, ReservedPage}, + page_set::{DeleteRowError, PageSet, PreparedCommit}, + storage::{MemoryObjectStore, ObjectStore, TieredStorage}, +}; + +pub struct TieredStorageConfig { + pub page_pool: PagePool, + pub local_object_store: Box, + pub remote_object_store: Option>, + pub resident_budget: ByteBudget, +} + +impl TieredStorageConfig { + pub fn new_for_test() -> Self { + Self::memory_unlimited(PagePool::new_for_test()) + } + + pub fn new_for_module_check(page_pool: PagePool) -> Self { + Self::memory_unlimited(page_pool) + } + + pub fn new_for_memory_database(page_pool: PagePool) -> Self { + Self::memory_unlimited(page_pool) + } + + fn memory_unlimited(page_pool: PagePool) -> Self { + Self { + page_pool, + local_object_store: Box::new(MemoryObjectStore::default()), + remote_object_store: None, + resident_budget: ByteBudget::unlimited(), + } + } +} + +#[derive(Clone)] +pub struct TieredDatabaseStorage { + pages: Arc, + blobs: Arc, + + cache: Arc, + store: Arc, +} + +impl TieredDatabaseStorage { + pub fn new( + TieredStorageConfig { + page_pool, + resident_budget, + local_object_store, + remote_object_store, + }: TieredStorageConfig, + ) -> Self { + let store = Arc::new(TieredStorage::new(local_object_store, remote_object_store)); + let cache = Arc::new(ResidentObjects::new(resident_budget)); + let pages = Arc::new(PageManager::new(cache.clone(), page_pool, store.clone())); + let blobs = Arc::new(BlobSet::new(cache.clone(), store.clone())); + + Self { + pages, + blobs, + cache, + store, + } + } + + pub fn with_blobs(config: TieredStorageConfig, blobs: BlobMap) -> Self { + let mut this = Self::new(config); + this.blobs = BlobSet::from_blob_map(this.cache.clone(), this.store.clone(), blobs).into(); + this + } + + pub fn pages(&self) -> &Arc { + &self.pages + } + + pub fn blobs(&self) -> &Arc { + &self.blobs + } + + pub fn physical_bytes_used_by_blobs(&self) -> u64 { + self.blobs.physical_bytes_used_by_blobs() + } + + pub fn merge_blobs_from(&self, blobs: &BlobSet) -> Result, BlobError> { + self.blobs.extend_from(blobs) + } + + pub fn new_empty_blob_set(&self) -> BlobSet { + BlobSet::new(self.cache.clone(), self.store.clone()) + } +} + +impl MemoryUsage for TieredDatabaseStorage { + fn heap_usage(&self) -> usize { + todo!() + } +} diff --git a/crates/table/src/tiered/page_manager.rs b/crates/table/src/tiered/page_manager.rs index e2a580395d8..20d30c400ee 100644 --- a/crates/table/src/tiered/page_manager.rs +++ b/crates/table/src/tiered/page_manager.rs @@ -1,24 +1,23 @@ use std::{ - fmt, io, - mem::{self, ManuallyDrop}, ops::{Deref, DerefMut}, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, Mutex, Weak, - }, + sync::{Arc, Mutex}, }; -use parking_lot::{ArcRwLockReadGuard, ArcRwLockWriteGuard, RawRwLock, RwLock}; -use slab::Slab; -use spacetimedb_lib::bsatn::DecodeError; +use parking_lot::{ArcRwLockReadGuard, ArcRwLockWriteGuard, RawRwLock, RwLock, RwLockWriteGuard}; +use spacetimedb_lib::bsatn; use spacetimedb_memory_usage::MemoryUsage; use spacetimedb_sats::layout::Size; use crate::{ - indexes::{PageIndex, PAGE_SIZE}, - page::{self, Page, PageMetadata}, + page::{Page, PageMetadata}, page_pool::PagePool, - tiered::{BudgetExceeded, BudgetPermit, ByteBudget, ByteBudgetConfig}, + tiered::{ + error::PageError, + map::ObjectKey, + mem::{self, PageEvictionCandidate, PageFrame, PageId, PooledPage}, + storage::TieredStorage, + BudgetExceeded, ByteBudget, ObjectStoreError, ResidentObjects, + }, }; #[cfg(test)] @@ -28,7 +27,9 @@ use crate::var_len::VarLenMembers; /// /// Created by [PageFrame::read()]. pub struct PageFrameReadGuard { - guard: ArcRwLockReadGuard, + guard: ArcRwLockReadGuard, + #[allow(unused)] + pin: Arc, } impl Deref for PageFrameReadGuard { @@ -46,9 +47,10 @@ impl Deref for PageFrameReadGuard { /// /// Created by [PageFrame::write]. pub struct PageFrameWriteGuard { - guard: ArcRwLockWriteGuard, - frame: Arc, - slot: Arc>, + guard: ArcRwLockWriteGuard, + pin: Arc, + + slot: Arc>, fixed_row_size: Size, eviction_policy: PageEvictionPolicy, } @@ -56,7 +58,7 @@ pub struct PageFrameWriteGuard { impl PageFrameWriteGuard { pub fn into_page_handle(self) -> PageHandle { PageHandle { - frame: self.frame.clone(), + frame: Arc::clone(&self.pin), } } } @@ -77,25 +79,25 @@ impl DerefMut for PageFrameWriteGuard { impl Drop for PageFrameWriteGuard { fn drop(&mut self) { - let metadata = self.guard.metadata(self.fixed_row_size); - let mut slot_guard = self.slot.lock().unwrap(); + let metadata = self.metadata(self.fixed_row_size); + let mut slot_guard = self.slot.write(); match &mut *slot_guard { PageSlot::Resident { handle, - state, + backing, metadata: cached_metadata, eviction_policy: cached_eviction_policy, - } if Arc::ptr_eq(&handle.frame, &self.frame) => { + } if Arc::ptr_eq(&handle.frame, &self.pin) => { *cached_metadata = metadata; *cached_eviction_policy = self.eviction_policy; - state.mark_dirty(None); + backing.take(); } _ => { *slot_guard = PageSlot::Resident { handle: PageHandle { - frame: self.frame.clone(), + frame: self.pin.clone(), }, - state: ResidentPageState::Dirty { hash: None }, + backing: None, metadata, eviction_policy: self.eviction_policy, }; @@ -104,47 +106,6 @@ impl Drop for PageFrameWriteGuard { } } -/// Placeholder trait for disk / object storage. -pub trait PageBackingStore: fmt::Debug + Send + Sync + 'static { - /// Load a [Page] by its content hash from backing storage . - fn load_page(&self, hash: blake3::Hash) -> Result, PageIoError>; -} - -impl PageBackingStore for () { - fn load_page(&self, _: blake3::Hash) -> Result, PageIoError> { - unimplemented!("no page backing store configured") - } -} - -#[derive(Debug, thiserror::Error)] -pub enum PageError { - #[error("maximum number of pages exceeded")] - TooManyPages, - #[error(transparent)] - MemoryLimitExceeded(#[from] BudgetExceeded), - #[error(transparent)] - Page(page::Error), - #[error("page at index {0:?} is missing")] - MissingPage(PageIndex), - #[error("object {0} is missing")] - MissingObject(blake3::Hash), - #[error(transparent)] - Io(#[from] PageIoError), - #[error("error decoding page from bsatn")] - Deserialize(DecodeError), -} - -#[derive(Debug, thiserror::Error)] -pub enum PageIoError { - #[error("expected page hash {expected} doesn't match computed page hash {computed}")] - HashMismatch { - expected: blake3::Hash, - computed: blake3::Hash, - }, - #[error(transparent)] - Io(#[from] io::Error), -} - #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PageEvictionPolicy { Evictable, @@ -157,47 +118,52 @@ pub enum PageEvictionPolicy { /// The public API is read-only. #[derive(Clone, Debug)] pub struct PageSlotHandle { - slot: Arc>, + slot: Arc>, } impl PageSlotHandle { pub fn is_resident(&self) -> bool { - self.slot.lock().unwrap().is_resident() + self.slot.read().is_resident() } pub fn is_absent(&self) -> bool { - self.slot.lock().unwrap().is_absent() + self.slot.read().is_absent() } pub fn has_space_for_row(&self, fixed_row_size: Size, num_var_len_granules: usize) -> Option { - self.slot - .lock() - .unwrap() - .has_space_for_row(fixed_row_size, num_var_len_granules) + self.slot.read().has_space_for_row(fixed_row_size, num_var_len_granules) } pub fn is_full(&self, fixed_row_size: Size) -> Option { - self.slot.lock().unwrap().is_full(fixed_row_size) + self.slot.read().is_full(fixed_row_size) } pub fn available_var_len_granules(&self) -> Option { - self.slot.lock().unwrap().available_var_len_granules() + self.slot.read().available_var_len_granules() } pub fn bytes_used_by_rows(&self) -> usize { - self.slot.lock().unwrap().bytes_used_by_rows() + self.slot.read().bytes_used_by_rows() } pub fn metadata(&self) -> Option { - self.slot.lock().unwrap().metadata() + self.slot.read().metadata() } pub fn page(&self) -> Option { - self.slot.lock().unwrap().page().cloned() + self.slot.read().page().cloned() + } + + pub fn page_id(&self) -> Option { + self.slot.read().page_id() + } + + pub(super) fn lock(&self) -> RwLockWriteGuard<'_, PageSlot> { + self.slot.write() } pub(super) fn free(&self) { - let mut slot = self.slot.lock().unwrap(); + let mut slot = self.slot.write(); *slot = PageSlot::Absent; } } @@ -209,23 +175,23 @@ impl PageSlotHandle { fixed_row_size: Size, var_len_visitor: &impl VarLenMembers, ) -> usize { - let slot = self.slot.lock().unwrap(); + let slot = self.slot.read(); unsafe { slot.reconstruct_bytes_used_by_rows(fixed_row_size, var_len_visitor) } } pub fn reconstruct_num_rows(&self) -> usize { - self.slot.lock().unwrap().reconstruct_num_rows() + self.slot.read().reconstruct_num_rows() } } impl MemoryUsage for PageSlotHandle { fn heap_usage(&self) -> usize { let maybe_resident = { - let slot = self.slot.lock().unwrap(); + let slot = self.slot.read(); slot.page().cloned() }; - let refcounts = mem::size_of::() * 2; - refcounts + mem::size_of::>() + maybe_resident.as_ref().map_or(0, MemoryUsage::heap_usage) + let refcounts = std::mem::size_of::() * 2; + refcounts + std::mem::size_of::>() + maybe_resident.as_ref().map_or(0, MemoryUsage::heap_usage) } } @@ -234,14 +200,14 @@ impl MemoryUsage for PageSlotHandle { #[allow(unused)] pub enum PageSlot { /// The page has been freed, so the slot is empty. - /// This exists to maintain stable [PageIndex]ex. + /// This exists to maintain stable [PageIndex]es. Absent, /// The page is resident in memory. Resident { handle: PageHandle, metadata: PageMetadata, - #[allow(unused)] - state: ResidentPageState, + // `Some` if the page is clean and present in backing storage. + backing: Option, eviction_policy: PageEvictionPolicy, }, /// The page is not resident in memory. @@ -260,8 +226,8 @@ impl PageSlot { pub fn has_space_for_row(&self, fixed_row_size: Size, num_var_len_granules: usize) -> Option { match self { - PageSlot::Absent => None, - PageSlot::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => { + Self::Absent => None, + Self::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => { Some(metadata.has_space_for_row(fixed_row_size, num_var_len_granules)) } } @@ -269,8 +235,8 @@ impl PageSlot { pub fn is_full(&self, fixed_row_size: Size) -> Option { match self { - PageSlot::Absent => None, - PageSlot::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => { + Self::Absent => None, + Self::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => { Some(metadata.is_full(fixed_row_size)) } } @@ -278,8 +244,8 @@ impl PageSlot { pub fn available_var_len_granules(&self) -> Option { match self { - PageSlot::Absent => None, - PageSlot::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => { + Self::Absent => None, + Self::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => { Some(metadata.available_var_len_granules()) } } @@ -287,8 +253,8 @@ impl PageSlot { pub fn bytes_used_by_rows(&self) -> usize { match self { - PageSlot::Absent => 0, - PageSlot::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => { + Self::Absent => 0, + Self::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => { metadata.bytes_used_by_rows as _ } } @@ -296,15 +262,22 @@ impl PageSlot { pub fn metadata(&self) -> Option { match self { - PageSlot::Absent => None, - PageSlot::Resident { metadata, .. } | PageSlot::NonResident { metadata, .. } => Some(*metadata), + Self::Absent => None, + Self::Resident { metadata, .. } | Self::NonResident { metadata, .. } => Some(*metadata), + } + } + + pub fn page_id(&self) -> Option { + match self { + Self::Resident { handle, .. } => Some(handle.page_id()), + Self::NonResident { .. } | Self::Absent => None, } } fn page(&self) -> Option<&PageHandle> { match self { - PageSlot::Absent | PageSlot::NonResident { .. } => None, - PageSlot::Resident { handle, .. } => Some(handle), + Self::Absent | PageSlot::NonResident { .. } => None, + Self::Resident { handle, .. } => Some(handle), } } } @@ -350,134 +323,44 @@ pub struct PageHandle { } impl PageHandle { - /// Lock the page for reading. - pub fn read(&self) -> PageFrameReadGuard { - self.frame.read() - } -} - -impl MemoryUsage for PageHandle { - fn heap_usage(&self) -> usize { - let page_heap_usage = self.read().heap_usage(); - // `Arc` + `Arc>` + `Arc` - let refcounts = 3 * (mem::size_of::() * 2); - - refcounts - + mem::size_of::() - + mem::size_of::>() - + mem::size_of::() - + mem::size_of::() - + page_heap_usage - } -} - -/// A page frame is basically an allocated page. -/// -/// Eviction is controlled via weak references to the internals of this type. -#[derive(Debug)] -pub struct PageFrame { - lease: Arc>, - access: Arc, -} - -impl PageFrame { - fn new(pool: PagePool, permit: BudgetPermit, page: Box, epoch: u64) -> Self { - Self { - lease: RwLock::new(PageLease::new(pool, permit, page)).into(), - access: FrameAccess::new(epoch).into(), - } + pub fn page_id(&self) -> PageId { + self.frame.id } pub fn read(&self) -> PageFrameReadGuard { PageFrameReadGuard { - guard: RwLock::read_arc(&self.lease), + guard: self.frame.read(), + pin: self.frame.clone(), } } + pub fn save_or_get_content_hash(&self) -> blake3::Hash { + self.frame.write().save_or_get_content_hash() + } + fn write( - self: Arc, - slot: Arc>, + &self, + slot: Arc>, fixed_row_size: Size, eviction_policy: PageEvictionPolicy, ) -> PageFrameWriteGuard { PageFrameWriteGuard { - guard: RwLock::write_arc(&self.lease), - frame: self, + guard: self.frame.write(), + pin: self.frame.clone(), + slot, fixed_row_size, eviction_policy, } } - - fn touch(&self, epoch: u64) { - self.access.touch(epoch); - } } -struct PageLease { - page: ManuallyDrop>, - pool: PagePool, - #[allow(unused)] - permit: BudgetPermit, -} - -impl PageLease { - pub fn new(pool: PagePool, permit: BudgetPermit, page: Box) -> Self { - Self { - page: ManuallyDrop::new(page), - pool, - permit, - } - } -} - -impl Drop for PageLease { - fn drop(&mut self) { - // SAFETY: we're in `drop`, so won't be using `ManuallyDrop` again. - let page = unsafe { ManuallyDrop::take(&mut self.page) }; - self.pool.put(page); - } -} - -impl Deref for PageLease { - type Target = Page; - - fn deref(&self) -> &Self::Target { - &self.page - } -} - -impl DerefMut for PageLease { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.page - } -} +impl MemoryUsage for PageHandle { + fn heap_usage(&self) -> usize { + let refcount = std::mem::size_of::() * 2; + let buffer_heap_usage = self.frame.heap_usage(); -impl fmt::Debug for PageLease { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("PageLease") - .field("page", &self.page) - .field("pool", &"") - .field("permit", &self.permit) - .finish() - } -} - -#[derive(Debug)] -#[allow(unused)] -pub enum ResidentPageState { - Clean { hash: Option }, - Dirty { hash: Option }, -} - -impl ResidentPageState { - fn mark_dirty(&mut self, maybe_hash: Option) { - match self { - ResidentPageState::Clean { .. } => *self = ResidentPageState::Dirty { hash: maybe_hash }, - ResidentPageState::Dirty { hash } => { - maybe_hash.map(|new| hash.replace(new)); - } - } + refcount + buffer_heap_usage } } @@ -487,8 +370,7 @@ impl ResidentPageState { /// respectively [PageManager::reserve()]. A registered page can be registered /// with the page manager infallibly (see [PageManager::redeem()]). pub struct ReservedPage { - permit: BudgetPermit, - page: Box, + inner: mem::ReservedPage, metadata: PageMetadata, } @@ -499,29 +381,25 @@ pub struct ReservedPage { /// allocate. #[derive(Debug)] pub struct PageManager { - frames: RwLock, pool: PagePool, - store: Arc, - memory: ByteBudget, - access_epoch: AtomicU64, + resident: Arc, + storage: Arc, } impl PageManager { - pub fn new(pool: PagePool, store: Arc, memory: ByteBudget) -> Self { + pub fn new(resident: Arc, pool: PagePool, storage: Arc) -> Self { Self { - frames: <_>::default(), pool, - store, - memory, - access_epoch: <_>::default(), + resident, + storage, } } pub fn new_for_test() -> Self { Self::new( + ResidentObjects::new(ByteBudget::unlimited()).into(), PagePool::new_for_test(), - Arc::new(()), - ByteBudget::new(ByteBudgetConfig::unlimited()).unwrap(), + TieredStorage::new_for_test().into(), ) } @@ -533,7 +411,7 @@ impl PageManager { slot: &PageSlotHandle, eviction_policy: PageEvictionPolicy, ) -> Result, PageError> { - Ok(self.may_fault(slot, eviction_policy)?.map(|frame| PageHandle { frame })) + self.may_fault(slot, eviction_policy) } /// Get a [PageFrameWriteGuard] for the page at the given `slot`. @@ -550,18 +428,18 @@ impl PageManager { eviction_policy: PageEvictionPolicy, fixed_row_size: Size, ) -> Result { - let frame = self + let handle = self .may_fault(slot, eviction_policy)? .expect("page requested for mutation to be present"); - Ok(frame.write(slot.slot.clone(), fixed_row_size, eviction_policy)) + Ok(handle.write(slot.slot.clone(), fixed_row_size, eviction_policy)) } fn may_fault( &self, slot: &PageSlotHandle, eviction_policy: PageEvictionPolicy, - ) -> Result>, PageError> { - let mut slot_guard = slot.slot.lock().unwrap(); + ) -> Result, PageError> { + let mut slot_guard = slot.slot.write(); match *slot_guard { PageSlot::Absent => Ok(None), PageSlot::Resident { @@ -573,34 +451,32 @@ impl PageManager { cached_eviction_policy, eviction_policy, "eviction policy cannot be changed for resident pages" ); - let frame = handle.frame.clone(); // Avoid atomics contention if the page isn't evictable. if matches!(eviction_policy, PageEvictionPolicy::Evictable) { - frame.touch(self.access_epoch.fetch_add(1, Ordering::Relaxed)); + self.resident.touch_page(&handle.frame); } - Ok(Some(frame)) + Ok(Some(handle.clone())) } PageSlot::NonResident { hash, metadata, .. } => { - let permit = self.acquire_memory_budget_permit()?; - let page = self.store.load_page(hash)?; - let frame = self.frames.write().register( - self.pool.clone(), - permit, - page, - self.access_epoch.fetch_add(1, Ordering::Relaxed), - |frame| { - let registry_entry = FrameRegistryEntry::new(&frame, &slot.slot, eviction_policy); - *slot_guard = PageSlot::Resident { - handle: PageHandle { frame }, - metadata, - state: ResidentPageState::Clean { hash: Some(hash) }, - eviction_policy, - }; - registry_entry - }, - ); + let reservation = self.resident.reserve_page(|| { + let bytes = self.storage.read(hash.into())?; + let page = self + .pool + .take_deserialize_from(&bytes) + .map(|page| PooledPage::new(page, self.pool.clone()))?; + + Ok::<_, PageError>(page) + })?; + let frame = self.resident.redeem_page(reservation); + let handle = PageHandle { frame }; + *slot_guard = PageSlot::Resident { + handle: handle.clone(), + metadata, + backing: Some(hash), + eviction_policy, + }; - Ok(Some(frame)) + Ok(Some(handle)) } } } @@ -623,11 +499,19 @@ impl PageManager { /// The returned [ReservedPage] is not registered with the manager. To do /// so, call [Self::redeem()]. pub fn reserve(&self, fixed_row_size: Size) -> Result { - let permit = self.acquire_memory_budget_permit()?; - let page = self.pool.take_with_fixed_row_size(fixed_row_size); - let metadata = page.metadata(fixed_row_size); + let mut metadata = None; + let reservation = self.resident.reserve_page(|| { + let page = self.pool.take_with_fixed_row_size(fixed_row_size); + metadata = Some(page.metadata(fixed_row_size)); + let page = PooledPage::new(page, self.pool.clone()); + + Ok::<_, BudgetExceeded>(page) + })?; - Ok(ReservedPage { permit, page, metadata }) + Ok(ReservedPage { + inner: reservation, + metadata: metadata.unwrap(), + }) } /// Register a [ReservedPage] with the page manager. @@ -636,28 +520,19 @@ impl PageManager { /// manager instance. pub fn redeem( &self, - ReservedPage { permit, page, metadata }: ReservedPage, + ReservedPage { inner, metadata }: ReservedPage, eviction_policy: PageEvictionPolicy, ) -> PageSlotHandle { - let slot = Arc::new(Mutex::new(PageSlot::Absent)); + let slot = Arc::new(RwLock::new(PageSlot::Absent)); { - let mut slot_guard = slot.lock().unwrap(); - self.frames.write().register( - self.pool.clone(), - permit, - page, - self.access_epoch.fetch_add(1, Ordering::Relaxed), - |frame| { - let registry_entry = FrameRegistryEntry::new(&frame, &slot, eviction_policy); - *slot_guard = PageSlot::Resident { - handle: PageHandle { frame }, - state: ResidentPageState::Clean { hash: None }, - metadata, - eviction_policy, - }; - registry_entry - }, - ); + let mut slot_guard = slot.write(); + let frame = self.resident.redeem_page(inner); + *slot_guard = PageSlot::Resident { + handle: PageHandle { frame }, + metadata, + backing: None, + eviction_policy, + }; } PageSlotHandle { slot } @@ -668,126 +543,51 @@ impl PageManager { /// This is used when restoring from a snapshot, and **NOT** subject to the /// [ByteBudget] of this page manager. I.e., after this method returns, more /// bytes than the high watermark of the budget may be resident. - pub(super) fn register( + pub(super) fn register_unchecked( &self, eviction_policy: PageEvictionPolicy, fixed_row_size: Size, pages: impl IntoIterator>>, - ) -> Vec { - let mut handles = Vec::new(); + slots: &mut Vec, + ) { for page in pages { - let slot = Arc::new(Mutex::new(PageSlot::Absent)); + let slot = Arc::new(RwLock::new(PageSlot::Absent)); if let Some(page) = page { let metadata = page.metadata(fixed_row_size); - // SAFETY: We are restoring from a snapshot, so are allowed to - // go over budget. - let permit = unsafe { self.unchecked_acquire_memory_buget_permit() }; - let mut slot_gard = slot.lock().unwrap(); - self.frames.write().register( - self.pool.clone(), - permit, - page, - self.access_epoch.fetch_add(1, Ordering::Relaxed), - |frame| { - let registry_entry = FrameRegistryEntry::new(&frame, &slot, eviction_policy); - *slot_gard = PageSlot::Resident { - handle: PageHandle { frame }, - state: ResidentPageState::Clean { hash: None }, - metadata, - eviction_policy, - }; - registry_entry - }, - ); + let frame = self.resident.register_page_unchecked(self.pool.clone(), page); + let mut slot_guard = slot.write(); + *slot_guard = PageSlot::Resident { + handle: PageHandle { frame }, + metadata, + // NOTE: The page is in snapshot storage, but not + // necessarily in the local tier. + backing: None, + eviction_policy, + }; } - handles.push(PageSlotHandle { slot }); - } - - handles - } - - unsafe fn unchecked_acquire_memory_buget_permit(&self) -> BudgetPermit { - self.memory.force_acquire(PAGE_SIZE as _) - } - - fn acquire_memory_budget_permit(&self) -> Result { - // TODO: Try to evict pages if acquisition fails. - self.memory.acquire(PAGE_SIZE as _) - } -} - -/// Essentially the LRU cache for pages. -/// -/// This is currently a stub and doesn't ever evict pages. -#[derive(Debug, Default)] -struct FrameRegistry { - frames: Slab, -} - -impl FrameRegistry { - pub fn register( - &mut self, - pool: PagePool, - permit: BudgetPermit, - page: Box, - epoch: u64, - mk_entry: impl FnOnce(Arc) -> FrameRegistryEntry, - ) -> Arc { - let entry = self.frames.vacant_entry(); - let frame = Arc::new(PageFrame::new(pool, permit, page, epoch)); - entry.insert(mk_entry(frame.clone())); - frame - } -} - -/// Access counters for cache eviction. -/// -/// Shared between [FrameRegistryEntry] and [PageFrame], to avoid lock -/// contention on the [FrameRegistry] for updates. -#[derive(Debug)] -struct FrameAccess { - last_access_epoch: AtomicU64, - last_access_count: AtomicU64, -} - -impl FrameAccess { - fn new(epoch: u64) -> Self { - Self { - last_access_epoch: AtomicU64::new(epoch), - last_access_count: <_>::default(), + slots.push(PageSlotHandle { slot }); } } - fn touch(&self, epoch: u64) { - self.last_access_epoch.store(epoch, Ordering::Relaxed); - self.last_access_count.fetch_add(1, Ordering::Relaxed); + pub(super) fn evictable(&self, candidates: &mut Vec) { + self.resident.evictable_pages(candidates); } -} -impl MemoryUsage for FrameAccess { - fn heap_usage(&self) -> usize { - mem::size_of::() + pub(super) fn evicted(&self, victim: PageEvictionCandidate) { + self.resident.evict_page(victim); } -} -#[derive(Debug)] -#[allow(unused)] -pub struct FrameRegistryEntry { - frame: Weak, - slot: Weak>, - eviction_policy: PageEvictionPolicy, - access: Arc, -} + pub(super) fn ensure_lower_tier_stored(&self, page: &PageHandle) -> Result { + let mut guard = page.frame.write_ref(); + let hash = guard.save_or_get_content_hash(); + let guard = RwLockWriteGuard::downgrade(guard); + let bytes = bsatn::to_vec(&**guard).unwrap(); + drop(guard); + let key = self.storage.write(&bytes)?; + assert_eq!(ObjectKey::from(hash), key); -impl FrameRegistryEntry { - pub fn new(frame: &Arc, slot: &Arc>, eviction_policy: PageEvictionPolicy) -> Self { - Self { - access: Arc::clone(&frame.access), - frame: Arc::downgrade(frame), - slot: Arc::downgrade(slot), - eviction_policy, - } + Ok(hash) } } diff --git a/crates/table/src/tiered/page_set.rs b/crates/table/src/tiered/page_set.rs index 257cbcd91e3..6a93a2a5e64 100644 --- a/crates/table/src/tiered/page_set.rs +++ b/crates/table/src/tiered/page_set.rs @@ -1,23 +1,34 @@ use std::{ collections::{btree_map, BTreeMap, BTreeSet}, convert::Infallible, - sync::Arc, + sync::{Arc, Mutex}, }; +use spacetimedb_data_structures::map::IntMap; use spacetimedb_lib::ProductValue; use spacetimedb_memory_usage::MemoryUsage; use spacetimedb_sats::layout::Size; use crate::{ - blob_store::BlobStore, indexes::{PageIndex, RowPointer}, page::{Page, PageCapacity}, - table::{BlobNumBytes, PreparedInsert, Table}, - tiered::page_manager::{PageEvictionPolicy, PageHandle, PageManager, PageSlotHandle, ReservedPage}, + table::{BlobNumBytes, PreparedInsert, Table, TableError}, + tiered::{ + blob_set::BlobHandle, + mem::{PageEvictionCandidate, PageId}, + page_manager::{PageEvictionPolicy, PageHandle, PageManager, PageSlot, PageSlotHandle, ReservedPage}, + BlobError, BlobSet, PageError, + }, var_len::VarLenMembers, }; -pub use crate::tiered::page_manager::PageError; +#[derive(Debug, thiserror::Error)] +pub enum DeleteRowError { + #[error(transparent)] + Page(#[from] PageError), + #[error(transparent)] + Blob(#[from] BlobError), +} /// The set of pages of a [Table]. #[derive(Debug)] @@ -28,6 +39,9 @@ pub struct PageSet { free_page_slots: BTreeSet, non_full_pages: BTreeSet<(usize, PageIndex)>, pub(crate) manager: Arc, + + // Scratch allocations for eviction. + evictable: Mutex, } type EvictionScratch = (Box>, Box>); @@ -41,6 +55,8 @@ impl PageSet { free_page_slots: <_>::default(), non_full_pages: <_>::default(), manager, + + evictable: <_>::default(), } } @@ -53,7 +69,8 @@ impl PageSet { // This is ok, because the table is empty. self.fixed_row_size = fixed_row_size; - self.slots = self.manager.register(self.eviction_policy, self.fixed_row_size, pages); + self.manager + .register_unchecked(self.eviction_policy, self.fixed_row_size, pages, &mut self.slots); self.non_full_pages = self .slots .iter() @@ -185,8 +202,8 @@ impl PageSet { var_len_visitor: &impl VarLenMembers, fixed_row_size: Size, row_ptr: RowPointer, - blob_store: &mut dyn BlobStore, - ) -> Result { + blobs: &BlobSet, + ) -> Result { let page_index = row_ptr.page_index(); let (blob_bytes_deleted, is_empty, available_granules) = self.with_page_mut(page_index, fixed_row_size, |page| { @@ -197,14 +214,14 @@ impl PageSet { // - `fixed_row_size` is consistent with the size in bytes of the fixed part of the row. // The size is also conistent with `var_len_visitor`. let blob_bytes_deleted = - unsafe { page.delete_row(row_ptr.page_offset(), fixed_row_size, var_len_visitor, blob_store) }; + unsafe { page.delete_row(row_ptr.page_offset(), fixed_row_size, var_len_visitor, blobs) }?; - ( + Ok::<_, BlobError>(( blob_bytes_deleted, page.num_rows() == 0, page.available_var_len_granules(), - ) - })?; + )) + })??; if is_empty { self.slots[page_index.idx()].free(); @@ -283,7 +300,7 @@ impl PageSet { if idx == self.slots.len() { self.slots.push(slot); } else { - assert!(self.slots[idx].is_absent()); + assert!(self.slots.get(idx).unwrap().is_absent()); self.free_page_slots.remove(&index); self.slots[idx] = slot; } @@ -360,8 +377,8 @@ impl PageSet { fixed_row_size: Size, visitor: &impl VarLenMembers, deletes: impl IntoIterator, - inserts: impl IntoIterator>, - ) -> Result { + inserts: impl IntoIterator>, + ) -> Result { let mut allocator = PageAllocator::new(self, fixed_row_size); let mut pinned = BTreeMap::new(); @@ -379,7 +396,7 @@ impl PageSet { .row_total_granules(row_ptr.page_offset(), fixed_row_size, visitor) }; allocator.prepare_delete::(page_index, granules, |page_index| { - Ok(pinned[&page_index].read().capacity(fixed_row_size)) + Ok(pinned[&page_index].read().metadata(fixed_row_size).into()) }); Ok(row_ptr) @@ -395,14 +412,14 @@ impl PageSet { btree_map::Entry::Vacant(entry) => { let page = self.get_page(page_index)?.expect("delete from absent page"); let page = entry.insert(page); - Ok(page.read().capacity(fixed_row_size)) + Ok(page.read().metadata(fixed_row_size).into()) } - btree_map::Entry::Occupied(entry) => Ok(entry.get().read().capacity(fixed_row_size)), + btree_map::Entry::Occupied(entry) => Ok(entry.get().read().metadata(fixed_row_size).into()), } })?; Ok(PlannedInsert { page_index, row }) }) - .collect::>()?; + .collect::>()?; let reserved = allocator .pages_to_allocate() @@ -410,6 +427,7 @@ impl PageSet { self.manager .reserve(fixed_row_size) .map(|reservation| (page_index, reservation)) + .map_err(PageError::from) }) .collect::, _>>()?; @@ -418,8 +436,54 @@ impl PageSet { reserved, deletes, inserts, + pinned_blobs: Vec::new(), }) } + + pub fn evict(&self) -> Result<(), PageError> { + let mut _guard = self.evictable.lock().unwrap(); + let (candidates, slots) = &mut *_guard; + + // Collect eviction candidates, returning early if there aren't any. + self.manager.evictable(candidates); + if candidates.is_empty() { + return Ok(()); + } + + // Only resident slots have a page id, so collect them now. + slots.clear(); + slots.extend(self.slots.iter().enumerate().filter_map(|(idx, slot_handle)| { + let page_id = slot_handle.page_id()?; + Some((page_id, PageIndex(idx as _))) + })); + + for candidate in candidates.drain(..) { + let Some(page_index) = slots.remove(&candidate.id) else { + continue; + }; + let mut slot = self.slots[page_index.idx()].lock(); + match &*slot { + PageSlot::Resident { + handle: page_handle, + metadata, + backing, + eviction_policy: PageEvictionPolicy::Evictable, + } => { + let hash = backing + .map(Ok) + .unwrap_or_else(|| self.manager.ensure_lower_tier_stored(page_handle))?; + *slot = PageSlot::NonResident { + hash, + metadata: *metadata, + }; + self.manager.evicted(candidate); + } + _ => continue, + } + } + + Ok(()) + } } #[cfg(test)] @@ -484,6 +548,7 @@ impl MemoryUsage for PageSet { // TODO: The manager is shared by all page sets of a database, how // should we account for it? manager: _, + evictable: _, } = self; slots.heap_usage() + free_page_slots.heap_usage() + non_full_pages.heap_usage() @@ -503,6 +568,9 @@ pub struct PreparedCommit { deletes: Vec, /// The rows to be inserted in this transaction. inserts: Vec, + /// Blobs referenced by rows which will be deleted. Holding these handles + /// makes blob reads during apply infallible. + pinned_blobs: Vec, } impl PreparedCommit { @@ -510,7 +578,7 @@ impl PreparedCommit { /// /// The table's [PageSet] must be the same as the one the [PreparedCommit] /// was created from. - pub fn apply(mut self, table: &mut Table, blob_store: &mut dyn BlobStore) -> AppliedCommit { + pub fn apply(mut self, table: &mut Table, blobs: &BlobSet) -> AppliedCommit { let mut pinned = self.pinned; fn collect_arc_slice(iter: impl ExactSizeIterator, mut f: impl FnMut(T) -> U) -> Arc<[U]> { @@ -528,7 +596,7 @@ impl PreparedCommit { let deletes = collect_arc_slice(self.deletes.into_iter(), |row_ptr| { table - .delete(blob_store, row_ptr, |row| row.to_product_value()) + .delete(blobs, row_ptr, |row| row.to_product_value()) .expect("no page faults") .expect("`Table::delete` never returns `None`") }); @@ -540,7 +608,7 @@ impl PreparedCommit { if !schema.is_event { let reservation = self.reserved.remove(&page_index).map(|page| (page_index, page)); let row_ref = table - .insert_prepared(blob_store, &row, reservation) + .insert_prepared(blobs, &row, reservation) .map(|(_, row_ref)| row_ref) .expect("failed to insert during transaction commit"); let (page, _) = row_ref.page_and_offset(); @@ -555,9 +623,14 @@ impl PreparedCommit { }); drop(pinned); + drop(self.pinned_blobs); AppliedCommit { deletes, inserts } } + + pub(crate) fn pin_blobs(&mut self, blobs: impl IntoIterator) { + self.pinned_blobs.extend(blobs); + } } /// The deleted and inserted rows of a [PreparedCommit::apply] operation, as @@ -674,11 +747,11 @@ impl<'a> PageAllocator<'a> { .range((row_granules, PageIndex(0))..) // A planned page shadows committed state. .filter(|(_, index)| !self.planned.contains_key(index)) - .find_map(|&(_, index)| { + .find_map(|(_, index)| { self.pages.slots[index.idx()] .has_space_for_row(self.fixed_row_size, row_granules) .unwrap() - .then_some(index) + .then_some(*index) }); match (planned, committed) { diff --git a/crates/table/src/tiered/storage.rs b/crates/table/src/tiered/storage.rs new file mode 100644 index 00000000000..800a5421f0f --- /dev/null +++ b/crates/table/src/tiered/storage.rs @@ -0,0 +1,140 @@ +use std::fmt; + +use parking_lot::RwLock; + +use crate::tiered::{ + error::ObjectStoreError, + map::{ObjectKey, ObjectMap}, +}; + +pub type ObjectBytes = Box<[u8]>; + +#[derive(Clone, Copy, Debug)] +pub struct ObjectMetadata { + pub size: u64, +} + +impl From<&ObjectCatalogEntry> for ObjectMetadata { + fn from(ObjectCatalogEntry { size, .. }: &ObjectCatalogEntry) -> Self { + ObjectMetadata { size: *size } + } +} + +pub trait ObjectStore: fmt::Debug + Send + Sync + 'static { + /// Obtain metadata about the object at `key`, or `None` if no such key + /// exists. + fn stat(&self, key: ObjectKey) -> Result, ObjectStoreError>; + /// Read the raw bytes of the object at `key`, or `None` if no such key + /// exists. + /// + /// The implementation must verify that the object hashes to `key`, and + /// return [ObjectStoreError::HashMismatch] if it doesn't. + fn read(&self, key: ObjectKey) -> Result, ObjectStoreError>; + /// Store `bytes` and return an [ObjectKey] by which the object can be + /// retrieved later. + fn write(&self, bytes: &[u8]) -> Result; +} + +impl ObjectStore for () { + fn stat(&self, _: ObjectKey) -> Result, ObjectStoreError> { + unimplemented!("null object store does not support `stat`") + } + + fn read(&self, _: ObjectKey) -> Result, ObjectStoreError> { + unimplemented!("null object store does not support `read`") + } + + fn write(&self, _: &[u8]) -> Result { + unimplemented!("null object store does not support `store`") + } +} + +#[derive(Debug, Default)] +pub struct MemoryObjectStore { + objects: RwLock>, +} + +impl ObjectStore for MemoryObjectStore { + fn stat(&self, key: ObjectKey) -> Result, ObjectStoreError> { + Ok(self.objects.read().get(&key).map(|bytes| ObjectMetadata { + size: bytes.len() as u64, + })) + } + + fn read(&self, key: ObjectKey) -> Result, ObjectStoreError> { + Ok(self.objects.read().get(&key).cloned()) + } + + fn write(&self, bytes: &[u8]) -> Result { + let key = blake3::hash(bytes).into(); + self.objects.write().insert(key, bytes.into()); + Ok(key) + } +} + +#[derive(Debug)] +pub struct TieredStorage { + local: Box, + remote: Option>, + + catalog: RwLock>, + // TODO: Should this be here? [ObjectStore]s should be managing their + // budgets. + //local_budget: ByteBudget, +} + +impl TieredStorage { + pub fn new(local: Box, remote: Option>) -> Self { + Self { + local, + remote, + catalog: <_>::default(), + } + } + + pub fn new_for_test() -> Self { + Self::new(Box::new(MemoryObjectStore::default()), None) + } + + pub fn stat(&self, key: ObjectKey) -> Result { + self.catalog + .read() + .get(&key) + .map(Into::into) + .ok_or(ObjectStoreError::UnknownObject(key)) + } + + pub fn contains(&self, key: ObjectKey) -> bool { + self.catalog.read().contains_key(&key) + } + + pub fn write(&self, bytes: &[u8]) -> Result { + let size = bytes.len() as u64; + let key = self.local.write(bytes)?; + self.catalog.write().insert(key, ObjectCatalogEntry { size }); + Ok(key) + } + + pub fn read(&self, key: ObjectKey) -> Result { + if !self.catalog.read().contains_key(&key) { + return Err(ObjectStoreError::UnknownObject(key)); + } + + if let Some(object) = self.local.read(key)? { + return Ok(object); + } + + if let Some(remote) = &self.remote + && let Some(object) = remote.read(key)? + { + return Ok(object); + } + + Err(ObjectStoreError::MissingObject(key)) + } +} + +#[derive(Debug)] +struct ObjectCatalogEntry { + size: u64, +}