diff --git a/docs/src/format/table/index.md b/docs/src/format/table/index.md index a20d4c564c2..e8c96a15d9a 100644 --- a/docs/src/format/table/index.md +++ b/docs/src/format/table/index.md @@ -157,6 +157,22 @@ or independently of column indices due to variable encoding widths (for Lance fi See the [5.0.0 migration guide](../../guide/migration.md#500) for a detailed example. +### File Metadata Size Hints + +`DataFile.file_metadata_size_bytes` and `IndexFile.file_metadata_size_bytes` +record the size in bytes of a file's metadata suffix, or 0 if unknown. For a +Lance file of format version 2.0 or later, this is the file size minus the +position of global buffer 0, which holds the file's `FileDescriptor`. That +position is stored in the first entry of the global buffer offset table, which +the footer locates; see the [file layout](../file/index.md#detailed-overview). +It is always 0 for legacy (0.1) files and for index files in other formats. +Writers record either this exact value or 0. + +The hint is advisory: it lets a reader fetch the last +`file_metadata_size_bytes` bytes of the file before decoding the footer. The +footer stays authoritative. Readers must ignore a value larger than the file +and read any metadata the footer references outside the hinted range. + ## Deletion Files Deletion files (a.k.a. deletion vectors) track deleted rows without rewriting data files. diff --git a/java/lance-jni/src/fragment.rs b/java/lance-jni/src/fragment.rs index 09ea3416e77..39e1a759640 100644 --- a/java/lance-jni/src/fragment.rs +++ b/java/lance-jni/src/fragment.rs @@ -1101,6 +1101,7 @@ impl FromJObjectWithEnv for JObject<'_> { file_major_version, file_minor_version, file_size_bytes, + file_metadata_size_bytes: None, base_id, }) } diff --git a/java/lance-jni/src/transaction.rs b/java/lance-jni/src/transaction.rs index e1ca961af72..2823f6c29d4 100644 --- a/java/lance-jni/src/transaction.rs +++ b/java/lance-jni/src/transaction.rs @@ -144,6 +144,7 @@ impl FromJObjectWithEnv for JObject<'_> { "newIndexFiles.sizeBytes", env.call_method(self, "getSizeBytes", "()J", &[])?.j()?, )?, + file_metadata_size_bytes: None, }) } } diff --git a/protos/fragment_metadata.proto b/protos/fragment_metadata.proto index 63e2651d1e8..b39bed6802c 100644 --- a/protos/fragment_metadata.proto +++ b/protos/fragment_metadata.proto @@ -180,6 +180,21 @@ message DataFile { * Lance use it as key of the base_paths field in Manifest to determine the actual base path of the data file. */ optional uint32 base_id = 7; + + /* Size in bytes of the file's metadata suffix, or 0 if unknown. + * + * For a Lance file of format version 2.0 or later, this is the file size minus + * the position of global buffer 0, which holds the file's `FileDescriptor`. + * That position is stored in the first entry of the global buffer offset + * table, which the footer locates. It is always 0 for legacy (0.1) files. + * Writers record either this exact value or 0. + * + * This advisory hint lets a reader fetch the last `file_metadata_size_bytes` + * bytes of the file before decoding the footer. The footer stays + * authoritative: readers must ignore a value larger than the file and read + * any metadata the footer references outside the hinted range. + */ + uint64 file_metadata_size_bytes = 8; } // DataFile /* An overlay file supplies new values for a subset of (row offset, field) cells diff --git a/protos/table.proto b/protos/table.proto index 1d0febbf215..a60a4bd35ca 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -407,6 +407,20 @@ message IndexFile { string path = 1; // Size of the file in bytes uint64 size_bytes = 2; + /* Size in bytes of the file's metadata suffix, or 0 if unknown. + * + * For a Lance file of format version 2.0 or later, this is the file size minus + * the position of global buffer 0, which holds the file's `FileDescriptor`. + * That position is stored in the first entry of the global buffer offset + * table, which the footer locates. It is always 0 for legacy (0.1) Lance files + * and for files in other formats. Writers record either this exact value or 0. + * + * This advisory hint lets a reader fetch the last `file_metadata_size_bytes` + * bytes of the file before decoding the footer. The footer stays + * authoritative: readers must ignore a value larger than the file and read + * any metadata the footer references outside the hinted range. + */ + uint64 file_metadata_size_bytes = 3; } // Index Section, containing a list of index metadata for one dataset version. diff --git a/python/python/lance/fragment.py b/python/python/lance/fragment.py index 1bc9c5e5d67..e45cd0455fb 100644 --- a/python/python/lance/fragment.py +++ b/python/python/lance/fragment.py @@ -249,6 +249,8 @@ class DataFile: The minor version of the data storage format. file_size_bytes : Optional[int] The size of the data file in bytes, if available. + file_metadata_size_bytes : Optional[int] + The size of the data file's metadata suffix in bytes, if available. """ _path: str @@ -258,6 +260,7 @@ class DataFile: file_minor_version: int = 0 file_size_bytes: Optional[int] = None base_id: Optional[int] = None + file_metadata_size_bytes: Optional[int] = None def __init__( self, @@ -268,6 +271,7 @@ def __init__( file_minor_version: int = 0, file_size_bytes: Optional[int] = None, base_id: Optional[int] = None, + file_metadata_size_bytes: Optional[int] = None, ): # TODO: only we eliminate the path method, we can remove this self._path = path @@ -277,6 +281,7 @@ def __init__( self.file_minor_version = file_minor_version self.file_size_bytes = file_size_bytes self.base_id = base_id + self.file_metadata_size_bytes = file_metadata_size_bytes def __repr__(self): # pretend we have a 'path' attribute diff --git a/python/python/tests/test_fragment.py b/python/python/tests/test_fragment.py index 8b4a317e8db..b8c1e596711 100644 --- a/python/python/tests/test_fragment.py +++ b/python/python/tests/test_fragment.py @@ -328,7 +328,12 @@ def test_fragment_meta(): data = { "id": 0, "files": [ - {"path": "0.lance", "fields": [0], "file_size_bytes": 100}, + { + "path": "0.lance", + "fields": [0], + "file_size_bytes": 100, + "file_metadata_size_bytes": 25, + }, {"path": "1.lance", "fields": [1]}, ], "deletion_file": None, @@ -340,7 +345,9 @@ def test_fragment_meta(): assert len(meta.files) == 2 with pytest.warns(DeprecationWarning): assert meta.files[0].path() == "0.lance" + assert meta.files[0].file_metadata_size_bytes == 25 assert meta.files[1].path == "1.lance" + assert meta.files[1].file_metadata_size_bytes is None assert repr(meta) == ( "FragmentMetadata(id=0, files=[DataFile(path='0.lance', fields=[0], " diff --git a/python/src/fragment.rs b/python/src/fragment.rs index 60f18b6d63e..6a8fceea738 100644 --- a/python/src/fragment.rs +++ b/python/src/fragment.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::fmt::Write as _; +use std::num::NonZero; use std::sync::Arc; use arrow::ffi_stream::ArrowArrayStreamReader; @@ -963,6 +964,8 @@ impl FromPyObject<'_, '_> for PyLance { fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { let file_size_bytes: Option = ob.getattr("file_size_bytes")?.extract()?; let file_size_bytes = CachedFileSize::new(file_size_bytes.unwrap_or(0)); + let file_metadata_size_bytes: Option = + ob.getattr("file_metadata_size_bytes")?.extract()?; let fields: Vec = ob.getattr("fields")?.extract()?; let column_indices: Vec = ob.getattr("column_indices")?.extract()?; Ok(Self(DataFile { @@ -972,6 +975,7 @@ impl FromPyObject<'_, '_> for PyLance { file_major_version: ob.getattr("file_major_version")?.extract()?, file_minor_version: ob.getattr("file_minor_version")?.extract()?, file_size_bytes, + file_metadata_size_bytes: file_metadata_size_bytes.and_then(NonZero::new), base_id: ob.getattr("base_id")?.extract()?, })) } @@ -997,6 +1001,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataFile> { self.0.file_minor_version, file_size_bytes, self.0.base_id, + self.0.file_metadata_size_bytes.map(NonZero::get), )) } } diff --git a/python/src/transaction.rs b/python/src/transaction.rs index b3f2311b3f4..0af51820911 100644 --- a/python/src/transaction.rs +++ b/python/src/transaction.rs @@ -28,7 +28,11 @@ impl FromPyObject<'_, '_> for PyLance { fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { let path = ob.getattr("path")?.extract()?; let size_bytes = ob.getattr("size_bytes")?.extract()?; - Ok(Self(IndexFile { path, size_bytes })) + Ok(Self(IndexFile { + path, + size_bytes, + file_metadata_size_bytes: None, + })) } } diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs index 018709e22ec..71d28accf7c 100644 --- a/rust/lance-index/src/scalar.rs +++ b/rust/lance-index/src/scalar.rs @@ -60,8 +60,8 @@ pub use inverted::tokenizer::InvertedIndexParams; /// Convert a `Vec<`[`lance_index_core::scalar::IndexFile`]`>` to a /// `Vec<`[`lance_table::format::IndexFile`]`>`. /// -/// These two structs have identical fields; this helper bridges the crate -/// boundary without relying on orphan-rule–violating `From` impls. +/// These structs share their path and total size fields; this helper bridges the +/// crate boundary without relying on orphan-rule–violating `From` impls. pub fn index_files_to_table( files: Vec, ) -> Vec { @@ -70,6 +70,7 @@ pub fn index_files_to_table( .map(|f| lance_table::format::IndexFile { path: f.path, size_bytes: f.size_bytes, + file_metadata_size_bytes: None, }) .collect() } @@ -77,8 +78,8 @@ pub fn index_files_to_table( /// Convert a `Vec<`[`lance_table::format::IndexFile`]`>` to a /// `Vec<`[`lance_index_core::scalar::IndexFile`]`>`. /// -/// These two structs have identical fields; this helper bridges the crate -/// boundary without relying on orphan-rule–violating `From` impls. +/// These structs share their path and total size fields; this helper bridges the +/// crate boundary without relying on orphan-rule–violating `From` impls. pub fn table_files_to_index( files: Vec, ) -> Vec { diff --git a/rust/lance-index/src/vector/distributed/index_merger.rs b/rust/lance-index/src/vector/distributed/index_merger.rs index 370011eb8bd..4081ee10cb4 100755 --- a/rust/lance-index/src/vector/distributed/index_merger.rs +++ b/rust/lance-index/src/vector/distributed/index_merger.rs @@ -1724,6 +1724,7 @@ async fn merge_partial_vector_auxiliary_files_inner( Ok(lance_table::format::IndexFile { path: INDEX_AUXILIARY_FILE_NAME.to_string(), size_bytes: summary.size_bytes, + file_metadata_size_bytes: None, }) } else { Err(Error::index( diff --git a/rust/lance-table/benches/manifest_intern.rs b/rust/lance-table/benches/manifest_intern.rs index 81bd57c1a22..2036d3bf325 100644 --- a/rust/lance-table/benches/manifest_intern.rs +++ b/rust/lance-table/benches/manifest_intern.rs @@ -57,6 +57,7 @@ fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec file_major_version: 2, file_minor_version: 0, file_size_bytes: 0, + file_metadata_size_bytes: 0, base_id: None, }], overlays: vec![], @@ -134,6 +135,7 @@ fn make_diverse_pb_fragments( file_major_version: 2, file_minor_version: 0, file_size_bytes: 0, + file_metadata_size_bytes: 0, base_id: None, }], overlays: vec![], diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs index b2fd3a850c6..48ca0dff7cc 100644 --- a/rust/lance-table/src/format/fragment.rs +++ b/rust/lance-table/src/format/fragment.rs @@ -28,7 +28,7 @@ use lance_core::error::Result; /// Lance Data File /// /// A data file is one piece of file storing data. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct DataFile { /// Relative path of the data file to dataset root. pub path: String, @@ -56,21 +56,38 @@ pub struct DataFile { /// The size of the file in bytes, if known. pub file_size_bytes: CachedFileSize, + /// The metadata suffix size in bytes, if known. + /// + /// The suffix starts at global buffer 0, which holds the `FileDescriptor`, + /// and ends at EOF. Readers treat this as an advisory hint and use the file + /// footer as authoritative. `None` preserves the footer-directed open path + /// used for older manifests. + pub file_metadata_size_bytes: Option>, + /// The base path of the datafile, when the datafile is outside the dataset. pub base_id: Option, } +impl DeepSizeOf for DataFile { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.path.deep_size_of_children(context) + + self.fields.deep_size_of_children(context) + + self.column_indices.deep_size_of_children(context) + } +} + // Custom Serialize: convert Arc<[i32]> to slice for transparent JSON output impl Serialize for DataFile { fn serialize(&self, serializer: S) -> std::result::Result { use serde::ser::SerializeStruct; - let mut s = serializer.serialize_struct("DataFile", 7)?; + let mut s = serializer.serialize_struct("DataFile", 8)?; s.serialize_field("path", &self.path)?; s.serialize_field("fields", self.fields.as_ref())?; s.serialize_field("column_indices", self.column_indices.as_ref())?; s.serialize_field("file_major_version", &self.file_major_version)?; s.serialize_field("file_minor_version", &self.file_minor_version)?; s.serialize_field("file_size_bytes", &self.file_size_bytes)?; + s.serialize_field("file_metadata_size_bytes", &self.file_metadata_size_bytes)?; s.serialize_field("base_id", &self.base_id)?; s.end() } @@ -90,6 +107,7 @@ impl<'de> Deserialize<'de> for DataFile { #[serde(default)] file_minor_version: u32, file_size_bytes: CachedFileSize, + file_metadata_size_bytes: Option>, base_id: Option, } @@ -101,6 +119,7 @@ impl<'de> Deserialize<'de> for DataFile { file_major_version: helper.file_major_version, file_minor_version: helper.file_minor_version, file_size_bytes: helper.file_size_bytes, + file_metadata_size_bytes: helper.file_metadata_size_bytes, base_id: helper.base_id, }) } @@ -124,6 +143,7 @@ impl DataFile { file_major_version, file_minor_version, file_size_bytes: file_size_bytes.into(), + file_metadata_size_bytes: None, base_id, } } @@ -138,6 +158,7 @@ impl DataFile { file_major_version, file_minor_version, file_size_bytes: Default::default(), + file_metadata_size_bytes: None, base_id: None, } } @@ -289,6 +310,7 @@ impl From<&DataFile> for pb::DataFile { file_minor_version: df.file_minor_version, file_size_bytes: df.file_size_bytes.get().map_or(0, |v| v.get()), base_id: df.base_id, + file_metadata_size_bytes: df.file_metadata_size_bytes.map_or(0, NonZero::get), } } } @@ -305,6 +327,7 @@ impl TryFrom for DataFile { file_minor_version: proto.file_minor_version, file_size_bytes: CachedFileSize::new(proto.file_size_bytes), base_id: proto.base_id, + file_metadata_size_bytes: NonZero::new(proto.file_metadata_size_bytes), }) } } @@ -418,6 +441,7 @@ impl DataFileFieldInterner { file_minor_version: proto.file_minor_version, file_size_bytes: CachedFileSize::new(proto.file_size_bytes), base_id: proto.base_id, + file_metadata_size_bytes: NonZero::new(proto.file_metadata_size_bytes), }) } @@ -918,6 +942,7 @@ mod tests { use lance_file::format::{MAJOR_VERSION, MINOR_VERSION}; use object_store::path::Path; use roaring::RoaringBitmap; + use rstest::rstest; use serde_json::{Value, json}; #[test] @@ -1048,6 +1073,7 @@ mod tests { let mut fragment = Fragment::new(123); let schema = ArrowSchema::new(vec![ArrowField::new("x", DataType::Float16, true)]); fragment.add_file_legacy("foobar.lance", &Schema::try_from(&schema).unwrap()); + fragment.files[0].file_metadata_size_bytes = NonZero::new(1_024); fragment.deletion_file = Some(DeletionFile { read_version: 123, id: 456, @@ -1197,11 +1223,38 @@ mod tests { assert_eq!(current_file.schema(&schema), schema); } - #[test] - fn test_to_json() { + #[rstest] + #[case::present(1_024, NonZero::new(1_024))] + #[case::absent(0, None)] + fn test_intern_fragment_file_metadata_size_bytes( + #[case] encoded: u64, + #[case] expected: Option>, + ) { + let proto = pb::DataFragment { + files: vec![pb::DataFile { + path: "data.lance".to_string(), + fields: vec![0], + file_metadata_size_bytes: encoded, + ..Default::default() + }], + ..Default::default() + }; + + let interned = DataFileFieldInterner::default() + .intern_fragment(proto.clone()) + .unwrap(); + assert_eq!(interned.files[0].file_metadata_size_bytes, expected); + assert_eq!(interned, Fragment::try_from(proto).unwrap()); + } + + #[rstest] + #[case::present(NonZero::new(1_024))] + #[case::absent(None)] + fn test_to_json(#[case] file_metadata_size_bytes: Option>) { let mut fragment = Fragment::new(123); let schema = ArrowSchema::new(vec![ArrowField::new("x", DataType::Float16, true)]); fragment.add_file_legacy("foobar.lance", &Schema::try_from(&schema).unwrap()); + fragment.files[0].file_metadata_size_bytes = file_metadata_size_bytes; fragment.deletion_file = Some(DeletionFile { read_version: 123, id: 456, @@ -1212,7 +1265,7 @@ mod tests { let json = serde_json::to_string(&fragment).unwrap(); - let value: Value = serde_json::from_str(&json).unwrap(); + let mut value: Value = serde_json::from_str(&json).unwrap(); assert_eq!( value, json!({ @@ -1220,7 +1273,8 @@ mod tests { "files":[ {"path": "foobar.lance", "fields": [0], "column_indices": [], "file_major_version": MAJOR_VERSION, "file_minor_version": MINOR_VERSION, - "file_size_bytes": null, "base_id": null } + "file_size_bytes": null, "file_metadata_size_bytes": file_metadata_size_bytes, + "base_id": null } ], "deletion_file": {"read_version": 123, "id": 456, "file_type": "array", "num_deleted_rows": 10, "base_id": null}, @@ -1229,6 +1283,14 @@ mod tests { let frag2 = Fragment::from_json(&json).unwrap(); assert_eq!(fragment, frag2); + + // JSON written before the hint existed has no key for it. + value["files"][0] + .as_object_mut() + .unwrap() + .remove("file_metadata_size_bytes"); + let frag3: Fragment = serde_json::from_value(value).unwrap(); + assert_eq!(frag3.files[0].file_metadata_size_bytes, None); } #[test] @@ -1241,6 +1303,7 @@ mod tests { file_major_version: MAJOR_VERSION as u32, file_minor_version: MINOR_VERSION as u32, file_size_bytes: Default::default(), + file_metadata_size_bytes: None, base_id: None, }; diff --git a/rust/lance-table/src/format/index.rs b/rust/lance-table/src/format/index.rs index 74303ab247b..e554f3415a3 100644 --- a/rust/lance-table/src/format/index.rs +++ b/rust/lance-table/src/format/index.rs @@ -4,6 +4,7 @@ //! Metadata for index use std::collections::{HashMap, HashSet}; +use std::num::NonZero; use std::sync::Arc; use chrono::{DateTime, Utc}; @@ -19,12 +20,25 @@ use lance_core::cache::{CacheEntryReader, CacheEntryWriter}; use lance_core::{Error, Result}; /// Metadata about a single file within an index segment. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +#[derive(Debug, Clone, PartialEq)] pub struct IndexFile { /// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx") pub path: String, /// Size of the file in bytes pub size_bytes: u64, + /// The metadata suffix size in bytes, if known. + /// + /// The suffix starts at global buffer 0, which holds the `FileDescriptor`, + /// and ends at EOF. Readers treat this as an advisory hint and use the file + /// footer as authoritative. `None` also represents legacy (0.1) Lance files + /// and index files that use another format. + pub file_metadata_size_bytes: Option>, +} + +impl DeepSizeOf for IndexFile { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.path.deep_size_of_children(context) + } } /// Index metadata @@ -284,6 +298,7 @@ impl TryFrom for IndexMetadata { .map(|f| IndexFile { path: f.path, size_bytes: f.size_bytes, + file_metadata_size_bytes: NonZero::new(f.file_metadata_size_bytes), }) .collect(), ) @@ -348,6 +363,9 @@ impl From<&IndexMetadata> for pb::IndexMetadata { .map(|f| pb::IndexFile { path: f.path.clone(), size_bytes: f.size_bytes, + file_metadata_size_bytes: f + .file_metadata_size_bytes + .map_or(0, NonZero::get), }) .collect() }) @@ -440,6 +458,7 @@ pub async fn list_index_files_with_sizes( files.push(IndexFile { path: relative_path, size_bytes: meta.size, + file_metadata_size_bytes: None, }); } Ok(files) @@ -505,6 +524,7 @@ mod tests { files: Some(vec![IndexFile { path: "index.idx".to_string(), size_bytes: 1024, + file_metadata_size_bytes: NonZero::new(256), }]), }, IndexMetadata { diff --git a/rust/lance-table/src/transaction/manifest_build.rs b/rust/lance-table/src/transaction/manifest_build.rs index 5c262965304..51f6b03bab7 100644 --- a/rust/lance-table/src/transaction/manifest_build.rs +++ b/rust/lance-table/src/transaction/manifest_build.rs @@ -3722,6 +3722,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::new(1000), + file_metadata_size_bytes: None, base_id: None, }); @@ -3733,6 +3734,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::new(500), + file_metadata_size_bytes: None, base_id: None, }); @@ -3744,6 +3746,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::new(750), + file_metadata_size_bytes: None, base_id: None, }); @@ -3755,6 +3758,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::new(250), + file_metadata_size_bytes: None, base_id: None, }); diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index 9f8159bc062..4b48673b432 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -22,6 +22,7 @@ use lance_core::{Error, Result}; use lance_file::datatypes::Fields; use roaring::RoaringBitmap; use std::collections::HashMap; +use std::num::NonZero; use std::sync::Arc; use uuid::Uuid; @@ -460,6 +461,7 @@ impl TryFrom<&pb::transaction::rewrite::RewrittenIndex> for RewrittenIndex { .map(|f| IndexFile { path: f.path.clone(), size_bytes: f.size_bytes, + file_metadata_size_bytes: NonZero::new(f.file_metadata_size_bytes), }) .collect(), ) @@ -746,6 +748,9 @@ impl From<&RewrittenIndex> for pb::transaction::rewrite::RewrittenIndex { .map(|f| pb::IndexFile { path: f.path.clone(), size_bytes: f.size_bytes, + file_metadata_size_bytes: f + .file_metadata_size_bytes + .map_or(0, NonZero::get), }) .collect() }) @@ -891,4 +896,29 @@ mod tests { other => panic!("expected DataOverlay, got {other:?}"), } } + + #[test] + fn test_rewritten_index_roundtrips() { + let rewritten_index = RewrittenIndex { + old_id: Uuid::new_v4(), + new_id: Uuid::new_v4(), + new_index_details: prost_types::Any::default(), + new_index_version: 1, + new_index_files: Some(vec![ + IndexFile { + path: "index.idx".to_string(), + size_bytes: 4_096, + file_metadata_size_bytes: NonZero::new(256), + }, + IndexFile { + path: "auxiliary.idx".to_string(), + size_bytes: 1_024, + file_metadata_size_bytes: None, + }, + ]), + }; + + let message = pb::transaction::rewrite::RewrittenIndex::from(&rewritten_index); + assert_eq!(RewrittenIndex::try_from(&message).unwrap(), rewritten_index); + } } diff --git a/rust/lance/src/dataset/files.rs b/rust/lance/src/dataset/files.rs index cd9c05d46c7..2e4ea70f434 100644 --- a/rust/lance/src/dataset/files.rs +++ b/rust/lance/src/dataset/files.rs @@ -903,6 +903,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id, }; diff --git a/rust/lance/src/dataset/fragment.rs b/rust/lance/src/dataset/fragment.rs index 4231d4f7928..377b3809cc1 100644 --- a/rust/lance/src/dataset/fragment.rs +++ b/rust/lance/src/dataset/fragment.rs @@ -7449,6 +7449,7 @@ mod tests { file_major_version: 2, file_minor_version: 1, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; diff --git a/rust/lance/src/dataset/tests/dataset_merge_update.rs b/rust/lance/src/dataset/tests/dataset_merge_update.rs index bb310f373b7..2706797357a 100644 --- a/rust/lance/src/dataset/tests/dataset_merge_update.rs +++ b/rust/lance/src/dataset/tests/dataset_merge_update.rs @@ -883,6 +883,7 @@ async fn test_datafile_partial_replacement() { file_major_version: major, file_minor_version: minor, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; @@ -944,6 +945,7 @@ async fn test_datafile_partial_replacement() { file_major_version: major, file_minor_version: minor, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; @@ -1044,6 +1046,7 @@ async fn test_datafile_replacement_error() { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; @@ -3266,6 +3269,7 @@ async fn test_cross_column_fast_search_blocks_column_local_stale_postings() { file_major_version, file_minor_version, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; let read_version = dataset.version().version; diff --git a/rust/lance/src/dataset/write/commit.rs b/rust/lance/src/dataset/write/commit.rs index 4af9a12fe93..ea439279c78 100644 --- a/rust/lance/src/dataset/write/commit.rs +++ b/rust/lance/src/dataset/write/commit.rs @@ -656,6 +656,7 @@ mod tests { file_major_version: major_version, file_minor_version: minor_version, file_size_bytes: CachedFileSize::new(100), + file_metadata_size_bytes: None, base_id: None, }], overlays: vec![], diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index 2c3e470155b..5f57ef1b0af 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -4770,6 +4770,7 @@ mod tests { files: Some(vec![lance_table::format::IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes: payload.len() as u64, + file_metadata_size_bytes: None, }]), } } diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs index 88122af00a3..4f5aa62c156 100644 --- a/rust/lance/src/index/vector/builder.rs +++ b/rust/lance/src/index/vector/builder.rs @@ -1961,10 +1961,12 @@ impl IvfIndexBuilder IndexFile { path: INDEX_AUXILIARY_FILE_NAME.to_string(), size_bytes: storage_summary.size_bytes, + file_metadata_size_bytes: None, }, IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes: index_summary.size_bytes, + file_metadata_size_bytes: None, }, ]) } diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs index a7b5aa3cd40..f24d2b4e220 100644 --- a/rust/lance/src/index/vector/ivf.rs +++ b/rust/lance/src/index/vector/ivf.rs @@ -1112,6 +1112,7 @@ async fn optimize_ivf_pq_indices( IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes, + file_metadata_size_bytes: None, }, )) } @@ -1271,10 +1272,12 @@ async fn optimize_ivf_hnsw_indices( IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes: index_size, + file_metadata_size_bytes: None, }, IndexFile { path: INDEX_AUXILIARY_FILE_NAME.to_string(), size_bytes: aux_size, + file_metadata_size_bytes: None, }, ], )) @@ -2292,6 +2295,7 @@ pub(crate) async fn remap_index_file( Ok(IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes, + file_metadata_size_bytes: None, }) } @@ -2360,6 +2364,7 @@ async fn write_ivf_pq_file( Ok(IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes, + file_metadata_size_bytes: None, }) } @@ -2908,6 +2913,7 @@ async fn write_root_vector_index_from_auxiliary( Ok(IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes: summary.size_bytes, + file_metadata_size_bytes: None, }) }