From 41b5c4e0cdd62b72f45904c88eef5eff081f431b Mon Sep 17 00:00:00 2001 From: Ali Arslan Date: Fri, 11 Sep 2026 14:06:20 -0700 Subject: [PATCH 1/3] feat(format): add file metadata size hint Co-authored-by: Cursor --- docs/src/format/table/index.md | 14 ++++++++ protos/table.proto | 17 ++++++++++ rust/lance-index/src/scalar.rs | 9 ++--- .../src/vector/distributed/index_merger.rs | 1 + rust/lance-table/benches/manifest_intern.rs | 2 ++ rust/lance-table/src/format/fragment.rs | 33 +++++++++++++++++-- rust/lance-table/src/format/index.rs | 21 +++++++++++- .../src/transaction/manifest_build.rs | 4 +++ rust/lance-table/src/transaction/proto.rs | 5 +++ rust/lance/src/dataset/files.rs | 1 + rust/lance/src/dataset/fragment.rs | 1 + .../src/dataset/tests/dataset_merge_update.rs | 4 +++ rust/lance/src/dataset/write/commit.rs | 1 + rust/lance/src/index.rs | 1 + rust/lance/src/index/vector/builder.rs | 2 ++ rust/lance/src/index/vector/ivf.rs | 6 ++++ 16 files changed, 115 insertions(+), 7 deletions(-) diff --git a/docs/src/format/table/index.md b/docs/src/format/table/index.md index f9da132cf3b..162ac58d2ab 100644 --- a/docs/src/format/table/index.md +++ b/docs/src/format/table/index.md @@ -135,6 +135,20 @@ or independently of column indices due to variable encoding widths (for Lance fi +### File Metadata Size Hints + +`DataFile.file_metadata_size_bytes` records the suffix length from the Lance +schema descriptor through the end of the data file. `IndexFile` records the +same value for Lance-format files in an index segment; it is zero for other +file formats. Combined with the corresponding total file size, a reader can +request this suffix before decoding the footer and remove a dependent metadata +request. + +The value is advisory. Zero means unavailable, values larger than the file are +invalid, and the offsets and counts decoded from the file footer remain +authoritative. A reader must preserve the ordinary footer-directed fallback +for missing, invalid, underestimated, or otherwise inconsistent hints. + !!! note "Field-to-column mapping differs between data storage versions" In **2.0**, all fields (including non-leaf fields like struct and list containers) are assigned diff --git a/protos/table.proto b/protos/table.proto index f110b471c82..aac37deb220 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -361,6 +361,14 @@ message IndexFile { string path = 1; // Size of the file in bytes uint64 size_bytes = 2; + /* Number of bytes from the Lance schema descriptor through the end of the file. + * + * A reader that knows `size_bytes` may use this advisory value to issue the + * complete metadata-suffix read without first decoding the footer. The footer + * remains authoritative, and readers must fall back when this value is zero or + * invalid. Zero also covers files that do not use the Lance file format. + */ + uint64 file_metadata_size_bytes = 3; } // Index Section, containing a list of index metadata for one dataset version. @@ -520,6 +528,15 @@ message DataFile { * Lance use it as key of the base_paths field in Manifest to determine the actual base path of the data file. */ optional uint32 base_id = 7; + + /* Number of bytes from the Lance schema descriptor through the end of the file. + * + * A reader that knows `file_size_bytes` may use this advisory value to issue + * the complete metadata-suffix read without first decoding the footer. The + * footer remains authoritative, and readers must fall back when this value is + * zero or invalid. + */ + uint64 file_metadata_size_bytes = 8; } // DataFile /* An overlay file supplies new values for a subset of (row offset, field) cells diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs index 6dcfc9018e0..696c1123843 100644 --- a/rust/lance-index/src/scalar.rs +++ b/rust/lance-index/src/scalar.rs @@ -52,8 +52,8 @@ pub use inverted::tokenizer::InvertedIndexParams; /// Convert a `Vec<`[`lance_index_core::scalar::IndexFile`]`>` to a /// `Vec<`[`lance_table::format::IndexFile`]`>`. /// -/// These two structs have identical fields; this helper bridges the crate -/// boundary without relying on orphan-rule–violating `From` impls. +/// These structs share their path and total size fields; this helper bridges the +/// crate boundary without relying on orphan-rule–violating `From` impls. pub fn index_files_to_table( files: Vec, ) -> Vec { @@ -62,6 +62,7 @@ pub fn index_files_to_table( .map(|f| lance_table::format::IndexFile { path: f.path, size_bytes: f.size_bytes, + file_metadata_size_bytes: None, }) .collect() } @@ -69,8 +70,8 @@ pub fn index_files_to_table( /// Convert a `Vec<`[`lance_table::format::IndexFile`]`>` to a /// `Vec<`[`lance_index_core::scalar::IndexFile`]`>`. /// -/// These two structs have identical fields; this helper bridges the crate -/// boundary without relying on orphan-rule–violating `From` impls. +/// These structs share their path and total size fields; this helper bridges the +/// crate boundary without relying on orphan-rule–violating `From` impls. pub fn table_files_to_index( files: Vec, ) -> Vec { diff --git a/rust/lance-index/src/vector/distributed/index_merger.rs b/rust/lance-index/src/vector/distributed/index_merger.rs index 370011eb8bd..4081ee10cb4 100755 --- a/rust/lance-index/src/vector/distributed/index_merger.rs +++ b/rust/lance-index/src/vector/distributed/index_merger.rs @@ -1724,6 +1724,7 @@ async fn merge_partial_vector_auxiliary_files_inner( Ok(lance_table::format::IndexFile { path: INDEX_AUXILIARY_FILE_NAME.to_string(), size_bytes: summary.size_bytes, + file_metadata_size_bytes: None, }) } else { Err(Error::index( diff --git a/rust/lance-table/benches/manifest_intern.rs b/rust/lance-table/benches/manifest_intern.rs index 81bd57c1a22..2036d3bf325 100644 --- a/rust/lance-table/benches/manifest_intern.rs +++ b/rust/lance-table/benches/manifest_intern.rs @@ -57,6 +57,7 @@ fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec file_major_version: 2, file_minor_version: 0, file_size_bytes: 0, + file_metadata_size_bytes: 0, base_id: None, }], overlays: vec![], @@ -134,6 +135,7 @@ fn make_diverse_pb_fragments( file_major_version: 2, file_minor_version: 0, file_size_bytes: 0, + file_metadata_size_bytes: 0, base_id: None, }], overlays: vec![], diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs index 82afee22c1d..014438f12ae 100644 --- a/rust/lance-table/src/format/fragment.rs +++ b/rust/lance-table/src/format/fragment.rs @@ -25,7 +25,7 @@ use lance_core::error::Result; /// Lance Data File /// /// A data file is one piece of file storing data. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct DataFile { /// Relative path of the data file to dataset root. pub path: String, @@ -53,21 +53,40 @@ pub struct DataFile { /// The size of the file in bytes, if known. pub file_size_bytes: CachedFileSize, + /// The metadata suffix size in bytes, if known. + /// + /// The suffix starts at the Lance schema descriptor and ends at EOF. Readers + /// treat this as an advisory hint and use the file footer as authoritative. + /// `None` preserves the footer-directed open path used for older manifests. + pub file_metadata_size_bytes: Option>, + /// The base path of the datafile, when the datafile is outside the dataset. pub base_id: Option, } +impl DeepSizeOf for DataFile { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.path.deep_size_of_children(context) + + self.fields.deep_size_of_children(context) + + self.column_indices.deep_size_of_children(context) + } +} + // Custom Serialize: convert Arc<[i32]> to slice for transparent JSON output impl Serialize for DataFile { fn serialize(&self, serializer: S) -> std::result::Result { use serde::ser::SerializeStruct; - let mut s = serializer.serialize_struct("DataFile", 7)?; + let field_count = 7 + usize::from(self.file_metadata_size_bytes.is_some()); + let mut s = serializer.serialize_struct("DataFile", field_count)?; s.serialize_field("path", &self.path)?; s.serialize_field("fields", self.fields.as_ref())?; s.serialize_field("column_indices", self.column_indices.as_ref())?; s.serialize_field("file_major_version", &self.file_major_version)?; s.serialize_field("file_minor_version", &self.file_minor_version)?; s.serialize_field("file_size_bytes", &self.file_size_bytes)?; + if let Some(file_metadata_size_bytes) = self.file_metadata_size_bytes { + s.serialize_field("file_metadata_size_bytes", &file_metadata_size_bytes)?; + } s.serialize_field("base_id", &self.base_id)?; s.end() } @@ -87,6 +106,8 @@ impl<'de> Deserialize<'de> for DataFile { #[serde(default)] file_minor_version: u32, file_size_bytes: CachedFileSize, + #[serde(default)] + file_metadata_size_bytes: Option>, base_id: Option, } @@ -98,6 +119,7 @@ impl<'de> Deserialize<'de> for DataFile { file_major_version: helper.file_major_version, file_minor_version: helper.file_minor_version, file_size_bytes: helper.file_size_bytes, + file_metadata_size_bytes: helper.file_metadata_size_bytes, base_id: helper.base_id, }) } @@ -121,6 +143,7 @@ impl DataFile { file_major_version, file_minor_version, file_size_bytes: file_size_bytes.into(), + file_metadata_size_bytes: None, base_id, } } @@ -135,6 +158,7 @@ impl DataFile { file_major_version, file_minor_version, file_size_bytes: Default::default(), + file_metadata_size_bytes: None, base_id: None, } } @@ -220,6 +244,7 @@ impl From<&DataFile> for pb::DataFile { file_minor_version: df.file_minor_version, file_size_bytes: df.file_size_bytes.get().map_or(0, |v| v.get()), base_id: df.base_id, + file_metadata_size_bytes: df.file_metadata_size_bytes.map_or(0, NonZero::get), } } } @@ -236,6 +261,7 @@ impl TryFrom for DataFile { file_minor_version: proto.file_minor_version, file_size_bytes: CachedFileSize::new(proto.file_size_bytes), base_id: proto.base_id, + file_metadata_size_bytes: NonZero::new(proto.file_metadata_size_bytes), }) } } @@ -357,6 +383,7 @@ impl DataFileFieldInterner { file_minor_version: proto.file_minor_version, file_size_bytes: CachedFileSize::new(proto.file_size_bytes), base_id: proto.base_id, + file_metadata_size_bytes: NonZero::new(proto.file_metadata_size_bytes), }) } @@ -899,6 +926,7 @@ mod tests { let mut fragment = Fragment::new(123); let schema = ArrowSchema::new(vec![ArrowField::new("x", DataType::Float16, true)]); fragment.add_file_legacy("foobar.lance", &Schema::try_from(&schema).unwrap()); + fragment.files[0].file_metadata_size_bytes = NonZero::new(1_024); fragment.deletion_file = Some(DeletionFile { read_version: 123, id: 456, @@ -1011,6 +1039,7 @@ mod tests { file_major_version: MAJOR_VERSION as u32, file_minor_version: MINOR_VERSION as u32, file_size_bytes: Default::default(), + file_metadata_size_bytes: None, base_id: None, }; diff --git a/rust/lance-table/src/format/index.rs b/rust/lance-table/src/format/index.rs index bfd195a2a7f..5e67aecd818 100644 --- a/rust/lance-table/src/format/index.rs +++ b/rust/lance-table/src/format/index.rs @@ -4,6 +4,7 @@ //! Metadata for index use std::collections::{HashMap, HashSet}; +use std::num::NonZero; use std::sync::Arc; use chrono::{DateTime, Utc}; @@ -19,12 +20,24 @@ use lance_core::cache::{CacheEntryReader, CacheEntryWriter}; use lance_core::{Error, Result}; /// Metadata about a single file within an index segment. -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +#[derive(Debug, Clone, PartialEq)] pub struct IndexFile { /// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx") pub path: String, /// Size of the file in bytes pub size_bytes: u64, + /// The metadata suffix size in bytes, if this is a Lance-format file. + /// + /// The suffix starts at the Lance schema descriptor and ends at EOF. Readers + /// treat this as an advisory hint and use the file footer as authoritative. + /// `None` also represents index files that use another format. + pub file_metadata_size_bytes: Option>, +} + +impl DeepSizeOf for IndexFile { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.path.deep_size_of_children(context) + } } /// Index metadata @@ -280,6 +293,7 @@ impl TryFrom for IndexMetadata { .map(|f| IndexFile { path: f.path, size_bytes: f.size_bytes, + file_metadata_size_bytes: NonZero::new(f.file_metadata_size_bytes), }) .collect(), ) @@ -344,6 +358,9 @@ impl From<&IndexMetadata> for pb::IndexMetadata { .map(|f| pb::IndexFile { path: f.path.clone(), size_bytes: f.size_bytes, + file_metadata_size_bytes: f + .file_metadata_size_bytes + .map_or(0, NonZero::get), }) .collect() }) @@ -436,6 +453,7 @@ pub async fn list_index_files_with_sizes( files.push(IndexFile { path: relative_path, size_bytes: meta.size, + file_metadata_size_bytes: None, }); } Ok(files) @@ -501,6 +519,7 @@ mod tests { files: Some(vec![IndexFile { path: "index.idx".to_string(), size_bytes: 1024, + file_metadata_size_bytes: NonZero::new(256), }]), }, IndexMetadata { diff --git a/rust/lance-table/src/transaction/manifest_build.rs b/rust/lance-table/src/transaction/manifest_build.rs index 0c48ff83dc6..07857f5f5c2 100644 --- a/rust/lance-table/src/transaction/manifest_build.rs +++ b/rust/lance-table/src/transaction/manifest_build.rs @@ -1764,6 +1764,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::new(1000), + file_metadata_size_bytes: None, base_id: None, }); @@ -1775,6 +1776,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::new(500), + file_metadata_size_bytes: None, base_id: None, }); @@ -1786,6 +1788,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::new(750), + file_metadata_size_bytes: None, base_id: None, }); @@ -1797,6 +1800,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::new(250), + file_metadata_size_bytes: None, base_id: None, }); diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index c51c8f16719..1c20c03b92c 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -22,6 +22,7 @@ use lance_core::{Error, Result}; use lance_file::datatypes::Fields; use roaring::RoaringBitmap; use std::collections::HashMap; +use std::num::NonZero; use std::sync::Arc; use uuid::Uuid; @@ -462,6 +463,7 @@ impl TryFrom<&pb::transaction::rewrite::RewrittenIndex> for RewrittenIndex { .map(|f| IndexFile { path: f.path.clone(), size_bytes: f.size_bytes, + file_metadata_size_bytes: NonZero::new(f.file_metadata_size_bytes), }) .collect(), ) @@ -739,6 +741,9 @@ impl From<&RewrittenIndex> for pb::transaction::rewrite::RewrittenIndex { .map(|f| pb::IndexFile { path: f.path.clone(), size_bytes: f.size_bytes, + file_metadata_size_bytes: f + .file_metadata_size_bytes + .map_or(0, NonZero::get), }) .collect() }) diff --git a/rust/lance/src/dataset/files.rs b/rust/lance/src/dataset/files.rs index cd9c05d46c7..2e4ea70f434 100644 --- a/rust/lance/src/dataset/files.rs +++ b/rust/lance/src/dataset/files.rs @@ -903,6 +903,7 @@ mod tests { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id, }; diff --git a/rust/lance/src/dataset/fragment.rs b/rust/lance/src/dataset/fragment.rs index a353eb148c4..824f96474b7 100644 --- a/rust/lance/src/dataset/fragment.rs +++ b/rust/lance/src/dataset/fragment.rs @@ -6989,6 +6989,7 @@ mod tests { file_major_version: 2, file_minor_version: 1, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; diff --git a/rust/lance/src/dataset/tests/dataset_merge_update.rs b/rust/lance/src/dataset/tests/dataset_merge_update.rs index 2a5a148fe79..730655438e2 100644 --- a/rust/lance/src/dataset/tests/dataset_merge_update.rs +++ b/rust/lance/src/dataset/tests/dataset_merge_update.rs @@ -883,6 +883,7 @@ async fn test_datafile_partial_replacement() { file_major_version: major, file_minor_version: minor, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; @@ -944,6 +945,7 @@ async fn test_datafile_partial_replacement() { file_major_version: major, file_minor_version: minor, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; @@ -1044,6 +1046,7 @@ async fn test_datafile_replacement_error() { file_major_version: 2, file_minor_version: 0, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; @@ -3266,6 +3269,7 @@ async fn test_cross_column_fast_search_blocks_column_local_stale_postings() { file_major_version, file_minor_version, file_size_bytes: CachedFileSize::unknown(), + file_metadata_size_bytes: None, base_id: None, }; let read_version = dataset.version().version; diff --git a/rust/lance/src/dataset/write/commit.rs b/rust/lance/src/dataset/write/commit.rs index c653ee7d8a3..c0c2f71764f 100644 --- a/rust/lance/src/dataset/write/commit.rs +++ b/rust/lance/src/dataset/write/commit.rs @@ -637,6 +637,7 @@ mod tests { file_major_version: major_version, file_minor_version: minor_version, file_size_bytes: CachedFileSize::new(100), + file_metadata_size_bytes: None, base_id: None, }], overlays: vec![], diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index 74b63cb5010..9b807fbbd79 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -3825,6 +3825,7 @@ mod tests { files: Some(vec![lance_table::format::IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes: payload.len() as u64, + file_metadata_size_bytes: None, }]), } } diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs index a7525529db4..5231990dd43 100644 --- a/rust/lance/src/index/vector/builder.rs +++ b/rust/lance/src/index/vector/builder.rs @@ -1857,10 +1857,12 @@ impl IvfIndexBuilder IndexFile { path: INDEX_AUXILIARY_FILE_NAME.to_string(), size_bytes: storage_summary.size_bytes, + file_metadata_size_bytes: None, }, IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes: index_summary.size_bytes, + file_metadata_size_bytes: None, }, ]) } diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs index ab8ee6fbf52..58cfd1884ce 100644 --- a/rust/lance/src/index/vector/ivf.rs +++ b/rust/lance/src/index/vector/ivf.rs @@ -1109,6 +1109,7 @@ async fn optimize_ivf_pq_indices( IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes, + file_metadata_size_bytes: None, }, )) } @@ -1268,10 +1269,12 @@ async fn optimize_ivf_hnsw_indices( IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes: index_size, + file_metadata_size_bytes: None, }, IndexFile { path: INDEX_AUXILIARY_FILE_NAME.to_string(), size_bytes: aux_size, + file_metadata_size_bytes: None, }, ], )) @@ -2237,6 +2240,7 @@ pub(crate) async fn remap_index_file( Ok(IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes, + file_metadata_size_bytes: None, }) } @@ -2305,6 +2309,7 @@ async fn write_ivf_pq_file( Ok(IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes, + file_metadata_size_bytes: None, }) } @@ -2853,6 +2858,7 @@ async fn write_root_vector_index_from_auxiliary( Ok(IndexFile { path: INDEX_FILE_NAME.to_string(), size_bytes: summary.size_bytes, + file_metadata_size_bytes: None, }) } From af9f6003bcd64ae32c315c20d53562917ab68366 Mon Sep 17 00:00:00 2001 From: Ali Arslan Date: Fri, 11 Sep 2026 16:38:07 -0700 Subject: [PATCH 2/3] fix(bindings): initialize metadata size hints Co-authored-by: Cursor --- java/lance-jni/src/fragment.rs | 1 + java/lance-jni/src/transaction.rs | 1 + python/src/fragment.rs | 1 + python/src/transaction.rs | 6 +++++- 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/java/lance-jni/src/fragment.rs b/java/lance-jni/src/fragment.rs index 9c35cf1a8fc..f59010ff9c7 100644 --- a/java/lance-jni/src/fragment.rs +++ b/java/lance-jni/src/fragment.rs @@ -1027,6 +1027,7 @@ impl FromJObjectWithEnv for JObject<'_> { file_major_version, file_minor_version, file_size_bytes, + file_metadata_size_bytes: None, base_id, }) } diff --git a/java/lance-jni/src/transaction.rs b/java/lance-jni/src/transaction.rs index eb06d599f2d..96a5de75e93 100644 --- a/java/lance-jni/src/transaction.rs +++ b/java/lance-jni/src/transaction.rs @@ -144,6 +144,7 @@ impl FromJObjectWithEnv for JObject<'_> { "newIndexFiles.sizeBytes", env.call_method(self, "getSizeBytes", "()J", &[])?.j()?, )?, + file_metadata_size_bytes: None, }) } } diff --git a/python/src/fragment.rs b/python/src/fragment.rs index ae7135443e0..a393f8d9d1d 100644 --- a/python/src/fragment.rs +++ b/python/src/fragment.rs @@ -966,6 +966,7 @@ impl FromPyObject<'_, '_> for PyLance { file_major_version: ob.getattr("file_major_version")?.extract()?, file_minor_version: ob.getattr("file_minor_version")?.extract()?, file_size_bytes, + file_metadata_size_bytes: None, base_id: ob.getattr("base_id")?.extract()?, })) } diff --git a/python/src/transaction.rs b/python/src/transaction.rs index b77d04e2513..d43b286574b 100644 --- a/python/src/transaction.rs +++ b/python/src/transaction.rs @@ -28,7 +28,11 @@ impl FromPyObject<'_, '_> for PyLance { fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { let path = ob.getattr("path")?.extract()?; let size_bytes = ob.getattr("size_bytes")?.extract()?; - Ok(Self(IndexFile { path, size_bytes })) + Ok(Self(IndexFile { + path, + size_bytes, + file_metadata_size_bytes: None, + })) } } From 2aaa5f4e29eeca464e64443d962be56f9af06626 Mon Sep 17 00:00:00 2001 From: Ali Arslan Date: Fri, 9 Oct 2026 10:50:09 -0700 Subject: [PATCH 3/3] fix(format): define file metadata size hint precisely Anchor the hint at global buffer 0, which holds the FileDescriptor, in both proto comments and the table format docs. Legacy (0.1) files and non-Lance index files always record 0, and writers record either the exact value or 0. Move the docs section after the field-to-column note. Always serialize DataFile.file_metadata_size_bytes, like file_size_bytes and base_id. Python's DataFile now accepts and carries the value because FragmentMetadata.from_json builds it from Rust-serialized fragment JSON. Cover the hint in the manifest interner, the RewrittenIndex conversion, and JSON serialization. Co-authored-by: Cursor --- docs/src/format/table/index.md | 30 ++++++------ protos/fragment_metadata.proto | 16 ++++-- protos/table.proto | 18 ++++--- python/python/lance/fragment.py | 5 ++ python/python/tests/test_fragment.py | 9 +++- python/src/fragment.rs | 6 ++- rust/lance-table/src/format/fragment.rs | 60 ++++++++++++++++++----- rust/lance-table/src/format/index.rs | 9 ++-- rust/lance-table/src/transaction/proto.rs | 25 ++++++++++ 9 files changed, 134 insertions(+), 44 deletions(-) diff --git a/docs/src/format/table/index.md b/docs/src/format/table/index.md index cc1000ab887..e8c96a15d9a 100644 --- a/docs/src/format/table/index.md +++ b/docs/src/format/table/index.md @@ -146,20 +146,6 @@ or independently of column indices due to variable encoding widths (for Lance fi -### File Metadata Size Hints - -`DataFile.file_metadata_size_bytes` records the suffix length from the Lance -schema descriptor through the end of the data file. `IndexFile` records the -same value for Lance-format files in an index segment; it is zero for other -file formats. Combined with the corresponding total file size, a reader can -request this suffix before decoding the footer and remove a dependent metadata -request. - -The value is advisory. Zero means unavailable, values larger than the file are -invalid, and the offsets and counts decoded from the file footer remain -authoritative. A reader must preserve the ordinary footer-directed fallback -for missing, invalid, underestimated, or otherwise inconsistent hints. - !!! note "Field-to-column mapping differs between data storage versions" In **2.0**, all fields (including non-leaf fields like struct and list containers) are assigned @@ -171,6 +157,22 @@ for missing, invalid, underestimated, or otherwise inconsistent hints. See the [5.0.0 migration guide](../../guide/migration.md#500) for a detailed example. +### File Metadata Size Hints + +`DataFile.file_metadata_size_bytes` and `IndexFile.file_metadata_size_bytes` +record the size in bytes of a file's metadata suffix, or 0 if unknown. For a +Lance file of format version 2.0 or later, this is the file size minus the +position of global buffer 0, which holds the file's `FileDescriptor`. That +position is stored in the first entry of the global buffer offset table, which +the footer locates; see the [file layout](../file/index.md#detailed-overview). +It is always 0 for legacy (0.1) files and for index files in other formats. +Writers record either this exact value or 0. + +The hint is advisory: it lets a reader fetch the last +`file_metadata_size_bytes` bytes of the file before decoding the footer. The +footer stays authoritative. Readers must ignore a value larger than the file +and read any metadata the footer references outside the hinted range. + ## Deletion Files Deletion files (a.k.a. deletion vectors) track deleted rows without rewriting data files. diff --git a/protos/fragment_metadata.proto b/protos/fragment_metadata.proto index 82ea18b4b10..b39bed6802c 100644 --- a/protos/fragment_metadata.proto +++ b/protos/fragment_metadata.proto @@ -181,12 +181,18 @@ message DataFile { */ optional uint32 base_id = 7; - /* Number of bytes from the Lance schema descriptor through the end of the file. + /* Size in bytes of the file's metadata suffix, or 0 if unknown. * - * A reader that knows `file_size_bytes` may use this advisory value to issue - * the complete metadata-suffix read without first decoding the footer. The - * footer remains authoritative, and readers must fall back when this value is - * zero or invalid. + * For a Lance file of format version 2.0 or later, this is the file size minus + * the position of global buffer 0, which holds the file's `FileDescriptor`. + * That position is stored in the first entry of the global buffer offset + * table, which the footer locates. It is always 0 for legacy (0.1) files. + * Writers record either this exact value or 0. + * + * This advisory hint lets a reader fetch the last `file_metadata_size_bytes` + * bytes of the file before decoding the footer. The footer stays + * authoritative: readers must ignore a value larger than the file and read + * any metadata the footer references outside the hinted range. */ uint64 file_metadata_size_bytes = 8; } // DataFile diff --git a/protos/table.proto b/protos/table.proto index b0d9cc61be7..a60a4bd35ca 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -407,12 +407,18 @@ message IndexFile { string path = 1; // Size of the file in bytes uint64 size_bytes = 2; - /* Number of bytes from the Lance schema descriptor through the end of the file. - * - * A reader that knows `size_bytes` may use this advisory value to issue the - * complete metadata-suffix read without first decoding the footer. The footer - * remains authoritative, and readers must fall back when this value is zero or - * invalid. Zero also covers files that do not use the Lance file format. + /* Size in bytes of the file's metadata suffix, or 0 if unknown. + * + * For a Lance file of format version 2.0 or later, this is the file size minus + * the position of global buffer 0, which holds the file's `FileDescriptor`. + * That position is stored in the first entry of the global buffer offset + * table, which the footer locates. It is always 0 for legacy (0.1) Lance files + * and for files in other formats. Writers record either this exact value or 0. + * + * This advisory hint lets a reader fetch the last `file_metadata_size_bytes` + * bytes of the file before decoding the footer. The footer stays + * authoritative: readers must ignore a value larger than the file and read + * any metadata the footer references outside the hinted range. */ uint64 file_metadata_size_bytes = 3; } diff --git a/python/python/lance/fragment.py b/python/python/lance/fragment.py index 1bc9c5e5d67..e45cd0455fb 100644 --- a/python/python/lance/fragment.py +++ b/python/python/lance/fragment.py @@ -249,6 +249,8 @@ class DataFile: The minor version of the data storage format. file_size_bytes : Optional[int] The size of the data file in bytes, if available. + file_metadata_size_bytes : Optional[int] + The size of the data file's metadata suffix in bytes, if available. """ _path: str @@ -258,6 +260,7 @@ class DataFile: file_minor_version: int = 0 file_size_bytes: Optional[int] = None base_id: Optional[int] = None + file_metadata_size_bytes: Optional[int] = None def __init__( self, @@ -268,6 +271,7 @@ def __init__( file_minor_version: int = 0, file_size_bytes: Optional[int] = None, base_id: Optional[int] = None, + file_metadata_size_bytes: Optional[int] = None, ): # TODO: only we eliminate the path method, we can remove this self._path = path @@ -277,6 +281,7 @@ def __init__( self.file_minor_version = file_minor_version self.file_size_bytes = file_size_bytes self.base_id = base_id + self.file_metadata_size_bytes = file_metadata_size_bytes def __repr__(self): # pretend we have a 'path' attribute diff --git a/python/python/tests/test_fragment.py b/python/python/tests/test_fragment.py index 8b4a317e8db..b8c1e596711 100644 --- a/python/python/tests/test_fragment.py +++ b/python/python/tests/test_fragment.py @@ -328,7 +328,12 @@ def test_fragment_meta(): data = { "id": 0, "files": [ - {"path": "0.lance", "fields": [0], "file_size_bytes": 100}, + { + "path": "0.lance", + "fields": [0], + "file_size_bytes": 100, + "file_metadata_size_bytes": 25, + }, {"path": "1.lance", "fields": [1]}, ], "deletion_file": None, @@ -340,7 +345,9 @@ def test_fragment_meta(): assert len(meta.files) == 2 with pytest.warns(DeprecationWarning): assert meta.files[0].path() == "0.lance" + assert meta.files[0].file_metadata_size_bytes == 25 assert meta.files[1].path == "1.lance" + assert meta.files[1].file_metadata_size_bytes is None assert repr(meta) == ( "FragmentMetadata(id=0, files=[DataFile(path='0.lance', fields=[0], " diff --git a/python/src/fragment.rs b/python/src/fragment.rs index 16a1638bd90..6a8fceea738 100644 --- a/python/src/fragment.rs +++ b/python/src/fragment.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::fmt::Write as _; +use std::num::NonZero; use std::sync::Arc; use arrow::ffi_stream::ArrowArrayStreamReader; @@ -963,6 +964,8 @@ impl FromPyObject<'_, '_> for PyLance { fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { let file_size_bytes: Option = ob.getattr("file_size_bytes")?.extract()?; let file_size_bytes = CachedFileSize::new(file_size_bytes.unwrap_or(0)); + let file_metadata_size_bytes: Option = + ob.getattr("file_metadata_size_bytes")?.extract()?; let fields: Vec = ob.getattr("fields")?.extract()?; let column_indices: Vec = ob.getattr("column_indices")?.extract()?; Ok(Self(DataFile { @@ -972,7 +975,7 @@ impl FromPyObject<'_, '_> for PyLance { file_major_version: ob.getattr("file_major_version")?.extract()?, file_minor_version: ob.getattr("file_minor_version")?.extract()?, file_size_bytes, - file_metadata_size_bytes: None, + file_metadata_size_bytes: file_metadata_size_bytes.and_then(NonZero::new), base_id: ob.getattr("base_id")?.extract()?, })) } @@ -998,6 +1001,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataFile> { self.0.file_minor_version, file_size_bytes, self.0.base_id, + self.0.file_metadata_size_bytes.map(NonZero::get), )) } } diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs index 8a68c7fd906..48ca0dff7cc 100644 --- a/rust/lance-table/src/format/fragment.rs +++ b/rust/lance-table/src/format/fragment.rs @@ -58,9 +58,10 @@ pub struct DataFile { /// The metadata suffix size in bytes, if known. /// - /// The suffix starts at the Lance schema descriptor and ends at EOF. Readers - /// treat this as an advisory hint and use the file footer as authoritative. - /// `None` preserves the footer-directed open path used for older manifests. + /// The suffix starts at global buffer 0, which holds the `FileDescriptor`, + /// and ends at EOF. Readers treat this as an advisory hint and use the file + /// footer as authoritative. `None` preserves the footer-directed open path + /// used for older manifests. pub file_metadata_size_bytes: Option>, /// The base path of the datafile, when the datafile is outside the dataset. @@ -79,17 +80,14 @@ impl DeepSizeOf for DataFile { impl Serialize for DataFile { fn serialize(&self, serializer: S) -> std::result::Result { use serde::ser::SerializeStruct; - let field_count = 7 + usize::from(self.file_metadata_size_bytes.is_some()); - let mut s = serializer.serialize_struct("DataFile", field_count)?; + let mut s = serializer.serialize_struct("DataFile", 8)?; s.serialize_field("path", &self.path)?; s.serialize_field("fields", self.fields.as_ref())?; s.serialize_field("column_indices", self.column_indices.as_ref())?; s.serialize_field("file_major_version", &self.file_major_version)?; s.serialize_field("file_minor_version", &self.file_minor_version)?; s.serialize_field("file_size_bytes", &self.file_size_bytes)?; - if let Some(file_metadata_size_bytes) = self.file_metadata_size_bytes { - s.serialize_field("file_metadata_size_bytes", &file_metadata_size_bytes)?; - } + s.serialize_field("file_metadata_size_bytes", &self.file_metadata_size_bytes)?; s.serialize_field("base_id", &self.base_id)?; s.end() } @@ -109,7 +107,6 @@ impl<'de> Deserialize<'de> for DataFile { #[serde(default)] file_minor_version: u32, file_size_bytes: CachedFileSize, - #[serde(default)] file_metadata_size_bytes: Option>, base_id: Option, } @@ -945,6 +942,7 @@ mod tests { use lance_file::format::{MAJOR_VERSION, MINOR_VERSION}; use object_store::path::Path; use roaring::RoaringBitmap; + use rstest::rstest; use serde_json::{Value, json}; #[test] @@ -1225,11 +1223,38 @@ mod tests { assert_eq!(current_file.schema(&schema), schema); } - #[test] - fn test_to_json() { + #[rstest] + #[case::present(1_024, NonZero::new(1_024))] + #[case::absent(0, None)] + fn test_intern_fragment_file_metadata_size_bytes( + #[case] encoded: u64, + #[case] expected: Option>, + ) { + let proto = pb::DataFragment { + files: vec![pb::DataFile { + path: "data.lance".to_string(), + fields: vec![0], + file_metadata_size_bytes: encoded, + ..Default::default() + }], + ..Default::default() + }; + + let interned = DataFileFieldInterner::default() + .intern_fragment(proto.clone()) + .unwrap(); + assert_eq!(interned.files[0].file_metadata_size_bytes, expected); + assert_eq!(interned, Fragment::try_from(proto).unwrap()); + } + + #[rstest] + #[case::present(NonZero::new(1_024))] + #[case::absent(None)] + fn test_to_json(#[case] file_metadata_size_bytes: Option>) { let mut fragment = Fragment::new(123); let schema = ArrowSchema::new(vec![ArrowField::new("x", DataType::Float16, true)]); fragment.add_file_legacy("foobar.lance", &Schema::try_from(&schema).unwrap()); + fragment.files[0].file_metadata_size_bytes = file_metadata_size_bytes; fragment.deletion_file = Some(DeletionFile { read_version: 123, id: 456, @@ -1240,7 +1265,7 @@ mod tests { let json = serde_json::to_string(&fragment).unwrap(); - let value: Value = serde_json::from_str(&json).unwrap(); + let mut value: Value = serde_json::from_str(&json).unwrap(); assert_eq!( value, json!({ @@ -1248,7 +1273,8 @@ mod tests { "files":[ {"path": "foobar.lance", "fields": [0], "column_indices": [], "file_major_version": MAJOR_VERSION, "file_minor_version": MINOR_VERSION, - "file_size_bytes": null, "base_id": null } + "file_size_bytes": null, "file_metadata_size_bytes": file_metadata_size_bytes, + "base_id": null } ], "deletion_file": {"read_version": 123, "id": 456, "file_type": "array", "num_deleted_rows": 10, "base_id": null}, @@ -1257,6 +1283,14 @@ mod tests { let frag2 = Fragment::from_json(&json).unwrap(); assert_eq!(fragment, frag2); + + // JSON written before the hint existed has no key for it. + value["files"][0] + .as_object_mut() + .unwrap() + .remove("file_metadata_size_bytes"); + let frag3: Fragment = serde_json::from_value(value).unwrap(); + assert_eq!(frag3.files[0].file_metadata_size_bytes, None); } #[test] diff --git a/rust/lance-table/src/format/index.rs b/rust/lance-table/src/format/index.rs index f82e940fb23..e554f3415a3 100644 --- a/rust/lance-table/src/format/index.rs +++ b/rust/lance-table/src/format/index.rs @@ -26,11 +26,12 @@ pub struct IndexFile { pub path: String, /// Size of the file in bytes pub size_bytes: u64, - /// The metadata suffix size in bytes, if this is a Lance-format file. + /// The metadata suffix size in bytes, if known. /// - /// The suffix starts at the Lance schema descriptor and ends at EOF. Readers - /// treat this as an advisory hint and use the file footer as authoritative. - /// `None` also represents index files that use another format. + /// The suffix starts at global buffer 0, which holds the `FileDescriptor`, + /// and ends at EOF. Readers treat this as an advisory hint and use the file + /// footer as authoritative. `None` also represents legacy (0.1) Lance files + /// and index files that use another format. pub file_metadata_size_bytes: Option>, } diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs index fae8348cc2c..4b48673b432 100644 --- a/rust/lance-table/src/transaction/proto.rs +++ b/rust/lance-table/src/transaction/proto.rs @@ -896,4 +896,29 @@ mod tests { other => panic!("expected DataOverlay, got {other:?}"), } } + + #[test] + fn test_rewritten_index_roundtrips() { + let rewritten_index = RewrittenIndex { + old_id: Uuid::new_v4(), + new_id: Uuid::new_v4(), + new_index_details: prost_types::Any::default(), + new_index_version: 1, + new_index_files: Some(vec![ + IndexFile { + path: "index.idx".to_string(), + size_bytes: 4_096, + file_metadata_size_bytes: NonZero::new(256), + }, + IndexFile { + path: "auxiliary.idx".to_string(), + size_bytes: 1_024, + file_metadata_size_bytes: None, + }, + ]), + }; + + let message = pb::transaction::rewrite::RewrittenIndex::from(&rewritten_index); + assert_eq!(RewrittenIndex::try_from(&message).unwrap(), rewritten_index); + } }