feat(format)!: add file metadata size hint

docs/src/format/table/index.md

@@ -157,6 +157,22 @@ or independently of column indices due to variable encoding widths (for Lance fi
 
     See the [5.0.0 migration guide](../../guide/migration.md#500) for a detailed example.
 
+### File Metadata Size Hints
+
+`DataFile.file_metadata_size_bytes` and `IndexFile.file_metadata_size_bytes`
+record the size in bytes of a file's metadata suffix, or 0 if unknown. For a
+Lance file of format version 2.0 or later, this is the file size minus the
+position of global buffer 0, which holds the file's `FileDescriptor`. That
+position is stored in the first entry of the global buffer offset table, which
+the footer locates; see the [file layout](../file/index.md#detailed-overview).
+It is always 0 for legacy (0.1) files and for index files in other formats.
+Writers record either this exact value or 0.
+
+The hint is advisory: it lets a reader fetch the last
+`file_metadata_size_bytes` bytes of the file before decoding the footer. The
+footer stays authoritative. Readers must ignore a value larger than the file
+and read any metadata the footer references outside the hinted range.
+
 ## Deletion Files
 
 Deletion files (a.k.a. deletion vectors) track deleted rows without rewriting data files.

java/lance-jni/src/fragment.rs

@@ -1101,6 +1101,7 @@ impl FromJObjectWithEnv<DataFile> for JObject<'_> {
             file_major_version,
             file_minor_version,
             file_size_bytes,
+            file_metadata_size_bytes: None,
             base_id,
         })
     }

java/lance-jni/src/transaction.rs

@@ -144,6 +144,7 @@ impl FromJObjectWithEnv<IndexFile> for JObject<'_> {
                 "newIndexFiles.sizeBytes",
                 env.call_method(self, "getSizeBytes", "()J", &[])?.j()?,
             )?,
+            file_metadata_size_bytes: None,
         })
     }
 }

protos/fragment_metadata.proto

@@ -180,6 +180,21 @@ message DataFile {
    * Lance use it as key of the base_paths field in Manifest to determine the actual base path of the data file.
    */
   optional uint32 base_id = 7;
+
+  /* Size in bytes of the file's metadata suffix, or 0 if unknown.
+   *
+   * For a Lance file of format version 2.0 or later, this is the file size minus
+   * the position of global buffer 0, which holds the file's `FileDescriptor`.
+   * That position is stored in the first entry of the global buffer offset
+   * table, which the footer locates. It is always 0 for legacy (0.1) files.
+   * Writers record either this exact value or 0.
+   *
+   * This advisory hint lets a reader fetch the last `file_metadata_size_bytes`
+   * bytes of the file before decoding the footer. The footer stays
+   * authoritative: readers must ignore a value larger than the file and read
+   * any metadata the footer references outside the hinted range.
+   */
+  uint64 file_metadata_size_bytes = 8;
 } // DataFile
 
 /* An overlay file supplies new values for a subset of (row offset, field) cells

protos/table.proto

@@ -407,6 +407,20 @@ message IndexFile {
   string path = 1;
   // Size of the file in bytes
   uint64 size_bytes = 2;
+  /* Size in bytes of the file's metadata suffix, or 0 if unknown.
+   *
+   * For a Lance file of format version 2.0 or later, this is the file size minus
+   * the position of global buffer 0, which holds the file's `FileDescriptor`.
+   * That position is stored in the first entry of the global buffer offset
+   * table, which the footer locates. It is always 0 for legacy (0.1) Lance files
+   * and for files in other formats. Writers record either this exact value or 0.
+   *
+   * This advisory hint lets a reader fetch the last `file_metadata_size_bytes`
+   * bytes of the file before decoding the footer. The footer stays
+   * authoritative: readers must ignore a value larger than the file and read
+   * any metadata the footer references outside the hinted range.
+   */
+  uint64 file_metadata_size_bytes = 3;
 }
 
 // Index Section, containing a list of index metadata for one dataset version.

python/python/lance/fragment.py

@@ -249,6 +249,8 @@ class DataFile:
         The minor version of the data storage format.
     file_size_bytes : Optional[int]
         The size of the data file in bytes, if available.
+    file_metadata_size_bytes : Optional[int]
+        The size of the data file's metadata suffix in bytes, if available.
     """
 
     _path: str
@@ -258,6 +260,7 @@ class DataFile:
     file_minor_version: int = 0
     file_size_bytes: Optional[int] = None
     base_id: Optional[int] = None
+    file_metadata_size_bytes: Optional[int] = None
 
     def __init__(
         self,
@@ -268,6 +271,7 @@ def __init__(
         file_minor_version: int = 0,
         file_size_bytes: Optional[int] = None,
         base_id: Optional[int] = None,
+        file_metadata_size_bytes: Optional[int] = None,
     ):
         # TODO: only we eliminate the path method, we can remove this
         self._path = path
@@ -277,6 +281,7 @@ def __init__(
         self.file_minor_version = file_minor_version
         self.file_size_bytes = file_size_bytes
         self.base_id = base_id
+        self.file_metadata_size_bytes = file_metadata_size_bytes
 
     def __repr__(self):
         # pretend we have a 'path' attribute

python/python/tests/test_fragment.py

@@ -328,7 +328,12 @@ def test_fragment_meta():
     data = {
         "id": 0,
         "files": [
-            {"path": "0.lance", "fields": [0], "file_size_bytes": 100},
+            {
+                "path": "0.lance",
+                "fields": [0],
+                "file_size_bytes": 100,
+                "file_metadata_size_bytes": 25,
+            },
             {"path": "1.lance", "fields": [1]},
         ],
         "deletion_file": None,
@@ -340,7 +345,9 @@ def test_fragment_meta():
     assert len(meta.files) == 2
     with pytest.warns(DeprecationWarning):
         assert meta.files[0].path() == "0.lance"
+    assert meta.files[0].file_metadata_size_bytes == 25
     assert meta.files[1].path == "1.lance"
+    assert meta.files[1].file_metadata_size_bytes is None
 
     assert repr(meta) == (
         "FragmentMetadata(id=0, files=[DataFile(path='0.lance', fields=[0], "

python/src/fragment.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::fmt::Write as _;
+use std::num::NonZero;
 use std::sync::Arc;
 
 use arrow::ffi_stream::ArrowArrayStreamReader;
@@ -963,6 +964,8 @@ impl FromPyObject<'_, '_> for PyLance<DataFile> {
     fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
         let file_size_bytes: Option<u64> = ob.getattr("file_size_bytes")?.extract()?;
         let file_size_bytes = CachedFileSize::new(file_size_bytes.unwrap_or(0));
+        let file_metadata_size_bytes: Option<u64> =
+            ob.getattr("file_metadata_size_bytes")?.extract()?;
         let fields: Vec<i32> = ob.getattr("fields")?.extract()?;
         let column_indices: Vec<i32> = ob.getattr("column_indices")?.extract()?;
         Ok(Self(DataFile {
@@ -972,6 +975,7 @@ impl FromPyObject<'_, '_> for PyLance<DataFile> {
             file_major_version: ob.getattr("file_major_version")?.extract()?,
             file_minor_version: ob.getattr("file_minor_version")?.extract()?,
             file_size_bytes,
+            file_metadata_size_bytes: file_metadata_size_bytes.and_then(NonZero::new),
             base_id: ob.getattr("base_id")?.extract()?,
         }))
     }
@@ -997,6 +1001,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataFile> {
             self.0.file_minor_version,
             file_size_bytes,
             self.0.base_id,
+            self.0.file_metadata_size_bytes.map(NonZero::get),
         ))
     }
 }

python/src/transaction.rs

@@ -28,7 +28,11 @@ impl FromPyObject<'_, '_> for PyLance<IndexFile> {
     fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
         let path = ob.getattr("path")?.extract()?;
         let size_bytes = ob.getattr("size_bytes")?.extract()?;
-        Ok(Self(IndexFile { path, size_bytes }))
+        Ok(Self(IndexFile {
+            path,
+            size_bytes,
+            file_metadata_size_bytes: None,
+        }))
     }
 }
 

rust/lance-index/src/scalar.rs

@@ -60,8 +60,8 @@ pub use inverted::tokenizer::InvertedIndexParams;
 /// Convert a `Vec<`[`lance_index_core::scalar::IndexFile`]`>` to a
 /// `Vec<`[`lance_table::format::IndexFile`]`>`.
 ///
-/// These two structs have identical fields; this helper bridges the crate
-/// boundary without relying on orphan-rule–violating `From` impls.
+/// These structs share their path and total size fields; this helper bridges the
+/// crate boundary without relying on orphan-rule–violating `From` impls.
 pub fn index_files_to_table(
     files: Vec<lance_index_core::scalar::IndexFile>,
 ) -> Vec<lance_table::format::IndexFile> {
@@ -70,15 +70,16 @@ pub fn index_files_to_table(
         .map(|f| lance_table::format::IndexFile {
             path: f.path,
             size_bytes: f.size_bytes,
+            file_metadata_size_bytes: None,
         })
         .collect()
 }
 
 /// Convert a `Vec<`[`lance_table::format::IndexFile`]`>` to a
 /// `Vec<`[`lance_index_core::scalar::IndexFile`]`>`.
 ///
-/// These two structs have identical fields; this helper bridges the crate
-/// boundary without relying on orphan-rule–violating `From` impls.
+/// These structs share their path and total size fields; this helper bridges the
+/// crate boundary without relying on orphan-rule–violating `From` impls.
 pub fn table_files_to_index(
     files: Vec<lance_table::format::IndexFile>,
 ) -> Vec<lance_index_core::scalar::IndexFile> {

rust/lance-index/src/vector/distributed/index_merger.rs

@@ -1724,6 +1724,7 @@ async fn merge_partial_vector_auxiliary_files_inner(
         Ok(lance_table::format::IndexFile {
             path: INDEX_AUXILIARY_FILE_NAME.to_string(),
             size_bytes: summary.size_bytes,
+            file_metadata_size_bytes: None,
         })
     } else {
         Err(Error::index(

rust/lance-table/benches/manifest_intern.rs

@@ -57,6 +57,7 @@ fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec<pb::DataFragment>
                 file_major_version: 2,
                 file_minor_version: 0,
                 file_size_bytes: 0,
+                file_metadata_size_bytes: 0,
                 base_id: None,
             }],
             overlays: vec![],
@@ -134,6 +135,7 @@ fn make_diverse_pb_fragments(
                     file_major_version: 2,
                     file_minor_version: 0,
                     file_size_bytes: 0,
+                    file_metadata_size_bytes: 0,
                     base_id: None,
                 }],
                 overlays: vec![],