perf(file): reduce metadata read scheduling rounds

docs/src/format/table/index.md

@@ -135,6 +135,20 @@ or independently of column indices due to variable encoding widths (for Lance fi
 
 </details>
 
+### File Metadata Size Hints
+
+`DataFile.file_metadata_size_bytes` records the suffix length from the Lance
+schema descriptor through the end of the data file. `IndexFile` records the
+same value for Lance-format files in an index segment; it is zero for other
+file formats. Combined with the corresponding total file size, a reader can
+request this suffix before decoding the footer and remove a dependent metadata
+request.
+
+The value is advisory. Zero means unavailable, values larger than the file are
+invalid, and the offsets and counts decoded from the file footer remain
+authoritative. A reader must preserve the ordinary footer-directed fallback
+for missing, invalid, underestimated, or otherwise inconsistent hints.
+
 !!! note "Field-to-column mapping differs between data storage versions"
 
     In **2.0**, all fields (including non-leaf fields like struct and list containers) are assigned

java/lance-jni/src/fragment.rs

@@ -17,7 +17,7 @@ use lance::table::format::{
 };
 use lance_io::utils::CachedFileSize;
 use lance_table::rowids::{RowIdSequence, write_row_ids};
-use std::iter::once;
+use std::{iter::once, num::NonZeroU64};
 
 use roaring::RoaringBitmap;
 
@@ -685,7 +685,7 @@ fn inner_encode_row_ids(env: &mut JNIEnv, row_ids: &JLongArray) -> Result<String
 
 const DATA_FILE_CLASS: &str = "org/lance/fragment/DataFile";
 const DATA_FILE_CONSTRUCTOR_SIG: &str =
-    "(Ljava/lang/String;[I[IIILjava/lang/Long;Ljava/lang/Integer;)V";
+    "(Ljava/lang/String;[I[IIILjava/lang/Long;Ljava/lang/Integer;Ljava/lang/Long;)V";
 const DELETE_FILE_CLASS: &str = "org/lance/fragment/DeletionFile";
 const DELETE_FILE_CONSTRUCTOR_SIG: &str =
     "(JJLjava/lang/Long;Lorg/lance/fragment/DeletionFileType;Ljava/lang/Integer;)V";
@@ -745,6 +745,18 @@ impl IntoJava for &DataFile {
             None => JObject::null(),
         };
         let base_id = convert_to_java_integer(env, self.base_id)?;
+        let file_metadata_size_bytes = match self.file_metadata_size_bytes {
+            Some(size) => {
+                let size = i64::try_from(size.get()).map_err(|_| {
+                    Error::runtime_error(format!(
+                        "Cannot convert DataFile.file_metadata_size_bytes={} to Java long",
+                        size.get()
+                    ))
+                })?;
+                JLance(size).into_java(env)?
+            }
+            None => JObject::null(),
+        };
         Ok(env.new_object(
             DATA_FILE_CLASS,
             DATA_FILE_CONSTRUCTOR_SIG,
@@ -756,6 +768,7 @@ impl IntoJava for &DataFile {
                 JValueGen::Int(self.file_minor_version as i32),
                 JValueGen::Object(&file_size_bytes),
                 JValueGen::Object(&base_id),
+                JValueGen::Object(&file_metadata_size_bytes),
             ],
         )?)
     }
@@ -1019,6 +1032,21 @@ impl FromJObjectWithEnv<DataFile> for JObject<'_> {
             .extract_object(env)?;
         let file_size_bytes =
             file_size_bytes.map_or(Default::default(), |r| CachedFileSize::new(r as u64));
+        let file_metadata_size_bytes: Option<i64> = env
+            .call_method(self, "getFileMetadataSizeBytes", "()Ljava/lang/Long;", &[])?
+            .l()?
+            .extract_object(env)?;
+        let file_metadata_size_bytes = match file_metadata_size_bytes {
+            Some(size) => {
+                let size = u64::try_from(size).map_err(|_| {
+                    Error::input_error(format!(
+                        "DataFile.fileMetadataSizeBytes must be non-negative, got {size}"
+                    ))
+                })?;
+                NonZeroU64::new(size)
+            }
+            None => None,
+        };
         let base_id = get_base_id(env, self)?;
         Ok(DataFile {
             path,
@@ -1027,6 +1055,7 @@ impl FromJObjectWithEnv<DataFile> for JObject<'_> {
             file_major_version,
             file_minor_version,
             file_size_bytes,
+            file_metadata_size_bytes,
             base_id,
         })
     }

java/lance-jni/src/transaction.rs

@@ -37,6 +37,7 @@ use prost_types::Any;
 use roaring::RoaringBitmap;
 use std::cell::Cell;
 use std::collections::HashMap;
+use std::num::NonZeroU64;
 use std::sync::Arc;
 use uuid::Uuid;
 
@@ -125,25 +126,46 @@ fn export_unsigned_longs<'a>(
 impl IntoJava for &IndexFile {
     fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result<JObject<'a>> {
         let path = env.new_string(&self.path)?;
+        let file_metadata_size_bytes = match self.file_metadata_size_bytes {
+            Some(size) => JLance(u64_to_jlong(
+                "newIndexFiles.fileMetadataSizeBytes",
+                size.get(),
+            )?)
+            .into_java(env)?,
+            None => JObject::null(),
+        };
         Ok(env.new_object(
             "org/lance/index/IndexFile",
-            "(Ljava/lang/String;J)V",
+            "(Ljava/lang/String;JLjava/lang/Long;)V",
             &[
                 JValue::Object(&path),
                 JValue::Long(u64_to_jlong("newIndexFiles.sizeBytes", self.size_bytes)?),
+                JValue::Object(&file_metadata_size_bytes),
             ],
         )?)
     }
 }
 
 impl FromJObjectWithEnv<IndexFile> for JObject<'_> {
     fn extract_object(&self, env: &mut JNIEnv<'_>) -> Result<IndexFile> {
+        let file_metadata_size_bytes: Option<i64> = env
+            .call_method(self, "getFileMetadataSizeBytes", "()Ljava/lang/Long;", &[])?
+            .l()?
+            .extract_object(env)?;
+        let file_metadata_size_bytes = match file_metadata_size_bytes {
+            Some(size) => {
+                let size = nonnegative_jlong_to_u64("newIndexFiles.fileMetadataSizeBytes", size)?;
+                NonZeroU64::new(size)
+            }
+            None => None,
+        };
         Ok(IndexFile {
             path: env.get_string_from_method(self, "getPath")?,
             size_bytes: nonnegative_jlong_to_u64(
                 "newIndexFiles.sizeBytes",
                 env.call_method(self, "getSizeBytes", "()J", &[])?.j()?,
             )?,
+            file_metadata_size_bytes,
         })
     }
 }

java/src/main/java/org/lance/fragment/DataFile.java

@@ -29,6 +29,7 @@ public class DataFile implements Serializable {
   private final int fileMinorVersion;
   private final Long fileSizeBytes;
   private final Integer baseId;
+  private final Long fileMetadataSizeBytes;
 
   public DataFile(
       String path,
@@ -38,13 +39,50 @@ public DataFile(
       int fileMinorVersion,
       Long fileSizeBytes,
       Integer baseId) {
+    this(
+        path,
+        fields,
+        columnIndices,
+        fileMajorVersion,
+        fileMinorVersion,
+        fileSizeBytes,
+        baseId,
+        null);
+  }
+
+  /**
+   * Creates data file metadata with an optional exact metadata suffix size.
+   *
+   * @param path path to the data file
+   * @param fields field IDs stored in the file
+   * @param columnIndices physical column indices for the fields
+   * @param fileMajorVersion major storage format version
+   * @param fileMinorVersion minor storage format version
+   * @param fileSizeBytes total file size, or null if unknown
+   * @param baseId base path ID, or null for the dataset base
+   * @param fileMetadataSizeBytes metadata suffix size; zero or null if unknown
+   */
+  public DataFile(
+      String path,
+      int[] fields,
+      int[] columnIndices,
+      int fileMajorVersion,
+      int fileMinorVersion,
+      Long fileSizeBytes,
+      Integer baseId,
+      Long fileMetadataSizeBytes) {
+    if (fileMetadataSizeBytes != null && fileMetadataSizeBytes < 0) {
+      throw new IllegalArgumentException("fileMetadataSizeBytes must be non-negative");
+    }
     this.path = path;
     this.fields = fields;
     this.columnIndices = columnIndices;
     this.fileMajorVersion = fileMajorVersion;
     this.fileMinorVersion = fileMinorVersion;
     this.fileSizeBytes = fileSizeBytes;
     this.baseId = baseId;
+    this.fileMetadataSizeBytes =
+        fileMetadataSizeBytes != null && fileMetadataSizeBytes == 0 ? null : fileMetadataSizeBytes;
   }
 
   public String getPath() {
@@ -75,6 +113,11 @@ public Optional<Integer> getBaseId() {
     return Optional.ofNullable(baseId);
   }
 
+  /** Returns the exact metadata suffix size, or null if it is unavailable. */
+  public Long getFileMetadataSizeBytes() {
+    return fileMetadataSizeBytes;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
@@ -85,7 +128,8 @@ public boolean equals(Object o) {
         && Objects.equals(path, that.path)
         && Arrays.equals(fields, that.fields)
         && Arrays.equals(columnIndices, that.columnIndices)
-        && Objects.equals(fileSizeBytes, that.fileSizeBytes);
+        && Objects.equals(fileSizeBytes, that.fileSizeBytes)
+        && Objects.equals(fileMetadataSizeBytes, that.fileMetadataSizeBytes);
   }
 
   @Override
@@ -98,6 +142,7 @@ public String toString() {
         .add("fileMinorVersion", fileMinorVersion)
         .add("fileSizeBytes", fileSizeBytes)
         .add("baseId", baseId)
+        .add("fileMetadataSizeBytes", fileMetadataSizeBytes)
         .toString();
   }
 }

java/src/main/java/org/lance/index/IndexFile.java

@@ -21,13 +21,30 @@
 public final class IndexFile {
   private final String path;
   private final long sizeBytes;
+  private final Long fileMetadataSizeBytes;
 
   public IndexFile(String path, long sizeBytes) {
+    this(path, sizeBytes, null);
+  }
+
+  /**
+   * Creates index file metadata with an optional exact metadata suffix size.
+   *
+   * @param path path relative to the index directory
+   * @param sizeBytes total file size
+   * @param fileMetadataSizeBytes metadata suffix size; zero or null if unavailable
+   */
+  public IndexFile(String path, long sizeBytes, Long fileMetadataSizeBytes) {
     this.path = Objects.requireNonNull(path, "path must not be null");
     if (sizeBytes < 0) {
       throw new IllegalArgumentException("sizeBytes must be non-negative");
     }
+    if (fileMetadataSizeBytes != null && fileMetadataSizeBytes < 0) {
+      throw new IllegalArgumentException("fileMetadataSizeBytes must be non-negative");
+    }
     this.sizeBytes = sizeBytes;
+    this.fileMetadataSizeBytes =
+        fileMetadataSizeBytes != null && fileMetadataSizeBytes == 0 ? null : fileMetadataSizeBytes;
   }
 
   public String getPath() {
@@ -38,24 +55,32 @@ public long getSizeBytes() {
     return sizeBytes;
   }
 
+  /** Returns the exact metadata suffix size, or null if it is unavailable. */
+  public Long getFileMetadataSizeBytes() {
+    return fileMetadataSizeBytes;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
     if (o == null || getClass() != o.getClass()) return false;
     IndexFile indexFile = (IndexFile) o;
-    return sizeBytes == indexFile.sizeBytes && Objects.equals(path, indexFile.path);
+    return sizeBytes == indexFile.sizeBytes
+        && Objects.equals(path, indexFile.path)
+        && Objects.equals(fileMetadataSizeBytes, indexFile.fileMetadataSizeBytes);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(path, sizeBytes);
+    return Objects.hash(path, sizeBytes, fileMetadataSizeBytes);
   }
 
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)
         .add("path", path)
         .add("sizeBytes", sizeBytes)
+        .add("fileMetadataSizeBytes", fileMetadataSizeBytes)
         .toString();
   }
 }

java/src/test/java/org/lance/FragmentTest.java

@@ -81,6 +81,15 @@ void testFragmentCreateFfiArray(@TempDir Path tempDir) {
 
   @Test
   void testFragmentCreate(@TempDir Path tempDir) throws Exception {
+    assertNull(
+        new org.lance.fragment.DataFile("unknown.lance", new int[0], new int[0], 2, 2, 1L, null, 0L)
+            .getFileMetadataSizeBytes());
+    assertThrows(
+        IllegalArgumentException.class,
+        () ->
+            new org.lance.fragment.DataFile(
+                "invalid.lance", new int[0], new int[0], 2, 2, 1L, null, -1L));
+
     String datasetPath = tempDir.resolve("new_fragment").toString();
     try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
       TestUtils.SimpleTestDataset testDataset =
@@ -89,6 +98,9 @@ void testFragmentCreate(@TempDir Path tempDir) throws Exception {
       int rowCount = 21;
       FragmentMetadata fragmentMeta = testDataset.createNewFragment(rowCount);
       assertEquals(fragmentMeta.getFiles(), fragmentMeta.getReferencedLanceFiles());
+      Long fileMetadataSizeBytes = fragmentMeta.getFiles().get(0).getFileMetadataSizeBytes();
+      assertNotNull(fileMetadataSizeBytes);
+      assertTrue(fileMetadataSizeBytes > 0);
 
       // Commit fragment
       FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(fragmentMeta));
@@ -97,6 +109,9 @@ void testFragmentCreate(@TempDir Path tempDir) throws Exception {
         assertEquals(2, dataset.latestVersion());
         assertEquals(rowCount, dataset.countRows());
         Fragment fragment = dataset.getFragments().get(0);
+        assertEquals(
+            fileMetadataSizeBytes,
+            fragment.metadata().getFiles().get(0).getFileMetadataSizeBytes());
 
         try (LanceScanner scanner = fragment.newScan()) {
           Schema schemaRes = scanner.schema();

java/src/test/java/org/lance/TransactionTest.java

@@ -58,6 +58,7 @@
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -922,6 +923,11 @@ public void testRewrittenIndexFilesModelPreservesAbsentAndEmpty() {
 
   @Test
   public void testRewrittenIndexFilesRoundTrip(@TempDir Path tempDir) {
+    assertNull(new IndexFile("unknown.idx", 123, 0L).getFileMetadataSizeBytes());
+    IllegalArgumentException invalidMetadataSize =
+        assertThrows(IllegalArgumentException.class, () -> new IndexFile("index.idx", 123, -1L));
+    assertTrue(invalidMetadataSize.getMessage().contains("must be non-negative"));
+
     String datasetPath = tempDir.resolve("rewritten_index_files").toString();
     try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
       TestUtils.SimpleTestDataset testDataset =
@@ -935,7 +941,7 @@ public void testRewrittenIndexFilesRoundTrip(@TempDir Path tempDir) {
                 .withIndexName("btree_id")
                 .build());
         Index oldIndex = dataset.getIndexes().get(0);
-        IndexFile indexFile = new IndexFile("index.idx", 123);
+        IndexFile indexFile = new IndexFile("index.idx", 123, 37L);
         RewrittenIndex rewrittenIndex =
             RewrittenIndex.builder()
                 .oldId(oldIndex.uuid())
@@ -960,6 +966,7 @@ public void testRewrittenIndexFilesRoundTrip(@TempDir Path tempDir) {
               operation.rewrittenIndices().get(0).getNewIndexFiles().orElseThrow().get(0);
           assertEquals("index.idx", readFile.getPath());
           assertEquals(123, readFile.getSizeBytes());
+          assertEquals(37L, readFile.getFileMetadataSizeBytes());
         }
       }
     }