[opt](lance) skip full-snapshot row-id prefilters and add profile counters

be/src/format_v2/table/lance_reader.cpp

@@ -651,10 +651,21 @@ void LanceTableReader::_init_scanner_profile() {
     _index_comparisons = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceIndexComparisons",
                                                       TUnit::UNIT, LANCE_READER_PROFILE, 1);
 
-    // These scan counts are emitted by Lance's FilteredRead execution node. For vector searches
-    // with an explicit fragment set, they normally describe the fragments, ranges, and rows read
-    // while applying the row-id prefilter. They are scan input counts, not ANN result counts.
+    // Prefilter counters isolate row-id materialization. The generic scan counts below come
+    // from Lance's FilteredRead execution node and are scan inputs, not ANN result counts.
     _lance_count_metrics = {
+            {"prefilter_loads",
+             ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePrefilterLoads", TUnit::UNIT,
+                                          LANCE_READER_PROFILE, 1)},
+            {"prefilter_input_rows",
+             ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePrefilterInputRows", TUnit::UNIT,
+                                          LANCE_READER_PROFILE, 1)},
+            {"prefilter_input_batches",
+             ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePrefilterInputBatches",
+                                          TUnit::UNIT, LANCE_READER_PROFILE, 1)},
+            {"prefilter_row_ids",
+             ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePrefilterRowIds", TUnit::UNIT,
+                                          LANCE_READER_PROFILE, 1)},
             {"fragments_scanned",
              ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceFragmentsScanned", TUnit::UNIT,
                                           LANCE_READER_PROFILE, 1)},
@@ -686,6 +697,18 @@ void LanceTableReader::_init_scanner_profile() {
                                           TUnit::UNIT, LANCE_READER_PROFILE, 1)},
     };
     _lance_time_metrics = {
+            // These are wall times in the ANN row-id loader. LoadTime includes input polling
+            // and set construction; it must not be added to its component timers.
+            {"prefilter_load_time",
+             ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LancePrefilterLoadTime",
+                                        LANCE_READER_PROFILE, 1)},
+            {"prefilter_input_time",
+             ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LancePrefilterInputTime",
+                                        LANCE_READER_PROFILE, 1)},
+            {"prefilter_build_time",
+             ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LancePrefilterBuildTime",
+                                        LANCE_READER_PROFILE, 1)},
+
             // This is wait time reported by the same Lance scan execution node described above,
             // rather than Doris scanner scheduling wait time.
             {"task_wait_time", ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceTaskWaitTime",

be/test/format_v2/table/lance_reader_test.cpp

@@ -911,6 +911,22 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorScoresFiltersOffsetsAndIndexed
                     }
                 }
                 EXPECT_TRUE(reader.close().ok());
+                if (indexed) {
+                    // Read metrics after close: lance-c publishes its final execution summary
+                    // when the stream is released, including for an early top-k stop.
+                    for (const char* name : {"LancePrefilterLoads", "LancePrefilterInputRows",
+                                             "LancePrefilterInputBatches", "LancePrefilterRowIds",
+                                             "LancePrefilterLoadTime", "LancePrefilterInputTime",
+                                             "LancePrefilterBuildTime"}) {
+                        auto* counter = profile.get_counter(name);
+                        ASSERT_NE(nullptr, counter) << name;
+                        if (filtered) {
+                            EXPECT_GT(counter->value(), 0) << name;
+                        } else {
+                            EXPECT_EQ(counter->value(), 0) << name;
+                        }
+                    }
+                }
             }
         }
     }

docs/lance-prefilter-profile.md

@@ -0,0 +1,58 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Lance vector row-ID prefilter profiling
+
+An explicit fragment list that covers every fragment in a fixed dataset snapshot does
+not restrict an unfiltered vector query. Lance can omit the row-ID prefilter scan in
+this case. The fragment selection remains attached to the scanner: indexed segment
+selection, unindexed-fragment fallback, snapshot visibility, deletion masks and overlay
+handling keep their existing semantics. A strict fragment subset or an actual filter
+continues to use the normal prefilter path.
+
+The following Doris counters describe Lance's ANN **row-ID prefilter loader**, not
+returned TopK rows, HNSW comparisons, or the deletion mask. Scalar-index selection
+vectors use a different loader and are not included in these counters.
+
+| Counter | Meaning |
+| --- | --- |
+| `LancePrefilterLoads` | Number of row-ID prefilter loader executions started. |
+| `LancePrefilterInputBatches` | Successfully consumed input batches. |
+| `LancePrefilterInputRows` | Non-null input row IDs, including duplicates. |
+| `LancePrefilterRowIds` | Sum of distinct row IDs in successfully completed allow sets. |
+| `LancePrefilterLoadTime` | Total loader wall time, including input polling and set construction. |
+| `LancePrefilterInputTime` | Wall time polling input batches, including upstream execution, I/O, decoding and scheduling. |
+| `LancePrefilterBuildTime` | Wall time inserting row IDs into the allow set, measured once per batch. |
+
+The timers overlap: do not add LoadTime to InputTime or BuildTime. They are not CPU
+timers. Across multiple loaders or scanners they accumulate and can exceed query
+wall time. RowIds is not peak resident memory and can count the same ID again when
+separate loaders build separate sets. An interrupted or failed load may contribute
+partial input counts without a completed set cardinality.
+
+For a full-snapshot, unfiltered ANN query, no row-ID loader is needed and these
+counters remain zero. Zero does not prove that no filtering occurred: native
+visibility/deletion filtering and scalar-index selection vectors are independent.
+The generic `LanceRowsScanned` counter can still include result materialization or
+unindexed fallback work; it is not an exact count of distance comparisons.
+
+To validate performance, hold the dataset version, query vectors, search parameters,
+cache state and recall target constant. Compare serial and concurrent runs using QPS,
+latency percentiles, process CPU and these counters. Removal of the redundant row-ID
+scan does not by itself establish the size of the end-to-end latency improvement.

regression-test/data/external_table_p0/lance/test_lance_vector_search_metrics.out

@@ -33,10 +33,10 @@ row_id	bigint	No	false	\N
 
 -- !ivf_pq_dot --
 1	item-0001	-8.262586
+408	item-0408	-5.387761
 819	item-0819	-5.1706505
 229	item-0229	-5.1260695
 800	item-0800	-4.9694424
-452	item-0452	-4.5769033
 
 -- !flat_cosine --
 1	item-0001	0.0

regression-test/suites/external_table_p0/lance/test_lance_vector_search_metrics.groovy

@@ -220,9 +220,12 @@ suite("test_lance_vector_search_metrics", "p0,external") {
     // generator skips the "the query's own row is at distance 0" assertion whenever the metric
     // is dot. Lance reports the score as a negated inner product, so ORDER BY _distance ASC
     // still puts the best match first and every distance below is negative.
+    // Work around different candidate rankings in Lance's filtered and unfiltered 4-bit
+    // PQ paths: refine all 1024 fixture rows (5 * 256 > 1024) while probing all partitions.
+    // This golden checks dot scores; the single-probe checks above retain ANN coverage.
     qt_ivf_pq_dot """
         SELECT row_id, label, _distance
-        FROM ${search("vs_ivf_pq_f32_dot", headQuery, "5", "4", "dot", tables["vs_ivf_pq_f32_dot"].refine)}
+        FROM ${search("vs_ivf_pq_f32_dot", headQuery, "5", "4", "dot", ', "refine_factor"="256"')}
         ORDER BY _distance, row_id
     """
 

thirdparty/download-thirdparty.sh

@@ -721,7 +721,7 @@ fi
 # Apply Doris lance-c patches as one chain to the pinned release archive.
 if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then
     cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}"
-    LANCE_C_PATCHED_MARK="${PATCHED_MARK}_community_pr83"
+    LANCE_C_PATCHED_MARK="${PATCHED_MARK}_community_pr83_prefilter"
     # Older source caches carry a different PR #73 and cannot accept this chain incrementally.
     if [[ -f "${PATCHED_MARK}" && ! -f "${LANCE_C_PATCHED_MARK}" ]]; then
         echo "The lance-c patch chain changed; remove ${TP_SOURCE_DIR}/${LANCE_C_SOURCE} and rebuild."
@@ -730,7 +730,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then
     if [[ ! -f "${LANCE_C_PATCHED_MARK}" ]]; then
         # PR #77 provides Lance v11 for the following community patches. PR #83
         # retains PR #79's scalar-segment path when adding multi-vector execution.
-        for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79 pr-80 pr-83; do
+        # The final patch pins the full-snapshot prefilter fix and its execution metrics.
+        for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79 pr-80 pr-83 prefilter; do
             patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \
                 -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-${lance_patch}.patch"
         done

thirdparty/patches/lance-c-0.1.9-prefilter.patch

@@ -0,0 +1,147 @@
+Subject: [PATCH] Use Lance full-snapshot prefilter optimization and loader metrics
+
+Upstream: https://github.com/lance-format/lance/pull/9460
+Commit: f75f3343b5e125c42da1bd7acc8d8217cd5660a6
+
+Pin the tested Lance v11 change without upgrading the release or changing
+the C API. Keep all Lance crates on the same revision.
+
+diff --git a/Cargo.toml b/Cargo.toml
+--- a/Cargo.toml
++++ b/Cargo.toml
+@@ -20,10 +20,10 @@
+ [dependencies]
+-lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
+-lance-core = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
+-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
+-lance-index = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
+-lance-io = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
+-lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
+-lance-table = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
+-lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
++lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
++lance-core = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
++lance-index = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
++lance-io = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
++lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
++lance-table = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
++lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
+ datafusion = { version = "54.0.0", default-features = false }
+@@ -53,5 +53,5 @@
+ [dev-dependencies]
+-lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] }
+-lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
+-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" }
++lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] }
++lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" }
+ tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
+diff --git a/Cargo.lock b/Cargo.lock
+--- a/Cargo.lock
++++ b/Cargo.lock
+@@ -2608,3 +2608,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -3820,3 +3820,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -3892,3 +3892,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -3914,3 +3914,3 @@
+ version = "58.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -3928,3 +3928,3 @@
+ version = "58.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -3938,3 +3938,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -3984,3 +3984,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4022,3 +4022,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4054,3 +4054,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4072,3 +4072,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4082,3 +4082,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4116,3 +4116,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4148,3 +4148,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4163,3 +4163,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4231,3 +4231,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4254,3 +4254,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4294,3 +4294,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4309,3 +4309,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4336,3 +4336,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4351,3 +4351,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [
+@@ -4390,3 +4390,3 @@
+ version = "11.0.0"
+-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842"
++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6"
+ dependencies = [