From 91a5396db74e87282ff92b87525361d50eb3d5d4 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 22 Sep 2026 15:43:54 +0800 Subject: [PATCH 1/3] [opt](lance) skip full-snapshot row-id prefilters and add profile counters --- be/src/format_v2/table/lance_reader.cpp | 29 +++- be/test/format_v2/table/lance_reader_test.cpp | 16 ++ docs/lance-prefilter-profile.md | 58 +++++++ thirdparty/download-thirdparty.sh | 5 +- .../patches/lance-c-0.1.9-prefilter.patch | 147 ++++++++++++++++++ 5 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 docs/lance-prefilter-profile.md create mode 100644 thirdparty/patches/lance-c-0.1.9-prefilter.patch diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index 840338e4b333f3..724495bf990329 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -651,10 +651,21 @@ void LanceTableReader::_init_scanner_profile() { _index_comparisons = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceIndexComparisons", TUnit::UNIT, LANCE_READER_PROFILE, 1); - // These scan counts are emitted by Lance's FilteredRead execution node. For vector searches - // with an explicit fragment set, they normally describe the fragments, ranges, and rows read - // while applying the row-id prefilter. They are scan input counts, not ANN result counts. + // Prefilter counters isolate row-id materialization. The generic scan counts below come + // from Lance's FilteredRead execution node and are scan inputs, not ANN result counts. _lance_count_metrics = { + {"prefilter_loads", + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePrefilterLoads", TUnit::UNIT, + LANCE_READER_PROFILE, 1)}, + {"prefilter_input_rows", + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePrefilterInputRows", TUnit::UNIT, + LANCE_READER_PROFILE, 1)}, + {"prefilter_input_batches", + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePrefilterInputBatches", + TUnit::UNIT, LANCE_READER_PROFILE, 1)}, + {"prefilter_row_ids", + ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LancePrefilterRowIds", TUnit::UNIT, + LANCE_READER_PROFILE, 1)}, {"fragments_scanned", ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "LanceFragmentsScanned", TUnit::UNIT, LANCE_READER_PROFILE, 1)}, @@ -686,6 +697,18 @@ void LanceTableReader::_init_scanner_profile() { TUnit::UNIT, LANCE_READER_PROFILE, 1)}, }; _lance_time_metrics = { + // These are wall times in the ANN row-id loader. LoadTime includes input polling + // and set construction; it must not be added to its component timers. + {"prefilter_load_time", + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LancePrefilterLoadTime", + LANCE_READER_PROFILE, 1)}, + {"prefilter_input_time", + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LancePrefilterInputTime", + LANCE_READER_PROFILE, 1)}, + {"prefilter_build_time", + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LancePrefilterBuildTime", + LANCE_READER_PROFILE, 1)}, + // This is wait time reported by the same Lance scan execution node described above, // rather than Doris scanner scheduling wait time. {"task_wait_time", ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "LanceTaskWaitTime", diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index b0e1b2bb831bf1..a751e3aa80e86b 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -911,6 +911,22 @@ TEST(LanceTableReaderVectorSearchTest, MultiVectorScoresFiltersOffsetsAndIndexed } } EXPECT_TRUE(reader.close().ok()); + if (indexed) { + // Read metrics after close: lance-c publishes its final execution summary + // when the stream is released, including for an early top-k stop. + for (const char* name : {"LancePrefilterLoads", "LancePrefilterInputRows", + "LancePrefilterInputBatches", "LancePrefilterRowIds", + "LancePrefilterLoadTime", "LancePrefilterInputTime", + "LancePrefilterBuildTime"}) { + auto* counter = profile.get_counter(name); + ASSERT_NE(nullptr, counter) << name; + if (filtered) { + EXPECT_GT(counter->value(), 0) << name; + } else { + EXPECT_EQ(counter->value(), 0) << name; + } + } + } } } } diff --git a/docs/lance-prefilter-profile.md b/docs/lance-prefilter-profile.md new file mode 100644 index 00000000000000..c387a076d77bbc --- /dev/null +++ b/docs/lance-prefilter-profile.md @@ -0,0 +1,58 @@ + + +# Lance vector row-ID prefilter profiling + +An explicit fragment list that covers every fragment in a fixed dataset snapshot does +not restrict an unfiltered vector query. Lance can omit the row-ID prefilter scan in +this case. The fragment selection remains attached to the scanner: indexed segment +selection, unindexed-fragment fallback, snapshot visibility, deletion masks and overlay +handling keep their existing semantics. A strict fragment subset or an actual filter +continues to use the normal prefilter path. + +The following Doris counters describe Lance's ANN **row-ID prefilter loader**, not +returned TopK rows, HNSW comparisons, or the deletion mask. Scalar-index selection +vectors use a different loader and are not included in these counters. + +| Counter | Meaning | +| --- | --- | +| `LancePrefilterLoads` | Number of row-ID prefilter loader executions started. | +| `LancePrefilterInputBatches` | Successfully consumed input batches. | +| `LancePrefilterInputRows` | Non-null input row IDs, including duplicates. | +| `LancePrefilterRowIds` | Sum of distinct row IDs in successfully completed allow sets. | +| `LancePrefilterLoadTime` | Total loader wall time, including input polling and set construction. | +| `LancePrefilterInputTime` | Wall time polling input batches, including upstream execution, I/O, decoding and scheduling. | +| `LancePrefilterBuildTime` | Wall time inserting row IDs into the allow set, measured once per batch. | + +The timers overlap: do not add LoadTime to InputTime or BuildTime. They are not CPU +timers. Across multiple loaders or scanners they accumulate and can exceed query +wall time. RowIds is not peak resident memory and can count the same ID again when +separate loaders build separate sets. An interrupted or failed load may contribute +partial input counts without a completed set cardinality. + +For a full-snapshot, unfiltered ANN query, no row-ID loader is needed and these +counters remain zero. Zero does not prove that no filtering occurred: native +visibility/deletion filtering and scalar-index selection vectors are independent. +The generic `LanceRowsScanned` counter can still include result materialization or +unindexed fallback work; it is not an exact count of distance comparisons. + +To validate performance, hold the dataset version, query vectors, search parameters, +cache state and recall target constant. Compare serial and concurrent runs using QPS, +latency percentiles, process CPU and these counters. Removal of the redundant row-ID +scan does not by itself establish the size of the end-to-end latency improvement. diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 5121d56544547f..21a321fc5b9513 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -721,7 +721,7 @@ fi # Apply Doris lance-c patches as one chain to the pinned release archive. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" - LANCE_C_PATCHED_MARK="${PATCHED_MARK}_community_pr83" + LANCE_C_PATCHED_MARK="${PATCHED_MARK}_community_pr83_prefilter" # Older source caches carry a different PR #73 and cannot accept this chain incrementally. if [[ -f "${PATCHED_MARK}" && ! -f "${LANCE_C_PATCHED_MARK}" ]]; then echo "The lance-c patch chain changed; remove ${TP_SOURCE_DIR}/${LANCE_C_SOURCE} and rebuild." @@ -730,7 +730,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then if [[ ! -f "${LANCE_C_PATCHED_MARK}" ]]; then # PR #77 provides Lance v11 for the following community patches. PR #83 # retains PR #79's scalar-segment path when adding multi-vector execution. - for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79 pr-80 pr-83; do + # The final patch pins the full-snapshot prefilter fix and its execution metrics. + for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79 pr-80 pr-83 prefilter; do patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-${lance_patch}.patch" done diff --git a/thirdparty/patches/lance-c-0.1.9-prefilter.patch b/thirdparty/patches/lance-c-0.1.9-prefilter.patch new file mode 100644 index 00000000000000..c3df2f5d26244d --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-prefilter.patch @@ -0,0 +1,147 @@ +Subject: [PATCH] Use Lance full-snapshot prefilter optimization and loader metrics + +Upstream: https://github.com/lance-format/lance/pull/9460 +Commit: f75f3343b5e125c42da1bd7acc8d8217cd5660a6 + +Pin the tested Lance v11 change without upgrading the release or changing +the C API. Keep all Lance crates on the same revision. + +diff --git a/Cargo.toml b/Cargo.toml +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -20,10 +20,10 @@ + [dependencies] +-lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } +-lance-core = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } +-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } +-lance-index = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } +-lance-io = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } +-lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } +-lance-table = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } +-lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } ++lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] } ++lance-core = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } ++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } ++lance-index = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } ++lance-io = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } ++lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } ++lance-table = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } ++lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] } + datafusion = { version = "54.0.0", default-features = false } +@@ -53,5 +53,5 @@ + [dev-dependencies] +-lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } +-lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } +-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] } ++lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } ++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } + tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +diff --git a/Cargo.lock b/Cargo.lock +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -2608,3 +2608,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -3820,3 +3820,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -3892,3 +3892,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -3914,3 +3914,3 @@ + version = "58.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -3928,3 +3928,3 @@ + version = "58.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -3938,3 +3938,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -3984,3 +3984,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4022,3 +4022,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4054,3 +4054,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4072,3 +4072,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4082,3 +4082,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4116,3 +4116,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4148,3 +4148,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4163,3 +4163,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4231,3 +4231,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4254,3 +4254,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4294,3 +4294,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4309,3 +4309,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4336,3 +4336,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4351,3 +4351,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ +@@ -4390,3 +4390,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" ++source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" + dependencies = [ From d23118234f36ead20abbd00c93c10891048c004f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 22 Sep 2026 19:11:00 +0800 Subject: [PATCH 2/3] [test](lance) stabilize dot metric golden with exhaustive refinement --- .../lance/test_lance_vector_search_metrics.out | 2 +- .../lance/test_lance_vector_search_metrics.groovy | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/regression-test/data/external_table_p0/lance/test_lance_vector_search_metrics.out b/regression-test/data/external_table_p0/lance/test_lance_vector_search_metrics.out index a155038bb0a93a..cadaee651b78e6 100644 --- a/regression-test/data/external_table_p0/lance/test_lance_vector_search_metrics.out +++ b/regression-test/data/external_table_p0/lance/test_lance_vector_search_metrics.out @@ -33,10 +33,10 @@ row_id bigint No false \N -- !ivf_pq_dot -- 1 item-0001 -8.262586 +408 item-0408 -5.387761 819 item-0819 -5.1706505 229 item-0229 -5.1260695 800 item-0800 -4.9694424 -452 item-0452 -4.5769033 -- !flat_cosine -- 1 item-0001 0.0 diff --git a/regression-test/suites/external_table_p0/lance/test_lance_vector_search_metrics.groovy b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_metrics.groovy index b557bc910a452a..2e5686fb8bb61d 100644 --- a/regression-test/suites/external_table_p0/lance/test_lance_vector_search_metrics.groovy +++ b/regression-test/suites/external_table_p0/lance/test_lance_vector_search_metrics.groovy @@ -220,9 +220,12 @@ suite("test_lance_vector_search_metrics", "p0,external") { // generator skips the "the query's own row is at distance 0" assertion whenever the metric // is dot. Lance reports the score as a negated inner product, so ORDER BY _distance ASC // still puts the best match first and every distance below is negative. + // Work around different candidate rankings in Lance's filtered and unfiltered 4-bit + // PQ paths: refine all 1024 fixture rows (5 * 256 > 1024) while probing all partitions. + // This golden checks dot scores; the single-probe checks above retain ANN coverage. qt_ivf_pq_dot """ SELECT row_id, label, _distance - FROM ${search("vs_ivf_pq_f32_dot", headQuery, "5", "4", "dot", tables["vs_ivf_pq_f32_dot"].refine)} + FROM ${search("vs_ivf_pq_f32_dot", headQuery, "5", "4", "dot", ', "refine_factor"="256"')} ORDER BY _distance, row_id """ From 71e92eb17e8cd5264cb117f3675fceb332c7c222 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 23 Sep 2026 18:04:17 +0800 Subject: [PATCH 3/3] [fix](lance) Refresh v11 prefilter metrics dependency ### What problem does this PR solve? Related PR: #68375, #68408 Refresh the pinned Lance v11 dependency with the FTS metric wiring and full-snapshot test coverage backported from the main-branch review. Apply the follow-up pin independently because existing source caches already carry the original prefilter patch marker. ### Release note Include FTS row-ID prefilter metrics in the Lance dependency and upgrade previously patched source caches to the same revision as fresh builds. ### Check List (For Author) - Test: v11 prefilter and FTS tests, Rust formatting and workspace Clippy, archive checksum, fresh/cached patch application, idempotence, rejection without a success marker, locked dependency metadata, shell syntax. - Behavior changed: FTS loader metrics are recorded; cached dependencies receive the updated pin. PQ scoring is unchanged. - Does this need documentation: Dependency scope and validation are recorded in the pull request description. --- thirdparty/download-thirdparty.sh | 6 + .../patches/lance-c-0.1.9-prefilter-fts.patch | 148 ++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 21a321fc5b9513..ad6242c710e044 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -737,6 +737,12 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then done touch "${PATCHED_MARK}" "${LANCE_C_PATCHED_MARK}" fi + # Cached sources may carry the earlier prefilter pin; upgrade FTS metrics independently. + if [[ ! -f "${PATCHED_MARK}_prefilter_fts" ]]; then + patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ + -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-prefilter-fts.patch" + touch "${PATCHED_MARK}_prefilter_fts" + fi cd - echo "Finished patching ${LANCE_C_SOURCE}" fi diff --git a/thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch b/thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch new file mode 100644 index 00000000000000..890093886fbcbb --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch @@ -0,0 +1,148 @@ +Subject: [PATCH] Include FTS prefilter metrics and full-snapshot test coverage + +Upstream: https://github.com/lance-format/lance/pull/9460 +Commit: f202fe41ac18323ca0cd7bc6efd8fd30b8722116 + +Backport the applicable review updates from Lance PR #9471 to v11. +Keep the C API and other dependencies unchanged, and upgrade previously +patched source caches with the same revision as fresh builds. + +diff --git a/Cargo.toml b/Cargo.toml +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -20,10 +20,10 @@ + [dependencies] +-lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] } +-lance-core = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } +-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } +-lance-index = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } +-lance-io = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } +-lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } +-lance-table = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } +-lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] } ++lance = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116", features = ["substrait"] } ++lance-core = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" } ++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" } ++lance-index = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" } ++lance-io = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" } ++lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" } ++lance-table = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" } ++lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116", features = ["substrait"] } + datafusion = { version = "54.0.0", default-features = false } +@@ -53,5 +53,5 @@ + [dev-dependencies] +-lance = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6", features = ["substrait"] } +-lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } +-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f75f3343b5e125c42da1bd7acc8d8217cd5660a6" } ++lance = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116", features = ["substrait"] } ++lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" } ++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "f202fe41ac18323ca0cd7bc6efd8fd30b8722116" } + tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +diff --git a/Cargo.lock b/Cargo.lock +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -2608,3 +2608,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -3820,3 +3820,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -3892,3 +3892,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -3914,3 +3914,3 @@ + version = "58.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -3928,3 +3928,3 @@ + version = "58.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -3938,3 +3938,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -3984,3 +3984,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4022,3 +4022,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4054,3 +4054,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4072,3 +4072,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4082,3 +4082,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4116,3 +4116,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4148,3 +4148,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4163,3 +4163,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4231,3 +4231,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4254,3 +4254,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4294,3 +4294,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4309,3 +4309,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4336,3 +4336,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4351,3 +4351,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [ +@@ -4390,3 +4390,3 @@ + version = "11.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=f75f3343b5e125c42da1bd7acc8d8217cd5660a6#f75f3343b5e125c42da1bd7acc8d8217cd5660a6" ++source = "git+https://github.com/lance-format/lance.git?rev=f202fe41ac18323ca0cd7bc6efd8fd30b8722116#f202fe41ac18323ca0cd7bc6efd8fd30b8722116" + dependencies = [