From b9a83e21e79d2b94de925a3c348518b0f39126ef Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Tue, 6 Oct 2026 21:45:08 +0700 Subject: [PATCH 01/10] tests: check ColModernVBERT mixed-size batches against the re-exported test model --- .../workflows/colmodernvbert-onnx-check.yml | 51 +++++++++++++++++++ .../colmodernvbert.py | 3 +- tests/test_late_interaction_multimodal.py | 23 +++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/colmodernvbert-onnx-check.yml diff --git a/.github/workflows/colmodernvbert-onnx-check.yml b/.github/workflows/colmodernvbert-onnx-check.yml new file mode 100644 index 00000000..4caaf573 --- /dev/null +++ b/.github/workflows/colmodernvbert-onnx-check.yml @@ -0,0 +1,51 @@ +name: ColModernVBERT ONNX runtime check + +# TEMPORARY, delete before merging: checks the re-exported model from the test repo +# jmzzomg/colmodernvbert with released fastembed 0.8.1 and the oldest and newest onnxruntime it allows on +# each Python version, on Windows and x86 Linux (the regular tests skip ColModernVBERT on Windows). +on: + push: + branches: [ colmodernvbert-fixes ] + +jobs: + check: + strategy: + fail-fast: false + matrix: + os: + - windows-latest + - ubuntu-latest + runtime: + - { python: '3.10', onnxruntime: '1.18.0', numpy: 'numpy<2' } + - { python: '3.10', onnxruntime: '1.23.2', numpy: 'numpy' } + - { python: '3.12', onnxruntime: '1.18.0', numpy: 'numpy<2' } + - { python: '3.12', onnxruntime: '1.30.0', numpy: 'numpy' } + - { python: '3.13', onnxruntime: '1.21.1', numpy: 'numpy' } + - { python: '3.13', onnxruntime: '1.30.0', numpy: 'numpy' } + - { python: '3.14', onnxruntime: '1.24.2', numpy: 'numpy' } + - { python: '3.14', onnxruntime: '1.30.0', numpy: 'numpy' } + exclude: + # onnxruntime 1.18.x on Linux aborts while optimizing this graph, with the published model.onnx too + - os: ubuntu-latest + runtime: { python: '3.10', onnxruntime: '1.18.0', numpy: 'numpy<2' } + - os: ubuntu-latest + runtime: { python: '3.12', onnxruntime: '1.18.0', numpy: 'numpy<2' } + + runs-on: ${{ matrix.os }} + + name: py${{ matrix.runtime.python }} onnxruntime ${{ matrix.runtime.onnxruntime }} on ${{ matrix.os }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.runtime.python }} + - name: Install released fastembed and the onnxruntime under test + run: | + python -m pip install "fastembed==0.8.1" + python -m pip install "onnxruntime==${{ matrix.runtime.onnxruntime }}" "${{ matrix.runtime.numpy }}" + - name: Check the candidate model + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: python scripts/check_colmodernvbert_onnx.py --repo jmzzomg/colmodernvbert --revision main diff --git a/fastembed/late_interaction_multimodal/colmodernvbert.py b/fastembed/late_interaction_multimodal/colmodernvbert.py index f8fe0154..bd215233 100644 --- a/fastembed/late_interaction_multimodal/colmodernvbert.py +++ b/fastembed/late_interaction_multimodal/colmodernvbert.py @@ -27,7 +27,8 @@ description="The late-interaction version of ModernVBERT, CPU friendly, English, 2025.", license="mit", size_in_GB=1.0, - sources=ModelSource(hf="Qdrant/colmodernvbert"), + # TEMPORARY: test copy of the re-exported model, switch back to Qdrant/colmodernvbert before merging + sources=ModelSource(hf="jmzzomg/colmodernvbert"), additional_files=["processor_config.json"], model_file="model.onnx", ), diff --git a/tests/test_late_interaction_multimodal.py b/tests/test_late_interaction_multimodal.py index 5237475a..bb50eae6 100644 --- a/tests/test_late_interaction_multimodal.py +++ b/tests/test_late_interaction_multimodal.py @@ -130,6 +130,29 @@ def test_single_embedding(model_cache): assert np.allclose(result[:token_num, :abridged_dim], expected_result, atol=2e-3) +def test_batch_embedding_mixed_tile_counts(model_cache): + # image splitting makes colmodernvbert slow on CI runners, so only linux runs it + if platform.system() != "Linux" and os.getenv("CI"): + pytest.skip("too slow on macOS and Windows CI runners") + + model_name = "Qdrant/colmodernvbert" + image = Image.open(TEST_MISC_DIR / "image.jpeg") + # the 4:1 strip is split into 5 tiles and the full image into 13, so the strip gets padded + # with empty tiles, which must not leak into the image after it + strip = image.crop((0, 0, image.width, image.width // 4)) + mixed_images = [strip, image] + with model_cache(model_name) as model: + batch_result = list(model.embed_image(mixed_images, batch_size=len(mixed_images))) + single_result = list(model.embed_image(mixed_images, batch_size=1)) + + for batch_value, single_value in zip(batch_result, single_result): + # padding tokens are zeroed out + batch_value = batch_value[np.any(batch_value, axis=-1)] + single_value = single_value[np.any(single_value, axis=-1)] + assert batch_value.shape == single_value.shape + assert np.allclose(batch_value, single_value, atol=1e-3) + + def test_single_embedding_query(model_cache): # text inputs get a zero image placeholder per token, so this is as slow as the image tests if platform.system() != "Linux" and os.getenv("CI"): From 311dce413c768c2efaa2488cc6953ef3a689bdad Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Tue, 6 Oct 2026 21:48:19 +0700 Subject: [PATCH 02/10] tests: add tmp test script --- scripts/check_colmodernvbert_onnx.py | 144 +++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 scripts/check_colmodernvbert_onnx.py diff --git a/scripts/check_colmodernvbert_onnx.py b/scripts/check_colmodernvbert_onnx.py new file mode 100644 index 00000000..dfc6e8cf --- /dev/null +++ b/scripts/check_colmodernvbert_onnx.py @@ -0,0 +1,144 @@ +"""Runtime check of a ColModernVBERT model.onnx with the installed fastembed and onnxruntime. + +Downloads the model from a Hugging Face revision (a branch that fastembed users never resolve), then checks: + 1. fastembed's canonical image and query values (atol 2e-3), through the installed fastembed's own code path + 2. text queries give the same output with one blank tile per token (fastembed <= 0.8.x), 1 tile and 0 tiles + 3. an image embedded inside a zero-padded batch equals the same image embedded alone +Exits 1 on any failure. + +Usage: python check_colmodernvbert_onnx.py --revision [--repo Qdrant/colmodernvbert] + python check_colmodernvbert_onnx.py --model-dir +""" + +import argparse +import platform +import sys +import time +from pathlib import Path + +import numpy as np +import onnxruntime as ort +from huggingface_hub import snapshot_download +from PIL import Image + +import fastembed +from fastembed import LateInteractionMultimodalEmbedding + +# tests/test_late_interaction_multimodal.py +CANONICAL_IMAGE = np.array( + [ + [0.11614, -0.15793, -0.11194, 0.0688, 0.08001, 0.10575, -0.07871], + [0.10094, -0.13301, -0.12069, 0.10932, 0.04645, 0.09884, 0.04048], + [0.13106, -0.18613, -0.13469, 0.10566, 0.03659, 0.07712, -0.03916], + [0.09754, -0.09596, -0.04839, 0.14991, 0.05692, 0.10569, -0.08349], + [0.02576, -0.15651, -0.09977, 0.09707, 0.13412, 0.09994, -0.09931], + [-0.06741, -0.1787, -0.19677, -0.07618, 0.13102, -0.02131, -0.02437], + [-0.02776, -0.10187, -0.13793, 0.03835, 0.04766, 0.04701, -0.15635], + ] +) +CANONICAL_QUERY = np.array( + [ + [0.05, 0.06557, 0.04026, 0.14981, 0.1842, 0.0263, -0.18706], + [-0.05664, -0.14028, 0.00649, -0.02849, 0.09034, -0.01494, 0.10693], + [-0.10147, -0.00716, 0.09084, -0.08236, -0.01849, -0.00972, -0.00461], + [-0.1233, -0.10814, -0.02337, -0.00329, 0.05984, 0.09934, 0.09846], + [-0.07053, -0.13119, -0.06487, 0.01508, 0.07459, 0.07655, 0.14821], + [0.00526, -0.13842, -0.05837, -0.02721, 0.13009, 0.05076, 0.17962], + [0.00924, -0.14383, -0.03057, -0.03691, 0.11718, 0.037, 0.13344], + ] +) +QUERIES = ["hello world", "flag embedding"] +TEST_IMAGE = Path(__file__).resolve().parents[1] / "tests" / "misc" / "image.jpeg" + +parser = argparse.ArgumentParser() +parser.add_argument("--repo", default="Qdrant/colmodernvbert") +group = parser.add_mutually_exclusive_group(required=True) +group.add_argument("--revision", help="Hugging Face branch or commit to download") +group.add_argument("--model-dir", help="local directory instead of a download") +args = parser.parse_args() + +model_dir = args.model_dir or snapshot_download( + args.repo, revision=args.revision, allow_patterns=["model.onnx", "*.json"] +) +print( + f"python {platform.python_version()} {platform.system()}-{platform.machine()}, onnxruntime {ort.__version__}, " + f"fastembed {fastembed.__version__}, model {args.model_dir or f'{args.repo}@{args.revision}'}" +) +failures = [] + + +def check(name, diff, tol, seconds): + status = "ok" if diff <= tol else "FAIL" + if status == "FAIL": + failures.append(name) + print(f" {name}: max diff {diff:.1e} (tol {tol:g}) {status} [{seconds:.2f}s]") + + +model = LateInteractionMultimodalEmbedding("Qdrant/colmodernvbert", specific_model_path=model_dir) +image = Image.open(TEST_IMAGE).convert("RGB") +wide = image.crop( + (0, 0, image.width, image.width // 4) +) # 5 tiles instead of 13, so batches get padding + +t = time.perf_counter() +single = list(model.embed_image([str(TEST_IMAGE)], batch_size=1))[0] +check( + "canonical image", + float(np.abs(single[:7, :7] - CANONICAL_IMAGE).max()), + 2e-3, + time.perf_counter() - t, +) + +t = time.perf_counter() +query = next(iter(model.embed_text(QUERIES))) +check( + "canonical query (installed fastembed's placeholder)", + float(np.abs(query[:7, :7] - CANONICAL_QUERY).max()), + 2e-3, + time.perf_counter() - t, +) + +t = time.perf_counter() +padded = list(model.embed_image([wide, image], batch_size=2)) +alone = list(model.embed_image([wide], batch_size=1))[0] + + +def real_rows( + x, +): # padding positions are zeroed by the attention mask; padding may be on either side + return x[np.abs(x).sum(-1) > 0] + + +diff = max( + float(np.abs(real_rows(padded[1]) - real_rows(single)).max()), + float(np.abs(real_rows(padded[0]) - real_rows(alone)).max()), +) +check("image in zero-padded batch == image alone", diff, 1e-4, time.perf_counter() - t) + +session = ort.InferenceSession( + str(Path(model_dir) / "model.onnx"), providers=["CPUExecutionProvider"] +) +encoded = model.model.tokenize(QUERIES) +ids = np.array([e.ids for e in encoded], dtype=np.int64) +mask = np.array([e.attention_mask for e in encoded], dtype=np.int64) +outs = {} +for tiles in (ids.shape[1], 1, 0): + t = time.perf_counter() + outs[tiles] = session.run( + None, + { + "input_ids": ids, + "attention_mask": mask, + "pixel_values": np.zeros((len(ids), tiles, 3, 512, 512), np.float32), + }, + )[0] + print(f" queries with {tiles} blank tiles per query: {time.perf_counter() - t:.2f}s") +check( + "queries: 1 tile vs one per token", float(np.abs(outs[1] - outs[ids.shape[1]]).max()), 1e-5, 0 +) +check( + "queries: 0 tiles vs one per token", float(np.abs(outs[0] - outs[ids.shape[1]]).max()), 1e-5, 0 +) + +print("FAILED: " + ", ".join(failures) if failures else "ALL PASSED") +sys.exit(1 if failures else 0) From 0bf2329d03ba794e1afdbf360a6eec2965addd7a Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Tue, 6 Oct 2026 21:57:11 +0700 Subject: [PATCH 03/10] tests: probe whether onnxruntime 1.18 crashes on the published ColModernVBERT on Windows too --- .../workflows/colmodernvbert-onnx-check.yml | 28 +++++++++++++++- scripts/probe_ort_load.py | 32 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 scripts/probe_ort_load.py diff --git a/.github/workflows/colmodernvbert-onnx-check.yml b/.github/workflows/colmodernvbert-onnx-check.yml index 4caaf573..f6b62a8d 100644 --- a/.github/workflows/colmodernvbert-onnx-check.yml +++ b/.github/workflows/colmodernvbert-onnx-check.yml @@ -48,4 +48,30 @@ jobs: - name: Check the candidate model env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: python scripts/check_colmodernvbert_onnx.py --repo jmzzomg/colmodernvbert --revision main + run: python -X faulthandler -u scripts/check_colmodernvbert_onnx.py --repo jmzzomg/colmodernvbert --revision main + + # onnxruntime 1.18.0 crashes natively on Windows while loading the new file. Does the published file crash + # the same way (as both do on Linux), and does basic graph optimization avoid it? + probe-onnxruntime-1-18: + strategy: + fail-fast: false + matrix: + repo: [ jmzzomg/colmodernvbert, Qdrant/colmodernvbert ] + optimization: [ all, basic ] + + runs-on: windows-latest + + name: onnxruntime 1.18.0 loads ${{ matrix.repo }} with ${{ matrix.optimization }} optimization on windows-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install onnxruntime 1.18.0 + run: python -m pip install "onnxruntime==1.18.0" "numpy<2" huggingface_hub + - name: Load the model and run a text query + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: python -X faulthandler -u scripts/probe_ort_load.py ${{ matrix.repo }} ${{ matrix.optimization }} diff --git a/scripts/probe_ort_load.py b/scripts/probe_ort_load.py new file mode 100644 index 00000000..5feda581 --- /dev/null +++ b/scripts/probe_ort_load.py @@ -0,0 +1,32 @@ +"""Load a ColModernVBERT model.onnx with one graph optimization level and run a tiny text query. + +TEMPORARY, delete before merging. Usage: python -X faulthandler -u probe_ort_load.py +""" + +import sys + +import numpy as np +import onnxruntime as ort +from huggingface_hub import hf_hub_download + +repo, level = sys.argv[1], sys.argv[2] +levels = { + "all": ort.GraphOptimizationLevel.ORT_ENABLE_ALL, + "basic": ort.GraphOptimizationLevel.ORT_ENABLE_BASIC, +} +path = hf_hub_download(repo, "model.onnx") +print(f"onnxruntime {ort.__version__}, {repo}, optimization {level}: loading", flush=True) +options = ort.SessionOptions() +options.graph_optimization_level = levels[level] +session = ort.InferenceSession(path, options, providers=["CPUExecutionProvider"]) +print("loaded", flush=True) +ids = np.array([[50281, 25521, 1533, 50282]], dtype=np.int64) +output = session.run( + None, + { + "input_ids": ids, + "attention_mask": np.ones_like(ids), + "pixel_values": np.zeros((1, 1, 3, 512, 512), np.float32), + }, +)[0] +print(f"query ok, output {output.shape}", flush=True) From 15a426cd3d4b6a145ac7ab7ef9b794e4a20eb631 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Tue, 6 Oct 2026 23:51:35 +0700 Subject: [PATCH 04/10] tests: drop onnxruntime 1.18 and the probe job from the temporary ColModernVBERT check --- .../workflows/colmodernvbert-onnx-check.yml | 35 ++----------------- .../colmodernvbert.py | 8 +++++ scripts/probe_ort_load.py | 32 ----------------- 3 files changed, 10 insertions(+), 65 deletions(-) delete mode 100644 scripts/probe_ort_load.py diff --git a/.github/workflows/colmodernvbert-onnx-check.yml b/.github/workflows/colmodernvbert-onnx-check.yml index f6b62a8d..1b998286 100644 --- a/.github/workflows/colmodernvbert-onnx-check.yml +++ b/.github/workflows/colmodernvbert-onnx-check.yml @@ -3,6 +3,8 @@ name: ColModernVBERT ONNX runtime check # TEMPORARY, delete before merging: checks the re-exported model from the test repo # jmzzomg/colmodernvbert with released fastembed 0.8.1 and the oldest and newest onnxruntime it allows on # each Python version, on Windows and x86 Linux (the regular tests skip ColModernVBERT on Windows). +# onnxruntime 1.18 is left out: it crashes while loading this graph on Windows and Linux, with the +# published model.onnx too, and fastembed now refuses onnxruntime<1.19 for this model. on: push: branches: [ colmodernvbert-fixes ] @@ -16,20 +18,12 @@ jobs: - windows-latest - ubuntu-latest runtime: - - { python: '3.10', onnxruntime: '1.18.0', numpy: 'numpy<2' } - { python: '3.10', onnxruntime: '1.23.2', numpy: 'numpy' } - - { python: '3.12', onnxruntime: '1.18.0', numpy: 'numpy<2' } - { python: '3.12', onnxruntime: '1.30.0', numpy: 'numpy' } - { python: '3.13', onnxruntime: '1.21.1', numpy: 'numpy' } - { python: '3.13', onnxruntime: '1.30.0', numpy: 'numpy' } - { python: '3.14', onnxruntime: '1.24.2', numpy: 'numpy' } - { python: '3.14', onnxruntime: '1.30.0', numpy: 'numpy' } - exclude: - # onnxruntime 1.18.x on Linux aborts while optimizing this graph, with the published model.onnx too - - os: ubuntu-latest - runtime: { python: '3.10', onnxruntime: '1.18.0', numpy: 'numpy<2' } - - os: ubuntu-latest - runtime: { python: '3.12', onnxruntime: '1.18.0', numpy: 'numpy<2' } runs-on: ${{ matrix.os }} @@ -50,28 +44,3 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: python -X faulthandler -u scripts/check_colmodernvbert_onnx.py --repo jmzzomg/colmodernvbert --revision main - # onnxruntime 1.18.0 crashes natively on Windows while loading the new file. Does the published file crash - # the same way (as both do on Linux), and does basic graph optimization avoid it? - probe-onnxruntime-1-18: - strategy: - fail-fast: false - matrix: - repo: [ jmzzomg/colmodernvbert, Qdrant/colmodernvbert ] - optimization: [ all, basic ] - - runs-on: windows-latest - - name: onnxruntime 1.18.0 loads ${{ matrix.repo }} with ${{ matrix.optimization }} optimization on windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.12' - - name: Install onnxruntime 1.18.0 - run: python -m pip install "onnxruntime==1.18.0" "numpy<2" huggingface_hub - - name: Load the model and run a text query - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: python -X faulthandler -u scripts/probe_ort_load.py ${{ matrix.repo }} ${{ matrix.optimization }} diff --git a/fastembed/late_interaction_multimodal/colmodernvbert.py b/fastembed/late_interaction_multimodal/colmodernvbert.py index bd215233..dbd90320 100644 --- a/fastembed/late_interaction_multimodal/colmodernvbert.py +++ b/fastembed/late_interaction_multimodal/colmodernvbert.py @@ -3,6 +3,7 @@ import json import numpy as np +import onnxruntime as ort from tokenizers import Encoding from PIL import Image @@ -126,6 +127,13 @@ def _list_supported_models(cls) -> list[DenseModelDescription]: return supported_colmodernvbert_models def load_onnx_model(self) -> None: + # onnxruntime 1.18 crashes the whole process while optimizing this graph (an access violation + # on Windows, a failed bounds check on Linux), and 1.17 can't read its IR version + if tuple(int(part) for part in ort.__version__.split(".")[:2]) < (1, 19): + raise RuntimeError( + f"Could not load {self.model_name}: it requires onnxruntime>=1.19, but " + f"onnxruntime {ort.__version__} is installed. Please upgrade onnxruntime." + ) self._load_onnx_model( model_dir=self._model_dir, model_file=self.model_description.model_file, diff --git a/scripts/probe_ort_load.py b/scripts/probe_ort_load.py deleted file mode 100644 index 5feda581..00000000 --- a/scripts/probe_ort_load.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Load a ColModernVBERT model.onnx with one graph optimization level and run a tiny text query. - -TEMPORARY, delete before merging. Usage: python -X faulthandler -u probe_ort_load.py -""" - -import sys - -import numpy as np -import onnxruntime as ort -from huggingface_hub import hf_hub_download - -repo, level = sys.argv[1], sys.argv[2] -levels = { - "all": ort.GraphOptimizationLevel.ORT_ENABLE_ALL, - "basic": ort.GraphOptimizationLevel.ORT_ENABLE_BASIC, -} -path = hf_hub_download(repo, "model.onnx") -print(f"onnxruntime {ort.__version__}, {repo}, optimization {level}: loading", flush=True) -options = ort.SessionOptions() -options.graph_optimization_level = levels[level] -session = ort.InferenceSession(path, options, providers=["CPUExecutionProvider"]) -print("loaded", flush=True) -ids = np.array([[50281, 25521, 1533, 50282]], dtype=np.int64) -output = session.run( - None, - { - "input_ids": ids, - "attention_mask": np.ones_like(ids), - "pixel_values": np.zeros((1, 1, 3, 512, 512), np.float32), - }, -)[0] -print(f"query ok, output {output.shape}", flush=True) From d509b046e0d75beec678e6cd4a0ca9ae3f2faa85 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Wed, 7 Oct 2026 01:02:40 +0700 Subject: [PATCH 05/10] fix: build ColModernVBERT image prompts from the actual tile grid Co-authored-by: Hulk --- fastembed/image/transform/operators.py | 29 +++++++++- .../colmodernvbert.py | 55 +++++++------------ 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/fastembed/image/transform/operators.py b/fastembed/image/transform/operators.py index 5c747b85..669cfcdc 100644 --- a/fastembed/image/transform/operators.py +++ b/fastembed/image/transform/operators.py @@ -191,7 +191,15 @@ def __init__( self.resample = resample def __call__(self, images: list[NumpyArray]) -> list[list[NumpyArray]]: # type: ignore[override] + patches, _ = self.split_with_grid(images) + return patches + + def split_with_grid( + self, images: list[NumpyArray] + ) -> tuple[list[list[NumpyArray]], list[tuple[int, int]]]: + """Return patches and their (rows, cols), or (0, 0) for unsplit images.""" result = [] + image_grid = [] for image in images: # Assume (C, H, W) format @@ -204,6 +212,7 @@ def __call__(self, images: list[NumpyArray]) -> list[list[NumpyArray]]: # type: # Calculate the number of splits needed num_splits_h = math.ceil(height / max_height) num_splits_w = math.ceil(width / max_width) + image_grid.append((num_splits_h, num_splits_w)) # Calculate optimal patch dimensions optimal_height = math.ceil(height / num_splits_h) @@ -234,12 +243,13 @@ def __call__(self, images: list[NumpyArray]) -> list[list[NumpyArray]]: # type: frames.append(global_view) else: # Image is small enough, no splitting needed + image_grid.append((0, 0)) frames.append(image) # Append (not extend) to preserve per-image grouping result.append(frames) - return result + return result, image_grid class SquareResize(Transform): @@ -272,10 +282,23 @@ def __init__(self, transforms: list[Transform]): self.transforms = transforms def __call__( - self, images: list[Image.Image] | list[NumpyArray] + self, + images: list[Image.Image] | list[NumpyArray], + *, + metadata: dict[str, Any] | None = None, ) -> list[NumpyArray] | list[Image.Image]: + """Apply transforms without changing their usual return type. + + If supplied, metadata["image_grid"] records (rows, cols) for each image, + or (0, 0) for images that are not split. + """ + if metadata is not None: + metadata["image_grid"] = [(0, 0) for _ in images] for transform in self.transforms: - images = transform(images) + if metadata is not None and isinstance(transform, ImageSplitter): + images, metadata["image_grid"] = transform.split_with_grid(images) # type: ignore[assignment,arg-type] + else: + images = transform(images) return images @classmethod diff --git a/fastembed/late_interaction_multimodal/colmodernvbert.py b/fastembed/late_interaction_multimodal/colmodernvbert.py index dbd90320..e5ef5c5b 100644 --- a/fastembed/late_interaction_multimodal/colmodernvbert.py +++ b/fastembed/late_interaction_multimodal/colmodernvbert.py @@ -231,11 +231,15 @@ def onnx_embed_image(self, images: list[ImageInput], **kwargs: Any) -> OnnxOutpu for image in images ] assert self.processor is not None, "Processor is not initialized" - processed = self.processor(image_files) + processor_metadata: dict[str, Any] = {} + processed = self.processor(image_files, metadata=processor_metadata) encoded, attention_mask, metadata = self._process_nested_patches(processed) # type: ignore[arg-type] + metadata.update(processor_metadata) onnx_input = {"pixel_values": encoded, "attention_mask": attention_mask} - onnx_input = self._preprocess_onnx_image_input(onnx_input, **kwargs) + onnx_input = self._preprocess_onnx_image_input( + onnx_input, image_grid=metadata["image_grid"], **kwargs + ) model_output = self.model.run(None, onnx_input) # type: ignore[union-attr] return OnnxOutputContext( @@ -285,17 +289,22 @@ def _process_nested_patches( return encoded, attention_mask, metadata # type: ignore[return-value] def _preprocess_onnx_image_input( - self, onnx_input: dict[str, np.ndarray], **kwargs: Any + self, + onnx_input: dict[str, np.ndarray], + *, + image_grid: list[tuple[int, int]] | None = None, + **kwargs: Any, ) -> dict[str, NumpyArray]: """ Add text input placeholders for image data, following Idefics3 processing logic. - Constructs input_ids dynamically based on the actual number of image patches, - using the same token expansion logic as Idefics3Processor. + Constructs input_ids from the actual image patch grid, using the same + token expansion logic as Idefics3Processor. Args: onnx_input: Dict with 'pixel_values' (batch, num_patches, C, H, W) and 'attention_mask' (batch, num_patches) indicating real patches + image_grid: Actual (rows, cols) from image splitting, (0, 0) if unsplit **kwargs: Additional arguments Returns: @@ -307,15 +316,18 @@ def _preprocess_onnx_image_input( pixel_values = onnx_input["pixel_values"] batch_size = pixel_values.shape[0] + if image_grid is None or len(image_grid) != batch_size: + raise ValueError("The image patch grid is required for each image in the batch") batch_input_ids = [] - # Build input_ids for each image based on its actual patch count - for i in range(batch_size): + # A patch count cannot distinguish portrait, landscape, and square grids. + for i, (rows, cols) in enumerate(image_grid): # Count real patches (non-padded) from attention mask patch_count = int(np.sum(patch_attention_mask[i])) - # Compute rows/cols from patch count - rows, cols = self._compute_rows_cols_from_patches(patch_count) + valid_grid = (rows == 0 and cols == 0) or (rows > 0 and cols > 0) + if not valid_grid or rows * cols + 1 != patch_count: + raise ValueError("The image patch grid does not match the number of patches") # Build input_ids for this image input_ids = self._build_input_ids_for_image(rows, cols) @@ -349,31 +361,6 @@ def _preprocess_onnx_image_input( onnx_input["attention_mask"] = attention_mask return onnx_input - @staticmethod - def _compute_rows_cols_from_patches(patch_count: int) -> tuple[int, int]: - if patch_count <= 1: - return 0, 0 - - # Subtract 1 for the global image - grid_patches = patch_count - 1 - - # Find rows and cols (assume square or near-square grid) - rows = int(grid_patches**0.5) - cols = grid_patches // rows - - # Verify the calculation - if rows * cols + 1 != patch_count: - # Handle non-square grids - for r in range(1, grid_patches + 1): - if grid_patches % r == 0: - c = grid_patches // r - if r * c + 1 == patch_count: - return r, c - # Fallback: treat as unsplit - return 0, 0 - - return rows, cols - def _create_single_image_prompt_string(self) -> str: return ( "" From 421b000aae3cc9d81748e835f570b0d0fc936b44 Mon Sep 17 00:00:00 2001 From: Li Haodong <1638272030@qq.com> Date: Wed, 7 Oct 2026 02:18:06 +0800 Subject: [PATCH 06/10] fix(image): keep longest-edge resize dimensions nonzero (#761) * fix(image): keep longest-edge resize dimensions nonzero * docs(image): document resize behavior and regression tests * tests: drop the extra tests and docstring from the nonzero resize fix --------- Co-authored-by: George Panchuk --- fastembed/image/transform/functional.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fastembed/image/transform/functional.py b/fastembed/image/transform/functional.py index e3b977db..bfe353bd 100644 --- a/fastembed/image/transform/functional.py +++ b/fastembed/image/transform/functional.py @@ -186,7 +186,8 @@ def resize_longest_edge( if new_width % 2 != 0: new_width += 1 - return image.resize((new_width, new_height), resample) + # A very narrow image can round down to zero along its shorter edge. + return image.resize((max(1, new_width), max(1, new_height)), resample) def crop_ndarray( From 0c5c4e3f3675d07c20848b7c9cf8547e77048a91 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Wed, 7 Oct 2026 02:01:13 +0700 Subject: [PATCH 07/10] fix: send ColModernVBERT text queries without image tiles and warn that the model was updated --- fastembed/late_interaction_multimodal/colmodernvbert.py | 5 +++-- .../late_interaction_multimodal_embedding.py | 9 +++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/fastembed/late_interaction_multimodal/colmodernvbert.py b/fastembed/late_interaction_multimodal/colmodernvbert.py index e5ef5c5b..35455421 100644 --- a/fastembed/late_interaction_multimodal/colmodernvbert.py +++ b/fastembed/late_interaction_multimodal/colmodernvbert.py @@ -177,9 +177,10 @@ def _preprocess_onnx_text_input( Returns: Iterable[NumpyArray]: Post-processed output as NumPy arrays. """ - batch_size, seq_length = onnx_input["input_ids"].shape + # no tiles: text has no image tokens, and the model skips the vision encoder without a tile + batch_size = onnx_input["input_ids"].shape[0] empty_image_placeholder: NumpyArray = np.zeros( - (batch_size, seq_length, 3, self.image_size, self.image_size), + (batch_size, 0, 3, self.image_size, self.image_size), dtype=np.float32, # type: ignore[type-var,arg-type,assignment] ) onnx_input["pixel_values"] = empty_image_placeholder diff --git a/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py b/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py index 4318409b..d58bff27 100644 --- a/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py +++ b/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py @@ -1,3 +1,4 @@ +import warnings from typing import Any, Iterable, Sequence, Type from dataclasses import asdict @@ -67,6 +68,14 @@ def __init__( **kwargs: Any, ): super().__init__(model_name, cache_dir, threads, **kwargs) + # keep for a couple of releases after the 2026-10 re-export of the model + if model_name.lower() == "qdrant/colmodernvbert": + warnings.warn( + "The model 'Qdrant/colmodernvbert' has been updated. Re-embed your images, and " + "delete a copy cached before the update, otherwise text queries fail.", + UserWarning, + stacklevel=2, + ) for EMBEDDING_MODEL_TYPE in self.EMBEDDINGS_REGISTRY: supported_models = EMBEDDING_MODEL_TYPE._list_supported_models() if any(model_name.lower() == model.model.lower() for model in supported_models): From e2b9f4c655ad20daf58ae724d53016f0d1263510 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Wed, 7 Oct 2026 02:04:33 +0700 Subject: [PATCH 08/10] tests: run the ColModernVBERT query test on all platforms and drop the temporary CI check --- .../workflows/colmodernvbert-onnx-check.yml | 46 ------ .../late_interaction_multimodal_embedding.py | 5 +- scripts/check_colmodernvbert_onnx.py | 144 ------------------ tests/test_late_interaction_multimodal.py | 4 - 4 files changed, 3 insertions(+), 196 deletions(-) delete mode 100644 .github/workflows/colmodernvbert-onnx-check.yml delete mode 100644 scripts/check_colmodernvbert_onnx.py diff --git a/.github/workflows/colmodernvbert-onnx-check.yml b/.github/workflows/colmodernvbert-onnx-check.yml deleted file mode 100644 index 1b998286..00000000 --- a/.github/workflows/colmodernvbert-onnx-check.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: ColModernVBERT ONNX runtime check - -# TEMPORARY, delete before merging: checks the re-exported model from the test repo -# jmzzomg/colmodernvbert with released fastembed 0.8.1 and the oldest and newest onnxruntime it allows on -# each Python version, on Windows and x86 Linux (the regular tests skip ColModernVBERT on Windows). -# onnxruntime 1.18 is left out: it crashes while loading this graph on Windows and Linux, with the -# published model.onnx too, and fastembed now refuses onnxruntime<1.19 for this model. -on: - push: - branches: [ colmodernvbert-fixes ] - -jobs: - check: - strategy: - fail-fast: false - matrix: - os: - - windows-latest - - ubuntu-latest - runtime: - - { python: '3.10', onnxruntime: '1.23.2', numpy: 'numpy' } - - { python: '3.12', onnxruntime: '1.30.0', numpy: 'numpy' } - - { python: '3.13', onnxruntime: '1.21.1', numpy: 'numpy' } - - { python: '3.13', onnxruntime: '1.30.0', numpy: 'numpy' } - - { python: '3.14', onnxruntime: '1.24.2', numpy: 'numpy' } - - { python: '3.14', onnxruntime: '1.30.0', numpy: 'numpy' } - - runs-on: ${{ matrix.os }} - - name: py${{ matrix.runtime.python }} onnxruntime ${{ matrix.runtime.onnxruntime }} on ${{ matrix.os }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: ${{ matrix.runtime.python }} - - name: Install released fastembed and the onnxruntime under test - run: | - python -m pip install "fastembed==0.8.1" - python -m pip install "onnxruntime==${{ matrix.runtime.onnxruntime }}" "${{ matrix.runtime.numpy }}" - - name: Check the candidate model - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: python -X faulthandler -u scripts/check_colmodernvbert_onnx.py --repo jmzzomg/colmodernvbert --revision main - diff --git a/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py b/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py index d58bff27..82bb9606 100644 --- a/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py +++ b/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py @@ -71,8 +71,9 @@ def __init__( # keep for a couple of releases after the 2026-10 re-export of the model if model_name.lower() == "qdrant/colmodernvbert": warnings.warn( - "The model 'Qdrant/colmodernvbert' has been updated. Re-embed your images, and " - "delete a copy cached before the update, otherwise text queries fail.", + "The model 'Qdrant/colmodernvbert' has been updated: re-embed images that were " + "embedded in batches with images of other aspect ratios. Delete a copy cached " + "before the update, otherwise text queries fail.", UserWarning, stacklevel=2, ) diff --git a/scripts/check_colmodernvbert_onnx.py b/scripts/check_colmodernvbert_onnx.py deleted file mode 100644 index dfc6e8cf..00000000 --- a/scripts/check_colmodernvbert_onnx.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Runtime check of a ColModernVBERT model.onnx with the installed fastembed and onnxruntime. - -Downloads the model from a Hugging Face revision (a branch that fastembed users never resolve), then checks: - 1. fastembed's canonical image and query values (atol 2e-3), through the installed fastembed's own code path - 2. text queries give the same output with one blank tile per token (fastembed <= 0.8.x), 1 tile and 0 tiles - 3. an image embedded inside a zero-padded batch equals the same image embedded alone -Exits 1 on any failure. - -Usage: python check_colmodernvbert_onnx.py --revision [--repo Qdrant/colmodernvbert] - python check_colmodernvbert_onnx.py --model-dir -""" - -import argparse -import platform -import sys -import time -from pathlib import Path - -import numpy as np -import onnxruntime as ort -from huggingface_hub import snapshot_download -from PIL import Image - -import fastembed -from fastembed import LateInteractionMultimodalEmbedding - -# tests/test_late_interaction_multimodal.py -CANONICAL_IMAGE = np.array( - [ - [0.11614, -0.15793, -0.11194, 0.0688, 0.08001, 0.10575, -0.07871], - [0.10094, -0.13301, -0.12069, 0.10932, 0.04645, 0.09884, 0.04048], - [0.13106, -0.18613, -0.13469, 0.10566, 0.03659, 0.07712, -0.03916], - [0.09754, -0.09596, -0.04839, 0.14991, 0.05692, 0.10569, -0.08349], - [0.02576, -0.15651, -0.09977, 0.09707, 0.13412, 0.09994, -0.09931], - [-0.06741, -0.1787, -0.19677, -0.07618, 0.13102, -0.02131, -0.02437], - [-0.02776, -0.10187, -0.13793, 0.03835, 0.04766, 0.04701, -0.15635], - ] -) -CANONICAL_QUERY = np.array( - [ - [0.05, 0.06557, 0.04026, 0.14981, 0.1842, 0.0263, -0.18706], - [-0.05664, -0.14028, 0.00649, -0.02849, 0.09034, -0.01494, 0.10693], - [-0.10147, -0.00716, 0.09084, -0.08236, -0.01849, -0.00972, -0.00461], - [-0.1233, -0.10814, -0.02337, -0.00329, 0.05984, 0.09934, 0.09846], - [-0.07053, -0.13119, -0.06487, 0.01508, 0.07459, 0.07655, 0.14821], - [0.00526, -0.13842, -0.05837, -0.02721, 0.13009, 0.05076, 0.17962], - [0.00924, -0.14383, -0.03057, -0.03691, 0.11718, 0.037, 0.13344], - ] -) -QUERIES = ["hello world", "flag embedding"] -TEST_IMAGE = Path(__file__).resolve().parents[1] / "tests" / "misc" / "image.jpeg" - -parser = argparse.ArgumentParser() -parser.add_argument("--repo", default="Qdrant/colmodernvbert") -group = parser.add_mutually_exclusive_group(required=True) -group.add_argument("--revision", help="Hugging Face branch or commit to download") -group.add_argument("--model-dir", help="local directory instead of a download") -args = parser.parse_args() - -model_dir = args.model_dir or snapshot_download( - args.repo, revision=args.revision, allow_patterns=["model.onnx", "*.json"] -) -print( - f"python {platform.python_version()} {platform.system()}-{platform.machine()}, onnxruntime {ort.__version__}, " - f"fastembed {fastembed.__version__}, model {args.model_dir or f'{args.repo}@{args.revision}'}" -) -failures = [] - - -def check(name, diff, tol, seconds): - status = "ok" if diff <= tol else "FAIL" - if status == "FAIL": - failures.append(name) - print(f" {name}: max diff {diff:.1e} (tol {tol:g}) {status} [{seconds:.2f}s]") - - -model = LateInteractionMultimodalEmbedding("Qdrant/colmodernvbert", specific_model_path=model_dir) -image = Image.open(TEST_IMAGE).convert("RGB") -wide = image.crop( - (0, 0, image.width, image.width // 4) -) # 5 tiles instead of 13, so batches get padding - -t = time.perf_counter() -single = list(model.embed_image([str(TEST_IMAGE)], batch_size=1))[0] -check( - "canonical image", - float(np.abs(single[:7, :7] - CANONICAL_IMAGE).max()), - 2e-3, - time.perf_counter() - t, -) - -t = time.perf_counter() -query = next(iter(model.embed_text(QUERIES))) -check( - "canonical query (installed fastembed's placeholder)", - float(np.abs(query[:7, :7] - CANONICAL_QUERY).max()), - 2e-3, - time.perf_counter() - t, -) - -t = time.perf_counter() -padded = list(model.embed_image([wide, image], batch_size=2)) -alone = list(model.embed_image([wide], batch_size=1))[0] - - -def real_rows( - x, -): # padding positions are zeroed by the attention mask; padding may be on either side - return x[np.abs(x).sum(-1) > 0] - - -diff = max( - float(np.abs(real_rows(padded[1]) - real_rows(single)).max()), - float(np.abs(real_rows(padded[0]) - real_rows(alone)).max()), -) -check("image in zero-padded batch == image alone", diff, 1e-4, time.perf_counter() - t) - -session = ort.InferenceSession( - str(Path(model_dir) / "model.onnx"), providers=["CPUExecutionProvider"] -) -encoded = model.model.tokenize(QUERIES) -ids = np.array([e.ids for e in encoded], dtype=np.int64) -mask = np.array([e.attention_mask for e in encoded], dtype=np.int64) -outs = {} -for tiles in (ids.shape[1], 1, 0): - t = time.perf_counter() - outs[tiles] = session.run( - None, - { - "input_ids": ids, - "attention_mask": mask, - "pixel_values": np.zeros((len(ids), tiles, 3, 512, 512), np.float32), - }, - )[0] - print(f" queries with {tiles} blank tiles per query: {time.perf_counter() - t:.2f}s") -check( - "queries: 1 tile vs one per token", float(np.abs(outs[1] - outs[ids.shape[1]]).max()), 1e-5, 0 -) -check( - "queries: 0 tiles vs one per token", float(np.abs(outs[0] - outs[ids.shape[1]]).max()), 1e-5, 0 -) - -print("FAILED: " + ", ".join(failures) if failures else "ALL PASSED") -sys.exit(1 if failures else 0) diff --git a/tests/test_late_interaction_multimodal.py b/tests/test_late_interaction_multimodal.py index bb50eae6..682750af 100644 --- a/tests/test_late_interaction_multimodal.py +++ b/tests/test_late_interaction_multimodal.py @@ -154,10 +154,6 @@ def test_batch_embedding_mixed_tile_counts(model_cache): def test_single_embedding_query(model_cache): - # text inputs get a zero image placeholder per token, so this is as slow as the image tests - if platform.system() != "Linux" and os.getenv("CI"): - pytest.skip("too slow on macOS and Windows CI runners") - for model_name, expected_result in CANONICAL_QUERY_VALUES.items(): if model_name.lower() == "Qdrant/colpali-v1.3-fp16".lower() and os.getenv("CI"): continue # colpali is too large for ci From f873fcf09a775a72d66a5d42ccc96729021390a5 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Wed, 7 Oct 2026 12:51:33 +0700 Subject: [PATCH 09/10] fix: switch ColModernVBERT to model_v2.onnx, avoid an empty tensor on CUDA, drop padding rows and lower the default image batch size --- .../colmodernvbert.py | 31 ++++++++++++------- .../late_interaction_multimodal_embedding.py | 5 ++- tests/test_late_interaction_multimodal.py | 3 -- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/fastembed/late_interaction_multimodal/colmodernvbert.py b/fastembed/late_interaction_multimodal/colmodernvbert.py index 35455421..4a2fbc47 100644 --- a/fastembed/late_interaction_multimodal/colmodernvbert.py +++ b/fastembed/late_interaction_multimodal/colmodernvbert.py @@ -31,7 +31,7 @@ # TEMPORARY: test copy of the re-exported model, switch back to Qdrant/colmodernvbert before merging sources=ModelSource(hf="jmzzomg/colmodernvbert"), additional_files=["processor_config.json"], - model_file="model.onnx", + model_file="model_v2.onnx", ), ] @@ -177,13 +177,13 @@ def _preprocess_onnx_text_input( Returns: Iterable[NumpyArray]: Post-processed output as NumPy arrays. """ - # no tiles: text has no image tokens, and the model skips the vision encoder without a tile - batch_size = onnx_input["input_ids"].shape[0] - empty_image_placeholder: NumpyArray = np.zeros( - (batch_size, 0, 3, self.image_size, self.image_size), + # one blank tile for the whole batch: the model drops blank tiles and skips the vision + # encoder; zero tiles would also skip it, but CUDA can't reduce the resulting empty tensor + blank_image_placeholder: NumpyArray = np.zeros( + (1, 1, 3, self.image_size, self.image_size), dtype=np.float32, # type: ignore[type-var,arg-type,assignment] ) - onnx_input["pixel_values"] = empty_image_placeholder + onnx_input["pixel_values"] = blank_image_placeholder return onnx_input def _post_process_onnx_text_output( @@ -199,7 +199,10 @@ def _post_process_onnx_text_output( Returns: Iterable[NumpyArray]: Post-processed output as NumPy arrays. """ - return output.model_output + assert output.attention_mask is not None + # drop the padding rows, so a query gets the same vectors whatever else is in its batch + for embedding, attention_mask in zip(output.model_output, output.attention_mask): + yield embedding[attention_mask == 1] def tokenize(self, documents: list[str], **kwargs: Any) -> list[Encoding]: # Add query augmentation tokens (matching process_queries logic from colpali-engine) @@ -245,7 +248,8 @@ def onnx_embed_image(self, images: list[ImageInput], **kwargs: Any) -> OnnxOutpu return OnnxOutputContext( model_output=model_output[0], - attention_mask=attention_mask, # type: ignore[arg-type] + # the token mask, not the tile mask: post-processing drops the padding rows with it + attention_mask=onnx_input["attention_mask"], # type: ignore[arg-type] metadata=metadata, ) @@ -424,9 +428,13 @@ def _post_process_onnx_image_output( Iterable[NumpyArray]: Post-processed output as NumPy arrays. """ assert self.model_description.dim is not None, "Model dim is not defined" - return output.model_output.reshape( + assert output.attention_mask is not None + embeddings: NumpyArray = output.model_output.reshape( output.model_output.shape[0], -1, self.model_description.dim ) + # drop the padding rows, so an image gets the same vectors whatever else is in its batch + for embedding, attention_mask in zip(embeddings, output.attention_mask): + yield embedding[attention_mask == 1] def embed_text( self, @@ -467,7 +475,7 @@ def embed_text( def embed_image( self, images: ImageInput | Iterable[ImageInput], - batch_size: int = 16, + batch_size: int = 2, parallel: Optional[int] = None, **kwargs: Any, ) -> Iterable[NumpyArray]: @@ -476,7 +484,8 @@ def embed_image( Args: images: Iterator of image paths or single image path to embed - batch_size: Batch size for encoding -- higher values will use more memory, but be faster + batch_size: Batch size for encoding -- every image is split into up to 17 tiles, so + memory grows quickly with it, while on CPU larger batches are not faster parallel: If > 1, data-parallel encoding will be used, recommended for offline encoding of large datasets. If 0, use all available cores. diff --git a/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py b/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py index 82bb9606..689b646f 100644 --- a/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py +++ b/fastembed/late_interaction_multimodal/late_interaction_multimodal_embedding.py @@ -72,8 +72,7 @@ def __init__( if model_name.lower() == "qdrant/colmodernvbert": warnings.warn( "The model 'Qdrant/colmodernvbert' has been updated: re-embed images that were " - "embedded in batches with images of other aspect ratios. Delete a copy cached " - "before the update, otherwise text queries fail.", + "embedded in batches with images of other aspect ratios.", UserWarning, stacklevel=2, ) @@ -157,7 +156,7 @@ def embed_text( def embed_image( self, images: ImageInput | Iterable[ImageInput], - batch_size: int = 16, + batch_size: int = 2, parallel: int | None = None, **kwargs: Any, ) -> Iterable[NumpyArray]: diff --git a/tests/test_late_interaction_multimodal.py b/tests/test_late_interaction_multimodal.py index 682750af..78e7f874 100644 --- a/tests/test_late_interaction_multimodal.py +++ b/tests/test_late_interaction_multimodal.py @@ -146,9 +146,6 @@ def test_batch_embedding_mixed_tile_counts(model_cache): single_result = list(model.embed_image(mixed_images, batch_size=1)) for batch_value, single_value in zip(batch_result, single_result): - # padding tokens are zeroed out - batch_value = batch_value[np.any(batch_value, axis=-1)] - single_value = single_value[np.any(single_value, axis=-1)] assert batch_value.shape == single_value.shape assert np.allclose(batch_value, single_value, atol=1e-3) From f6a2686c3bb3cadc37846a9ccbf9c01dd0fea909 Mon Sep 17 00:00:00 2001 From: George Panchuk Date: Wed, 7 Oct 2026 13:03:56 +0700 Subject: [PATCH 10/10] fix: set qdrant hf account as model source for colmodernvbert --- fastembed/late_interaction_multimodal/colmodernvbert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastembed/late_interaction_multimodal/colmodernvbert.py b/fastembed/late_interaction_multimodal/colmodernvbert.py index 4a2fbc47..f64b14fa 100644 --- a/fastembed/late_interaction_multimodal/colmodernvbert.py +++ b/fastembed/late_interaction_multimodal/colmodernvbert.py @@ -29,7 +29,7 @@ license="mit", size_in_GB=1.0, # TEMPORARY: test copy of the re-exported model, switch back to Qdrant/colmodernvbert before merging - sources=ModelSource(hf="jmzzomg/colmodernvbert"), + sources=ModelSource(hf="Qdrant/colmodernvbert"), additional_files=["processor_config.json"], model_file="model_v2.onnx", ),