From a3808bc3912d431ee644ee164197371fe5b11bd1 Mon Sep 17 00:00:00 2001 From: Hulk Date: Thu, 8 Oct 2026 09:09:29 -0700 Subject: [PATCH 1/2] fix: recover from malformed Hugging Face cache metadata --- fastembed/common/model_management.py | 14 +- tests/test_model_metadata_recovery.py | 213 ++++++++++++++++++++++++++ 2 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 tests/test_model_metadata_recovery.py diff --git a/fastembed/common/model_management.py b/fastembed/common/model_management.py index 583bef5a..942f7cc7 100644 --- a/fastembed/common/model_management.py +++ b/fastembed/common/model_management.py @@ -290,6 +290,16 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]]) except (OSError, ValueError) as e: logger.warning(f"Error saving metadata: {str(e)}") + def _read_file_metadata(metadata_file: Path) -> dict[str, Any]: + try: + return json.loads(metadata_file.read_text()) + except json.JSONDecodeError as e: + # An interrupted write can leave metadata incomplete even when the model + # files are intact. Treat it as absent: offline loading can still try the + # snapshot, and an online download will collect and verify fresh metadata. + logger.warning(f"Error reading metadata from {metadata_file}: {e}") + return {} + allow_patterns = [ "config.json", "tokenizer.json", @@ -327,7 +337,7 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]]) ) metadata_file = snapshot_dir / cls.METADATA_FILE if metadata_file.exists(): - metadata = json.loads(metadata_file.read_text()) + metadata = _read_file_metadata(metadata_file) if not _verify_files_from_metadata(snapshot_dir, metadata, repo_files=[]): # A size mismatch on one of this model's own files (its weights, or # whatever else its description lists) means loading would later fail @@ -404,7 +414,7 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]]) # a forced download re-fetches every file, so show progress and re-collect the metadata if not kwargs.get("force_download") and snapshot_dir.exists() and metadata_file.exists(): - metadata = json.loads(metadata_file.read_text()) + metadata = _read_file_metadata(metadata_file) # metadata lacking a requested file, e.g. a new model_file, can't vouch for it requested_files = {f.path for f in repo_files if f.path in allow_patterns} stored_files = {_repo_relative_path(Path(rel_path)) for rel_path in metadata} diff --git a/tests/test_model_metadata_recovery.py b/tests/test_model_metadata_recovery.py new file mode 100644 index 00000000..f29458da --- /dev/null +++ b/tests/test_model_metadata_recovery.py @@ -0,0 +1,213 @@ +"""Malformed cache metadata must not prevent normal snapshot resolution and repair.""" + +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from huggingface_hub.hf_api import RepoFile + +from fastembed.common import model_management +from fastembed.common.model_description import BaseModelDescription, ModelSource +from fastembed.common.model_management import CorruptedCacheError, ModelManagement + +REVISION = "0123456789abcdef0123456789abcdef01234567" +REPO = "qdrant/fake-onnx" +PAYLOAD = b"intact dummy model" + + +@pytest.fixture +def cached_model(tmp_path: Path) -> tuple[Path, Path, dict[str, dict[str, int | str]]]: + repo_dir = tmp_path / "models--qdrant--fake-onnx" + snapshot = repo_dir / "snapshots" / REVISION + snapshot.mkdir(parents=True) + model_file = snapshot / "model.onnx" + model_file.write_bytes(PAYLOAD) + metadata = { + str(model_file.relative_to(repo_dir)): {"size": len(PAYLOAD), "blob_id": "modelblob"} + } + metadata_file = repo_dir / ModelManagement.METADATA_FILE + metadata_file.write_text(json.dumps(metadata)) + return snapshot, metadata_file, metadata + + +@pytest.mark.parametrize("metadata_text", ["", "{"]) +@pytest.mark.parametrize("local_files_only", [False, True]) +def test_download_model_reuses_cache_with_malformed_metadata( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], + metadata_text: str, + local_files_only: bool, +) -> None: + snapshot, metadata_file, _ = cached_model + metadata_file.write_text(metadata_text) + download = Mock(return_value=str(snapshot)) + info = Mock(side_effect=AssertionError("The cached model needs no hub request")) + tree = Mock(side_effect=AssertionError("The cached model needs no hub request")) + monkeypatch.setattr(model_management, "snapshot_download", download) + monkeypatch.setattr(model_management, "model_info", info) + monkeypatch.setattr(model_management, "list_repo_tree", tree) + sleep = Mock() + monkeypatch.setattr(model_management.time, "sleep", sleep) + monkeypatch.delenv("HF_HUB_OFFLINE", raising=False) + model = BaseModelDescription( + model="test/fake", + sources=ModelSource(hf=REPO), + model_file="model.onnx", + description="test", + license="test", + size_in_GB=0.0, + ) + + assert ( + ModelManagement.download_model(model, str(tmp_path), local_files_only=local_files_only) + == snapshot + ) + download.assert_called_once() + assert download.call_args.kwargs["local_files_only"] is True + assert not download.call_args.kwargs.get("force_download", False) + info.assert_not_called() + tree.assert_not_called() + sleep.assert_not_called() + assert (snapshot / "model.onnx").read_bytes() == PAYLOAD + # Offline reuse does not manufacture authoritative metadata. + assert metadata_file.read_text() == metadata_text + + +@pytest.mark.parametrize("metadata_text", ["", "{"]) +@pytest.mark.parametrize("bad_size", [False, True]) +def test_online_download_rebuilds_malformed_metadata_and_checks_sizes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], + metadata_text: str, + bad_size: bool, +) -> None: + snapshot, metadata_file, metadata = cached_model + metadata_file.write_text(metadata_text) + old_file = snapshot.parent / "older-revision" / "model.onnx" + old_file.parent.mkdir() + old_file.write_bytes(b"old") + hub_file = RepoFile(path="model.onnx", size=len(PAYLOAD) + int(bad_size), oid="modelblob") + download = Mock(return_value=str(snapshot)) + info = Mock(return_value=SimpleNamespace(sha=REVISION)) + tree = Mock(return_value=[hub_file]) + monkeypatch.setattr(model_management, "snapshot_download", download) + monkeypatch.setattr(model_management, "model_info", info) + monkeypatch.setattr(model_management, "list_repo_tree", tree) + + def load() -> str: + return ModelManagement.download_files_from_huggingface(REPO, str(tmp_path), ["model.onnx"]) + + if bad_size: + with pytest.raises(ValueError, match="corrupted during downloading"): + load() + assert metadata_file.read_text() == metadata_text + else: + assert load() == str(snapshot) + assert json.loads(metadata_file.read_text()) == metadata + info.assert_called_once_with(REPO, timeout=model_management.constants.HF_HUB_ETAG_TIMEOUT) + tree.assert_called_once_with(REPO, revision=REVISION, repo_type="model", recursive=True) + download.assert_called_once() + assert download.call_args.kwargs["local_files_only"] is False + assert not download.call_args.kwargs.get("force_download", False) + assert (snapshot / "model.onnx").read_bytes() == PAYLOAD + assert old_file.read_bytes() == b"old" + + +def test_malformed_canonical_metadata_does_not_block_legacy_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], +) -> None: + snapshot, _, _ = cached_model + canonical_dir = tmp_path / "models--Qdrant--fake-onnx" + if canonical_dir.exists(): + pytest.skip("Distinct repository casing requires a case-sensitive filesystem") + canonical_dir.mkdir() + (canonical_dir / ModelManagement.METADATA_FILE).write_text("{") + download = Mock(side_effect=[FileNotFoundError("No canonical snapshot"), str(snapshot)]) + info = Mock(side_effect=AssertionError("Offline loading must not reach the hub")) + tree = Mock(side_effect=AssertionError("Offline loading must not reach the hub")) + monkeypatch.setattr(model_management, "snapshot_download", download) + monkeypatch.setattr(model_management, "model_info", info) + monkeypatch.setattr(model_management, "list_repo_tree", tree) + + assert ModelManagement.download_files_from_huggingface( + "Qdrant/fake-onnx", str(tmp_path), ["model.onnx"], local_files_only=True + ) == str(snapshot) + assert [call.kwargs["repo_id"] for call in download.call_args_list] == [ + "Qdrant/fake-onnx", + REPO, + ] + assert all(call.kwargs["local_files_only"] for call in download.call_args_list) + info.assert_not_called() + tree.assert_not_called() + + +@pytest.mark.parametrize("error", [PermissionError("denied"), ValueError("unrelated error")]) +def test_metadata_reader_does_not_swallow_unrelated_errors( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], + error: Exception, +) -> None: + def fail_read(*args: object, **kwargs: object) -> str: + raise error + + monkeypatch.setattr(Path, "read_text", fail_read) + with pytest.raises(type(error)) as exc: + ModelManagement.download_files_from_huggingface( + REPO, str(tmp_path), ["model.onnx"], local_files_only=True + ) + assert exc.value is error + + +@pytest.mark.parametrize("metadata_text", ["[]", "null", '"invalid schema"']) +def test_valid_json_with_wrong_root_type_is_not_silently_accepted( + tmp_path: Path, + cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], + metadata_text: str, +) -> None: + _, metadata_file, _ = cached_model + metadata_file.write_text(metadata_text) + with pytest.raises(AttributeError): + ModelManagement.download_files_from_huggingface( + REPO, str(tmp_path), ["model.onnx"], local_files_only=True + ) + + +def test_valid_metadata_still_rejects_corrupt_model( + tmp_path: Path, + cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], +) -> None: + snapshot, _, _ = cached_model + (snapshot / "model.onnx").write_bytes(b"short") + with pytest.raises(CorruptedCacheError): + ModelManagement.download_files_from_huggingface( + REPO, str(tmp_path), ["model.onnx"], local_files_only=True + ) + + +@pytest.mark.parametrize("local_files_only", [False, True]) +def test_hub_failure_with_malformed_metadata_is_preserved( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], + local_files_only: bool, +) -> None: + _, metadata_file, _ = cached_model + metadata_file.write_text("{") + error = OSError("Hub unavailable") + monkeypatch.setattr(model_management, "snapshot_download", Mock(side_effect=error)) + monkeypatch.setattr( + model_management, "model_info", Mock(return_value=SimpleNamespace(sha=REVISION)) + ) + monkeypatch.setattr(model_management, "list_repo_tree", Mock(return_value=[])) + with pytest.raises(OSError) as exc: + ModelManagement.download_files_from_huggingface( + REPO, str(tmp_path), ["model.onnx"], local_files_only=local_files_only + ) + assert exc.value is error From 83dc69bb60d40371f03ea9a2bacad0e20320f031 Mon Sep 17 00:00:00 2001 From: Hulk Date: Thu, 8 Oct 2026 09:35:59 -0700 Subject: [PATCH 2/2] fix: recover from invalid cache metadata encoding --- fastembed/common/model_management.py | 8 ++++---- tests/test_model_metadata_recovery.py | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/fastembed/common/model_management.py b/fastembed/common/model_management.py index 942f7cc7..c949f960 100644 --- a/fastembed/common/model_management.py +++ b/fastembed/common/model_management.py @@ -293,10 +293,10 @@ def _save_file_metadata(model_dir: Path, meta: dict[str, dict[str, int | str]]) def _read_file_metadata(metadata_file: Path) -> dict[str, Any]: try: return json.loads(metadata_file.read_text()) - except json.JSONDecodeError as e: - # An interrupted write can leave metadata incomplete even when the model - # files are intact. Treat it as absent: offline loading can still try the - # snapshot, and an online download will collect and verify fresh metadata. + except (json.JSONDecodeError, UnicodeDecodeError) as e: + # Corruption can leave metadata unreadable even when the model files are intact. + # Treat it as absent: offline loading can still try the snapshot, and an online + # download will collect and verify fresh metadata. logger.warning(f"Error reading metadata from {metadata_file}: {e}") return {} diff --git a/tests/test_model_metadata_recovery.py b/tests/test_model_metadata_recovery.py index f29458da..4d025982 100644 --- a/tests/test_model_metadata_recovery.py +++ b/tests/test_model_metadata_recovery.py @@ -32,17 +32,17 @@ def cached_model(tmp_path: Path) -> tuple[Path, Path, dict[str, dict[str, int | return snapshot, metadata_file, metadata -@pytest.mark.parametrize("metadata_text", ["", "{"]) +@pytest.mark.parametrize("metadata_bytes", [b"", b"{", b"\xff"]) @pytest.mark.parametrize("local_files_only", [False, True]) def test_download_model_reuses_cache_with_malformed_metadata( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], - metadata_text: str, + metadata_bytes: bytes, local_files_only: bool, ) -> None: snapshot, metadata_file, _ = cached_model - metadata_file.write_text(metadata_text) + metadata_file.write_bytes(metadata_bytes) download = Mock(return_value=str(snapshot)) info = Mock(side_effect=AssertionError("The cached model needs no hub request")) tree = Mock(side_effect=AssertionError("The cached model needs no hub request")) @@ -73,20 +73,20 @@ def test_download_model_reuses_cache_with_malformed_metadata( sleep.assert_not_called() assert (snapshot / "model.onnx").read_bytes() == PAYLOAD # Offline reuse does not manufacture authoritative metadata. - assert metadata_file.read_text() == metadata_text + assert metadata_file.read_bytes() == metadata_bytes -@pytest.mark.parametrize("metadata_text", ["", "{"]) +@pytest.mark.parametrize("metadata_bytes", [b"", b"{", b"\xff"]) @pytest.mark.parametrize("bad_size", [False, True]) def test_online_download_rebuilds_malformed_metadata_and_checks_sizes( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], - metadata_text: str, + metadata_bytes: bytes, bad_size: bool, ) -> None: snapshot, metadata_file, metadata = cached_model - metadata_file.write_text(metadata_text) + metadata_file.write_bytes(metadata_bytes) old_file = snapshot.parent / "older-revision" / "model.onnx" old_file.parent.mkdir() old_file.write_bytes(b"old") @@ -104,7 +104,7 @@ def load() -> str: if bad_size: with pytest.raises(ValueError, match="corrupted during downloading"): load() - assert metadata_file.read_text() == metadata_text + assert metadata_file.read_bytes() == metadata_bytes else: assert load() == str(snapshot) assert json.loads(metadata_file.read_text()) == metadata @@ -117,17 +117,19 @@ def load() -> str: assert old_file.read_bytes() == b"old" +@pytest.mark.parametrize("metadata_bytes", [b"{", b"\xff"]) def test_malformed_canonical_metadata_does_not_block_legacy_cache( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cached_model: tuple[Path, Path, dict[str, dict[str, int | str]]], + metadata_bytes: bytes, ) -> None: snapshot, _, _ = cached_model canonical_dir = tmp_path / "models--Qdrant--fake-onnx" if canonical_dir.exists(): pytest.skip("Distinct repository casing requires a case-sensitive filesystem") canonical_dir.mkdir() - (canonical_dir / ModelManagement.METADATA_FILE).write_text("{") + (canonical_dir / ModelManagement.METADATA_FILE).write_bytes(metadata_bytes) download = Mock(side_effect=[FileNotFoundError("No canonical snapshot"), str(snapshot)]) info = Mock(side_effect=AssertionError("Offline loading must not reach the hub")) tree = Mock(side_effect=AssertionError("Offline loading must not reach the hub"))