diff --git a/codenib/web/config.py b/codenib/web/config.py index 4eac15d2..4b3db6e4 100644 --- a/codenib/web/config.py +++ b/codenib/web/config.py @@ -147,6 +147,15 @@ class QAConfig: wiki_media_api_base: Optional[str] = None wiki_media_api_key: Optional[str] = field(default=None, repr=False) wiki_media_options: Dict[str, Any] = field(default_factory=dict) + # Optional OpenAI-compatible VLM endpoint for extracting structured visual + # facts from repository-owned images/diagrams before grounding them to code. + # Disabled by default so local/offline builds keep using deterministic + # metadata extraction. + wiki_visual_facts_enabled: bool = False + wiki_visual_facts_model: Optional[str] = None + wiki_visual_facts_api_base: Optional[str] = None + wiki_visual_facts_api_key: Optional[str] = field(default=None, repr=False) + wiki_visual_facts_options: Dict[str, Any] = field(default_factory=dict) # Optional OpenAI-compatible endpoint for the Ask agent. Provider-native # models (for example Vertex or Anthropic) normally leave these unset. model_api_base: Optional[str] = None @@ -224,6 +233,10 @@ def __post_init__(self) -> None: self.wiki_media_options, source="wiki_media_options", ) + self.wiki_visual_facts_options = validate_model_options( + self.wiki_visual_facts_options, + source="wiki_visual_facts_options", + ) def index_types(self) -> List[str]: return ["bm25", "vector"] if self.mode == "hybrid" else ["bm25"] @@ -297,6 +310,16 @@ def wiki_media_generation_enabled(self) -> bool: return True return bool(self.wiki_media_model and self.wiki_media_api_base) + @property + def wiki_visual_fact_extraction_enabled(self) -> bool: + """Whether repository media should be sent to a configured VLM.""" + + return bool( + self.wiki_visual_facts_enabled + and self.wiki_visual_facts_model + and self.wiki_visual_facts_api_base + ) + def load_config(path: Optional[str] = None) -> QAConfig: """Load a layered demo config from YAML, then apply env overrides.""" @@ -316,6 +339,19 @@ def load_config(path: Optional[str] = None) -> QAConfig: data.get("wiki_media_options"), source="wiki_media_options", ), + wiki_visual_facts_enabled=bool( + data.get( + "wiki_visual_facts_enabled", + defaults.wiki_visual_facts_enabled, + ) + ), + wiki_visual_facts_model=data.get("wiki_visual_facts_model"), + wiki_visual_facts_api_base=data.get("wiki_visual_facts_api_base"), + wiki_visual_facts_api_key=data.get("wiki_visual_facts_api_key"), + wiki_visual_facts_options=validate_model_options( + data.get("wiki_visual_facts_options"), + source="wiki_visual_facts_options", + ), model_api_base=data.get("model_api_base"), model_api_key=data.get("model_api_key"), model_options=validate_model_options( @@ -374,6 +410,18 @@ def load_config(path: Optional[str] = None) -> QAConfig: cfg.wiki_media_api_base = os.environ["CODENIB_WIKI_MEDIA_API_BASE"] if os.environ.get("CODENIB_WIKI_MEDIA_API_KEY"): cfg.wiki_media_api_key = os.environ["CODENIB_WIKI_MEDIA_API_KEY"] + if os.environ.get("CODENIB_WIKI_VISUAL_FACTS_ENABLED") is not None: + cfg.wiki_visual_facts_enabled = os.environ[ + "CODENIB_WIKI_VISUAL_FACTS_ENABLED" + ].strip().lower() in ("1", "true", "yes", "on") + if os.environ.get("CODENIB_WIKI_VISUAL_FACTS_MODEL"): + cfg.wiki_visual_facts_model = os.environ["CODENIB_WIKI_VISUAL_FACTS_MODEL"] + if os.environ.get("CODENIB_WIKI_VISUAL_FACTS_API_BASE"): + cfg.wiki_visual_facts_api_base = os.environ[ + "CODENIB_WIKI_VISUAL_FACTS_API_BASE" + ] + if os.environ.get("CODENIB_WIKI_VISUAL_FACTS_API_KEY"): + cfg.wiki_visual_facts_api_key = os.environ["CODENIB_WIKI_VISUAL_FACTS_API_KEY"] if os.environ.get("CODENIB_DEMO_API_BASE"): cfg.model_api_base = os.environ["CODENIB_DEMO_API_BASE"] if os.environ.get("CODENIB_DEMO_API_KEY"): @@ -402,6 +450,14 @@ def load_config(path: Optional[str] = None) -> QAConfig: source="CODENIB_WIKI_MEDIA_OPTIONS", ), ) + if os.environ.get("CODENIB_WIKI_VISUAL_FACTS_OPTIONS"): + cfg.wiki_visual_facts_options = merge_model_options( + cfg.wiki_visual_facts_options, + parse_model_options_json( + os.environ["CODENIB_WIKI_VISUAL_FACTS_OPTIONS"], + source="CODENIB_WIKI_VISUAL_FACTS_OPTIONS", + ), + ) if os.environ.get("CODENIB_DEMO_DATA_DIR"): cfg.data_dir = os.environ["CODENIB_DEMO_DATA_DIR"] if os.environ.get("CODENIB_DEMO_PREBUILT_DIR"): diff --git a/codenib/wiki/__init__.py b/codenib/wiki/__init__.py index 89ba32d0..0be0b9a2 100644 --- a/codenib/wiki/__init__.py +++ b/codenib/wiki/__init__.py @@ -35,16 +35,30 @@ search_visual_context, ) from .media_pipeline import build_multimodal_repository_knowledge +from .media_storage import ( + MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA, + MULTIMODAL_KNOWLEDGE_BUNDLE_VERSION, + build_multimodal_knowledge_bundle, + load_multimodal_knowledge_bundle, + save_multimodal_knowledge_bundle, + validate_multimodal_knowledge_bundle, +) from .media_tools import MULTIMODAL_TOOL_SCHEMAS, MultimodalKnowledgeToolRouter -from .media_vlm import OpenAICompatibleVisualFactExtractor +from .media_vlm import ( + OpenAICompatibleVisualFactExtractor, + visual_fact_extractor_from_config, +) __all__ = [ "MULTIMODAL_TOOL_SCHEMAS", + "MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA", + "MULTIMODAL_KNOWLEDGE_BUNDLE_VERSION", "MultimodalKnowledgeToolRouter", "WikiBuilder", "OpenAICompatibleVisualFactExtractor", "build_media_evidence_pack", "build_multimodal_knowledge_view", + "build_multimodal_knowledge_bundle", "build_multimodal_repository_knowledge", "build_visual_facts_manifest", "deterministic_visual_facts", @@ -59,5 +73,9 @@ "ground_visual_facts_to_sources", "merge_incremental_visual_facts", "plan_incremental_visual_fact_update", + "load_multimodal_knowledge_bundle", + "save_multimodal_knowledge_bundle", "search_visual_context", + "validate_multimodal_knowledge_bundle", + "visual_fact_extractor_from_config", ] diff --git a/codenib/wiki/media_pipeline.py b/codenib/wiki/media_pipeline.py index e9087d43..b87a5959 100644 --- a/codenib/wiki/media_pipeline.py +++ b/codenib/wiki/media_pipeline.py @@ -20,6 +20,7 @@ ground_visual_facts_to_sources, ) from .media_knowledge import build_multimodal_knowledge_view +from .media_storage import build_multimodal_knowledge_bundle def build_multimodal_repository_knowledge( @@ -60,13 +61,13 @@ def build_multimodal_repository_knowledge( visual_facts_manifest, grounding_manifest, ) - return { - "media_manifest": media_manifest, - "visual_facts_manifest": visual_facts_manifest, - "source_candidate_count": len(source_candidates), - "grounding_manifest": grounding_manifest, - "knowledge_view": knowledge_view, - } + return build_multimodal_knowledge_bundle( + media_manifest=media_manifest, + visual_facts_manifest=visual_facts_manifest, + source_candidate_count=len(source_candidates), + grounding_manifest=grounding_manifest, + knowledge_view=knowledge_view, + ) __all__ = ["build_multimodal_repository_knowledge"] diff --git a/codenib/wiki/media_storage.py b/codenib/wiki/media_storage.py new file mode 100644 index 00000000..a49ebccc --- /dev/null +++ b/codenib/wiki/media_storage.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +"""Stable storage helpers for multimodal repository knowledge bundles.""" + +from __future__ import annotations + +import json +import os +import tempfile +from hashlib import sha256 +from pathlib import Path +from typing import Any, Mapping + +MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA = "codenib.multimodal-knowledge-bundle.v1" +MULTIMODAL_KNOWLEDGE_BUNDLE_VERSION = 1 +_MAX_BUNDLE_BYTES = 128 * 1024 * 1024 + + +def build_multimodal_knowledge_bundle( + *, + media_manifest: Mapping[str, Any], + visual_facts_manifest: Mapping[str, Any], + source_candidate_count: int, + grounding_manifest: Mapping[str, Any], + knowledge_view: Mapping[str, Any], +) -> dict[str, Any]: + """Wrap multimodal pipeline outputs in a versioned, hashable bundle.""" + + bundle: dict[str, Any] = { + "schema": MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA, + "schema_version": MULTIMODAL_KNOWLEDGE_BUNDLE_VERSION, + "media_manifest": dict(media_manifest), + "visual_facts_manifest": dict(visual_facts_manifest), + "source_candidate_count": int(source_candidate_count), + "grounding_manifest": dict(grounding_manifest), + "knowledge_view": dict(knowledge_view), + "component_sha256": { + "media_manifest": str(media_manifest.get("manifest_sha256") or ""), + "visual_facts_manifest": str( + visual_facts_manifest.get("manifest_sha256") or "" + ), + "grounding_manifest": str(grounding_manifest.get("manifest_sha256") or ""), + "knowledge_view": str(knowledge_view.get("view_sha256") or ""), + }, + } + bundle["bundle_sha256"] = _stable_sha256( + {key: value for key, value in bundle.items() if key != "bundle_sha256"} + ) + return bundle + + +def save_multimodal_knowledge_bundle( + bundle: Mapping[str, Any], + path: str | Path, +) -> None: + """Atomically write a multimodal knowledge bundle as stable JSON.""" + + validated = validate_multimodal_knowledge_bundle(bundle) + destination = Path(path) + destination.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp( + prefix=f".{destination.name}.", + suffix=".tmp", + dir=str(destination.parent), + text=True, + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(validated, handle, ensure_ascii=False, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temp_name, destination) + finally: + try: + os.unlink(temp_name) + except FileNotFoundError: + pass + + +def load_multimodal_knowledge_bundle(path: str | Path) -> dict[str, Any]: + """Load and validate a persisted multimodal knowledge bundle.""" + + source = Path(path) + size = source.stat().st_size + if size > _MAX_BUNDLE_BYTES: + raise ValueError("multimodal knowledge bundle exceeds the byte limit") + with source.open("rb") as handle: + raw = handle.read(_MAX_BUNDLE_BYTES + 1) + if len(raw) > _MAX_BUNDLE_BYTES: + raise ValueError("multimodal knowledge bundle exceeds the byte limit") + data = json.loads(raw.decode("utf-8")) + return validate_multimodal_knowledge_bundle(data) + + +def validate_multimodal_knowledge_bundle(bundle: Mapping[str, Any]) -> dict[str, Any]: + """Return a normalized bundle or raise ``ValueError`` for invalid input.""" + + if not isinstance(bundle, Mapping): + raise ValueError("multimodal knowledge bundle must be an object") + data = dict(bundle) + if data.get("schema") != MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA: + raise ValueError("multimodal knowledge bundle schema is unsupported") + if data.get("schema_version") != MULTIMODAL_KNOWLEDGE_BUNDLE_VERSION: + raise ValueError("multimodal knowledge bundle version is unsupported") + for key in ( + "media_manifest", + "visual_facts_manifest", + "grounding_manifest", + "knowledge_view", + "component_sha256", + ): + if not isinstance(data.get(key), Mapping): + raise ValueError(f"multimodal knowledge bundle field {key!r} is invalid") + data[key] = dict(data[key]) + source_candidate_count = data.get("source_candidate_count") + if isinstance(source_candidate_count, bool) or not isinstance( + source_candidate_count, int + ): + raise ValueError( + "multimodal knowledge bundle source_candidate_count is invalid" + ) + if source_candidate_count < 0: + raise ValueError( + "multimodal knowledge bundle source_candidate_count is invalid" + ) + expected_hash = _stable_sha256( + {key: value for key, value in data.items() if key != "bundle_sha256"} + ) + recorded_hash = data.get("bundle_sha256") + if recorded_hash: + if recorded_hash != expected_hash: + raise ValueError("multimodal knowledge bundle hash does not match") + else: + data["bundle_sha256"] = expected_hash + return data + + +def _stable_sha256(payload: Mapping[str, Any]) -> str: + return sha256( + json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + ).hexdigest() + + +__all__ = [ + "MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA", + "MULTIMODAL_KNOWLEDGE_BUNDLE_VERSION", + "build_multimodal_knowledge_bundle", + "load_multimodal_knowledge_bundle", + "save_multimodal_knowledge_bundle", + "validate_multimodal_knowledge_bundle", +] diff --git a/codenib/wiki/media_vlm.py b/codenib/wiki/media_vlm.py index d1e38870..0a6ce0d2 100644 --- a/codenib/wiki/media_vlm.py +++ b/codenib/wiki/media_vlm.py @@ -135,6 +135,27 @@ def _post_json(self, payload: Mapping[str, Any]) -> dict[str, Any]: return data +def visual_fact_extractor_from_config( + config: Any, +) -> OpenAICompatibleVisualFactExtractor | None: + """Build a visual-fact extractor from ``QAConfig``-shaped settings.""" + + if not bool(getattr(config, "wiki_visual_fact_extraction_enabled", False)): + return None + model = str(getattr(config, "wiki_visual_facts_model", None) or "").strip() + api_base = str(getattr(config, "wiki_visual_facts_api_base", None) or "").strip() + options = dict(getattr(config, "wiki_visual_facts_options", {}) or {}) + timeout = options.get("timeout", 120.0) + provider = str(options.get("provider") or "openai-compatible") + return OpenAICompatibleVisualFactExtractor( + model=model, + api_base=api_base, + api_key=getattr(config, "wiki_visual_facts_api_key", None), + timeout=timeout, + provider=provider, + ) + + def _response_content_json(response: Mapping[str, Any]) -> dict[str, Any]: choices = response.get("choices") if not isinstance(choices, list) or not choices: @@ -240,4 +261,4 @@ def _chat_completions_endpoint(api_base: str) -> str: return parsed._replace(path=path, fragment="").geturl() -__all__ = ["OpenAICompatibleVisualFactExtractor"] +__all__ = ["OpenAICompatibleVisualFactExtractor", "visual_fact_extractor_from_config"] diff --git a/docs/experiments/multimodal_repository_knowledge.md b/docs/experiments/multimodal_repository_knowledge.md index c2daa1b4..77f38578 100644 --- a/docs/experiments/multimodal_repository_knowledge.md +++ b/docs/experiments/multimodal_repository_knowledge.md @@ -54,6 +54,31 @@ bounded local artifact as a data URL, asks for JSON-only structured visual facts, and normalizes the response into the same `VisualFactPack` schema. This keeps the multimodal knowledge pipeline independent of a specific model family. +The extractor is disabled by default. It can be configured through `QAConfig` +or environment variables: + +```yaml +wiki_visual_facts_enabled: true +wiki_visual_facts_model: qwen-vl +wiki_visual_facts_api_base: http://localhost:8000/v1 +wiki_visual_facts_options: + provider: qwen + timeout: 120 +``` + +Equivalent environment variables: + +```text +CODENIB_WIKI_VISUAL_FACTS_ENABLED=true +CODENIB_WIKI_VISUAL_FACTS_MODEL=qwen-vl +CODENIB_WIKI_VISUAL_FACTS_API_BASE=http://localhost:8000/v1 +CODENIB_WIKI_VISUAL_FACTS_API_KEY=... +CODENIB_WIKI_VISUAL_FACTS_OPTIONS='{"provider":"qwen","timeout":120}' +``` + +Offline and CI runs keep using deterministic local extraction unless the VLM is +explicitly enabled and both model and endpoint are provided. + ### VisualGroundingManifest `codenib.wiki.media_grounding` grounds extracted visual entities to repository @@ -78,6 +103,26 @@ surface as an MCP-compatible tool router with stable tool schemas and bounded input validation. This keeps the query surface testable before wiring it into a server-specific MCP registration path. +### Multimodal knowledge bundle + +`codenib.wiki.media_storage` wraps the pipeline output as a versioned bundle: + +```text +schema: codenib.multimodal-knowledge-bundle.v1 +schema_version: 1 +media_manifest +visual_facts_manifest +grounding_manifest +knowledge_view +component_sha256 +bundle_sha256 +``` + +The storage helper writes bundle JSON atomically and validates loaded bundles, +including schema version, required object fields, byte limits, and bundle hash. +This gives downstream consumers a stable artifact boundary instead of an ad hoc +script JSON dump. + ### Incremental updates `codenib.wiki.media_incremental` provides deterministic update planning for @@ -174,6 +219,17 @@ python scripts/build_multimodal_knowledge.py /path/to/repository \ --output /tmp/multimodal-knowledge.json ``` +To use an OpenAI-compatible VLM for visual fact extraction: + +```text +export CODENIB_WIKI_VISUAL_FACTS_API_KEY=... +python scripts/build_multimodal_knowledge.py /path/to/repository \ + --output /tmp/multimodal-knowledge.json \ + --visual-facts-model qwen-vl \ + --visual-facts-api-base http://localhost:8000/v1 \ + --visual-facts-provider qwen +``` + ```python from codenib.wiki import OpenAICompatibleVisualFactExtractor diff --git a/scripts/build_multimodal_knowledge.py b/scripts/build_multimodal_knowledge.py index 34bf55e3..b64e1677 100644 --- a/scripts/build_multimodal_knowledge.py +++ b/scripts/build_multimodal_knowledge.py @@ -9,9 +9,14 @@ import argparse import json +import os from pathlib import Path -from codenib.wiki import build_multimodal_repository_knowledge +from codenib.wiki import ( + OpenAICompatibleVisualFactExtractor, + build_multimodal_repository_knowledge, + save_multimodal_knowledge_bundle, +) def build_parser() -> argparse.ArgumentParser: @@ -40,23 +45,49 @@ def build_parser() -> argparse.ArgumentParser: default=8192, help="Maximum source-symbol candidates to consider for grounding", ) + parser.add_argument( + "--visual-facts-model", + default=None, + help=( + "Optional OpenAI-compatible VLM model for extracting visual facts. " + "When omitted, deterministic local extraction is used." + ), + ) + parser.add_argument( + "--visual-facts-api-base", + default=None, + help="OpenAI-compatible API base URL for --visual-facts-model", + ) + parser.add_argument( + "--visual-facts-api-key-env", + default="CODENIB_WIKI_VISUAL_FACTS_API_KEY", + help="Environment variable that contains the visual-facts API key", + ) + parser.add_argument( + "--visual-facts-provider", + default="openai-compatible", + help="Provider label recorded in extracted visual facts", + ) + parser.add_argument( + "--visual-facts-timeout", + type=float, + default=120.0, + help="Timeout in seconds for each visual-fact VLM request", + ) return parser def main(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) + extractor = _build_visual_fact_extractor(args) bundle = build_multimodal_repository_knowledge( args.repo, commit=args.commit, + extractor=extractor, max_artifacts=args.max_artifacts, max_source_candidates=args.max_source_candidates, ) - output = Path(args.output) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text( - json.dumps(bundle, ensure_ascii=False, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) + save_multimodal_knowledge_bundle(bundle, args.output) counts = { "media_artifacts": bundle["media_manifest"]["artifact_count"], "visual_fact_packs": bundle["visual_facts_manifest"]["fact_count"], @@ -68,5 +99,21 @@ def main(argv: list[str] | None = None) -> int: return 0 +def _build_visual_fact_extractor(args: argparse.Namespace): + model = str(args.visual_facts_model or "").strip() + api_base = str(args.visual_facts_api_base or "").strip() + if not model and not api_base: + return None + extractor = OpenAICompatibleVisualFactExtractor( + model=model, + api_base=api_base, + api_key=os.environ.get(str(args.visual_facts_api_key_env or "")), + timeout=args.visual_facts_timeout, + provider=args.visual_facts_provider, + ) + repo_path = Path(args.repo) + return lambda artifact: extractor.extract(artifact, repo_path=repo_path) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/test/scripts/test_build_multimodal_knowledge.py b/test/scripts/test_build_multimodal_knowledge.py index e7e6f4fa..ee655e83 100644 --- a/test/scripts/test_build_multimodal_knowledge.py +++ b/test/scripts/test_build_multimodal_knowledge.py @@ -8,6 +8,8 @@ import subprocess import sys +from scripts.build_multimodal_knowledge import build_parser + def test_build_multimodal_knowledge_script_writes_bundle(tmp_path): repo = tmp_path / "repo" @@ -47,5 +49,35 @@ def test_build_multimodal_knowledge_script_writes_bundle(tmp_path): bundle = json.loads(output.read_text(encoding="utf-8")) assert counts["media_artifacts"] == 1 assert counts["knowledge_entries"] == 1 + assert bundle["schema"] == "codenib.multimodal-knowledge-bundle.v1" + assert len(bundle["bundle_sha256"]) == 64 assert bundle["media_manifest"]["commit"] == "abc123" assert bundle["knowledge_view"]["entry_count"] == 1 + + +def test_build_multimodal_knowledge_parser_accepts_vlm_options(tmp_path): + output = tmp_path / "bundle.json" + + args = build_parser().parse_args( + [ + str(tmp_path), + "--output", + str(output), + "--visual-facts-model", + "qwen-vl", + "--visual-facts-api-base", + "https://vlm.example/v1", + "--visual-facts-api-key-env", + "TEST_VLM_KEY", + "--visual-facts-provider", + "qwen", + "--visual-facts-timeout", + "15", + ] + ) + + assert args.visual_facts_model == "qwen-vl" + assert args.visual_facts_api_base == "https://vlm.example/v1" + assert args.visual_facts_api_key_env == "TEST_VLM_KEY" + assert args.visual_facts_provider == "qwen" + assert args.visual_facts_timeout == 15 diff --git a/test/web/test_config.py b/test/web/test_config.py index 65690e8e..85d8e8d8 100644 --- a/test/web/test_config.py +++ b/test/web/test_config.py @@ -28,12 +28,14 @@ def test_wiki_media_config_enables_local_renderer_without_endpoint( tmp_path: Path, ) -> None: config_path = tmp_path / "config.yaml" - config_path.write_text(""" + config_path.write_text( + """ wiki_media_model: local/svg wiki_media_options: provider: local width: 1024 -""".lstrip()) +""".lstrip() + ) config = load_config(str(config_path)) @@ -47,11 +49,13 @@ def test_wiki_media_environment_overrides_file_config( monkeypatch, ) -> None: config_path = tmp_path / "config.yaml" - config_path.write_text(""" + config_path.write_text( + """ wiki_media_model: local/svg wiki_media_options: provider: local -""".lstrip()) +""".lstrip() + ) monkeypatch.setenv("CODENIB_WIKI_MEDIA_MODEL", "openai/image-1") monkeypatch.setenv("CODENIB_WIKI_MEDIA_API_BASE", "https://images.example/v1") monkeypatch.setenv("CODENIB_WIKI_MEDIA_API_KEY", "secret") @@ -72,6 +76,50 @@ def test_wiki_media_environment_overrides_file_config( } +def test_wiki_visual_facts_config_is_disabled_by_default(tmp_path: Path) -> None: + config_path = tmp_path / "config.yaml" + config_path.write_text("wiki_agent: false\n") + + config = load_config(str(config_path)) + + assert config.wiki_visual_fact_extraction_enabled is False + assert config.wiki_visual_facts_options == {} + + +def test_wiki_visual_facts_environment_overrides_file_config( + tmp_path: Path, + monkeypatch, +) -> None: + config_path = tmp_path / "config.yaml" + config_path.write_text( + """ +wiki_visual_facts_enabled: false +wiki_visual_facts_model: file-model +wiki_visual_facts_options: + provider: file-provider +""".lstrip() + ) + monkeypatch.setenv("CODENIB_WIKI_VISUAL_FACTS_ENABLED", "true") + monkeypatch.setenv("CODENIB_WIKI_VISUAL_FACTS_MODEL", "qwen-vl") + monkeypatch.setenv("CODENIB_WIKI_VISUAL_FACTS_API_BASE", "https://vlm.example/v1") + monkeypatch.setenv("CODENIB_WIKI_VISUAL_FACTS_API_KEY", "secret") + monkeypatch.setenv( + "CODENIB_WIKI_VISUAL_FACTS_OPTIONS", + '{"provider":"local-vlm","timeout":45}', + ) + + config = load_config(str(config_path)) + + assert config.wiki_visual_fact_extraction_enabled is True + assert config.wiki_visual_facts_model == "qwen-vl" + assert config.wiki_visual_facts_api_base == "https://vlm.example/v1" + assert config.wiki_visual_facts_api_key == "secret" + assert config.wiki_visual_facts_options == { + "provider": "local-vlm", + "timeout": 45, + } + + def test_config_profile_extends_relative_base_and_merges_options( tmp_path: Path, ) -> None: diff --git a/test/wiki/test_media_pipeline.py b/test/wiki/test_media_pipeline.py index b8aef1bd..bb8cdb60 100644 --- a/test/wiki/test_media_pipeline.py +++ b/test/wiki/test_media_pipeline.py @@ -75,6 +75,9 @@ def test_build_multimodal_repository_knowledge_wraps_pipeline(tmp_path): bundle = build_multimodal_repository_knowledge(tmp_path, commit="abc123") + assert bundle["schema"] == "codenib.multimodal-knowledge-bundle.v1" + assert bundle["schema_version"] == 1 + assert len(bundle["bundle_sha256"]) == 64 assert bundle["media_manifest"]["artifact_count"] == 1 assert bundle["visual_facts_manifest"]["fact_count"] == 1 assert bundle["source_candidate_count"] >= 1 diff --git a/test/wiki/test_media_storage.py b/test/wiki/test_media_storage.py new file mode 100644 index 00000000..81e008ee --- /dev/null +++ b/test/wiki/test_media_storage.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: 2025-2026 CodeNib Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json + +import pytest + +import codenib.wiki.media_storage as media_storage +from codenib.wiki.media_storage import ( + MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA, + build_multimodal_knowledge_bundle, + load_multimodal_knowledge_bundle, + save_multimodal_knowledge_bundle, + validate_multimodal_knowledge_bundle, +) + + +def _bundle(): + return build_multimodal_knowledge_bundle( + media_manifest={ + "manifest_sha256": "media-hash", + "artifact_count": 1, + "artifacts": [], + }, + visual_facts_manifest={ + "manifest_sha256": "facts-hash", + "fact_count": 1, + "facts": [], + }, + source_candidate_count=3, + grounding_manifest={ + "manifest_sha256": "grounding-hash", + "binding_count": 2, + "bindings": [], + }, + knowledge_view={ + "view_sha256": "view-hash", + "entry_count": 1, + "entries": [], + }, + ) + + +def test_build_multimodal_knowledge_bundle_records_schema_and_hashes(): + bundle = _bundle() + + assert bundle["schema"] == MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA + assert bundle["schema_version"] == 1 + assert bundle["component_sha256"] == { + "media_manifest": "media-hash", + "visual_facts_manifest": "facts-hash", + "grounding_manifest": "grounding-hash", + "knowledge_view": "view-hash", + } + assert len(bundle["bundle_sha256"]) == 64 + assert ( + validate_multimodal_knowledge_bundle(bundle)["bundle_sha256"] + == bundle["bundle_sha256"] + ) + + +def test_save_and_load_multimodal_knowledge_bundle_round_trips(tmp_path): + path = tmp_path / "nested" / "bundle.json" + bundle = _bundle() + + save_multimodal_knowledge_bundle(bundle, path) + loaded = load_multimodal_knowledge_bundle(path) + + assert loaded == bundle + assert json.loads(path.read_text(encoding="utf-8"))["schema"] == ( + MULTIMODAL_KNOWLEDGE_BUNDLE_SCHEMA + ) + + +def test_validate_multimodal_knowledge_bundle_rejects_tampering(): + bundle = _bundle() + bundle["source_candidate_count"] = 4 + + with pytest.raises(ValueError, match="hash"): + validate_multimodal_knowledge_bundle(bundle) + + +def test_load_multimodal_knowledge_bundle_rejects_oversized_file(tmp_path, monkeypatch): + monkeypatch.setattr(media_storage, "_MAX_BUNDLE_BYTES", 8) + path = tmp_path / "bundle.json" + path.write_text("x" * 9, encoding="utf-8") + + with pytest.raises(ValueError, match="byte limit"): + load_multimodal_knowledge_bundle(path) diff --git a/test/wiki/test_media_vlm.py b/test/wiki/test_media_vlm.py index cc32ead6..a017ed1a 100644 --- a/test/wiki/test_media_vlm.py +++ b/test/wiki/test_media_vlm.py @@ -5,11 +5,15 @@ from __future__ import annotations import json +from types import SimpleNamespace import pytest import codenib.wiki.media_vlm as media_vlm -from codenib.wiki.media_vlm import OpenAICompatibleVisualFactExtractor +from codenib.wiki.media_vlm import ( + OpenAICompatibleVisualFactExtractor, + visual_fact_extractor_from_config, +) class _Response: @@ -195,3 +199,30 @@ def test_visual_fact_extractor_rejects_non_json_content(): with pytest.raises(json.JSONDecodeError): extractor.extract(_artifact()) + + +def test_visual_fact_extractor_from_config_returns_none_when_disabled(): + config = SimpleNamespace(wiki_visual_fact_extraction_enabled=False) + + assert visual_fact_extractor_from_config(config) is None + + +def test_visual_fact_extractor_from_config_builds_provider(): + config = SimpleNamespace( + wiki_visual_fact_extraction_enabled=True, + wiki_visual_facts_model="qwen-vl", + wiki_visual_facts_api_base="https://vlm.example/v1", + wiki_visual_facts_api_key="secret", + wiki_visual_facts_options={ + "provider": "qwen", + "timeout": 9, + }, + ) + + extractor = visual_fact_extractor_from_config(config) + + assert isinstance(extractor, OpenAICompatibleVisualFactExtractor) + assert extractor.model == "qwen-vl" + assert extractor.endpoint == "https://vlm.example/v1/chat/completions" + assert extractor.provider == "qwen" + assert extractor.timeout == 9