From a29c33bc46429929143c2d4c93c5ec29c4d3a97a Mon Sep 17 00:00:00 2001 From: marinamackay Date: Sat, 22 Aug 2026 04:30:48 +0800 Subject: [PATCH] feat(wiki): support pluggable visual grounding scorers --- codenib/wiki/media_grounding.py | 62 ++++++++++++++++++- .../multimodal_repository_knowledge.md | 4 +- test/wiki/test_media_grounding.py | 43 +++++++++++++ 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/codenib/wiki/media_grounding.py b/codenib/wiki/media_grounding.py index 6f16426e..7656c953 100644 --- a/codenib/wiki/media_grounding.py +++ b/codenib/wiki/media_grounding.py @@ -11,7 +11,7 @@ import re from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Iterable, Mapping +from typing import Any, Callable, Iterable, Mapping from ..repository_filters import walk_repository_files from ..repository_source_selection import ( @@ -44,6 +44,9 @@ r"\b(?:class|def|function|const|let|var|interface|type|struct|enum)\s+([A-Za-z_][A-Za-z0-9_]*)" ) _CAMEL_RE = re.compile(r"\b[A-Z][A-Za-z0-9_]{2,}\b") +VisualGroundingScorer = Callable[ + [Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any] | None +] @dataclass(frozen=True) @@ -159,8 +162,14 @@ def ground_visual_facts_to_sources( source_candidates: Iterable[Mapping[str, Any]], *, max_bindings_per_entity: int = _MAX_BINDINGS_PER_ENTITY, + scorer: VisualGroundingScorer | None = None, ) -> dict[str, Any]: - """Ground visual entities to a source inventory using deterministic scoring.""" + """Ground visual entities to a source inventory. + + The default scorer is deterministic and lexical. Callers can pass a scorer + backed by BM25, embeddings, CodeGraph, LSP facts, or FactQueryIndex without + changing the visual-code binding manifest schema. + """ candidates = [ _candidate_from_mapping(candidate) @@ -188,11 +197,13 @@ def ground_visual_facts_to_sources( scored = [ binding for binding in ( - _score_candidate( + _score_with_optional_scorer( artifact_path=artifact_path, + entity=entity, entity_name=entity_name, hints=hints, candidate=candidate, + scorer=scorer, ) for candidate in candidates ) @@ -229,6 +240,40 @@ def ground_visual_facts_to_sources( return manifest.to_dict() +def _score_with_optional_scorer( + *, + artifact_path: str, + entity: Mapping[str, Any], + entity_name: str, + hints: Iterable[str], + candidate: SourceSymbolCandidate, + scorer: VisualGroundingScorer | None, +) -> VisualCodeBinding | None: + if scorer is None: + return _score_candidate( + artifact_path=artifact_path, + entity_name=entity_name, + hints=hints, + candidate=candidate, + ) + raw = scorer(entity, candidate.to_dict()) + if not isinstance(raw, Mapping): + return None + score = _confidence(raw.get("score")) + if score <= 0: + return None + return VisualCodeBinding( + artifact_path=artifact_path, + entity_name=entity_name, + source_path=candidate.path, + symbol=candidate.symbol, + kind=candidate.kind, + line=candidate.line, + score=round(score, 4), + evidence=_safe_text(raw.get("evidence") or "custom scorer"), + ) + + def _score_candidate( *, artifact_path: str, @@ -324,6 +369,16 @@ def _positive_int(value: Any) -> int: return max(0, number) +def _confidence(value: Any) -> float: + if isinstance(value, bool): + return 0.0 + try: + confidence = float(value) + except (TypeError, ValueError): + return 0.0 + return min(1.0, max(0.0, confidence)) + + def _sha256_json(payload: Mapping[str, Any]) -> str: encoded = json.dumps( payload, @@ -343,6 +398,7 @@ def _safe_text(value: Any) -> str: "MEDIA_GROUNDING_SCHEMA", "MEDIA_GROUNDING_VERSION", "SourceSymbolCandidate", + "VisualGroundingScorer", "VisualCodeBinding", "VisualGroundingManifest", "discover_source_symbol_candidates", diff --git a/docs/experiments/multimodal_repository_knowledge.md b/docs/experiments/multimodal_repository_knowledge.md index 10119b8c..e3224a02 100644 --- a/docs/experiments/multimodal_repository_knowledge.md +++ b/docs/experiments/multimodal_repository_knowledge.md @@ -60,7 +60,9 @@ keeps the multimodal knowledge pipeline independent of a specific model family. files and symbols. The first implementation uses deterministic lexical scoring against a bounded source-symbol inventory. Later versions can replace the scorer with BM25, embeddings, CodeGraph, LSP facts, or `FactQueryIndex` / -`FactBatch`. +`FactBatch`. The `ground_visual_facts_to_sources(..., scorer=...)` hook already +accepts a custom scorer, so graph/fact-backed ranking can be added without +changing the binding manifest schema. ### MultimodalKnowledgeView diff --git a/test/wiki/test_media_grounding.py b/test/wiki/test_media_grounding.py index 33370d68..b4340554 100644 --- a/test/wiki/test_media_grounding.py +++ b/test/wiki/test_media_grounding.py @@ -170,3 +170,46 @@ def test_ground_visual_facts_to_sources_deduplicates_bindings(): manifest = ground_visual_facts_to_sources(visual_facts, [source, source]) assert len(manifest["bindings"]) == 1 + + +def test_ground_visual_facts_to_sources_accepts_custom_scorer(): + visual_facts = { + "manifest_sha256": "visual-facts-hash", + "facts": [ + { + "artifact_path": "docs/architecture.svg", + "entities": [{"name": "DiagramBox"}], + } + ], + } + + def scorer(entity, candidate): + if entity["name"] == "DiagramBox" and candidate["symbol"] == "WikiService": + return {"score": 0.88, "evidence": "graph scorer match"} + return None + + manifest = ground_visual_facts_to_sources( + visual_facts, + [ + { + "path": "codenib/wiki/service.py", + "symbol": "WikiService", + "kind": "symbol", + "line": 17, + } + ], + scorer=scorer, + ) + + assert manifest["bindings"] == [ + { + "artifact_path": "docs/architecture.svg", + "entity_name": "DiagramBox", + "source_path": "codenib/wiki/service.py", + "symbol": "WikiService", + "kind": "symbol", + "line": 17, + "score": 0.88, + "evidence": "graph scorer match", + } + ]