diff --git a/application/tests/librarian/explicit_link_resolver_test.py b/application/tests/librarian/explicit_link_resolver_test.py new file mode 100644 index 000000000..e7a8775e3 --- /dev/null +++ b/application/tests/librarian/explicit_link_resolver_test.py @@ -0,0 +1,83 @@ +"""Tests for the C.0.5 deterministic explicit-CRE fast path. + +Covers extraction (pattern boundaries, ordering, dedup) and resolution +(the fail-safe rule: only a single known reference auto-links; unknown or +conflicting references must fall through to review). +""" + +import unittest + +from application.utils.librarian.explicit_link_resolver import ( + Resolution, + ResolutionOutcome, + extract_cre_refs, + resolve, +) + +KNOWN = {"027-555", "123-456", "764-507"} + + +class ExtractCreRefsTest(unittest.TestCase): + def test_extraction_table(self) -> None: + cases = [ + ("plain reference", "Per CRE 027-555, verify passwords.", ["027-555"]), + ( + "inside an opencre url", + "See https://opencre.org/cre/123-456 for details.", + ["123-456"], + ), + ( + "multiple distinct, in order", + "Maps to 123-456 and also 027-555.", + ["123-456", "027-555"], + ), + ("repeated reference deduped", "027-555 then 027-555 again.", ["027-555"]), + ("no reference", "Use MFA for all administrative access.", []), + ("too many leading digits", "CVE-2024-1234-567 is unrelated.", []), + ("too many trailing digits", "Item 027-5555 is not a CRE id.", []), + ("punctuation boundary", "(see CRE 764-507).", ["764-507"]), + ] + for name, text, expected in cases: + with self.subTest(name): + self.assertEqual(extract_cre_refs(text), expected) + + +class ResolveTest(unittest.TestCase): + def test_single_known_reference_resolves(self) -> None: + resolution = resolve("Per CRE 027-555, verify passwords.", KNOWN) + self.assertEqual( + resolution, + Resolution(ResolutionOutcome.resolved, ("027-555",), ()), + ) + + def test_no_reference_falls_through_to_semantic_path(self) -> None: + resolution = resolve("Use MFA for all administrative access.", KNOWN) + self.assertEqual(resolution.outcome, ResolutionOutcome.no_reference) + self.assertEqual(resolution.cre_ids, ()) + + def test_unknown_reference_never_auto_links(self) -> None: + resolution = resolve("Per CRE 999-999, do the thing.", KNOWN) + self.assertEqual(resolution.outcome, ResolutionOutcome.unknown_reference) + self.assertEqual(resolution.cre_ids, ()) + self.assertEqual(resolution.unknown_refs, ("999-999",)) + + def test_mixed_known_and_unknown_routes_to_review(self) -> None: + resolution = resolve("Maps to 027-555 and 999-999.", KNOWN) + self.assertEqual(resolution.outcome, ResolutionOutcome.unknown_reference) + # The known id survives as a review suggestion, but must not auto-link. + self.assertEqual(resolution.cre_ids, ("027-555",)) + self.assertEqual(resolution.unknown_refs, ("999-999",)) + + def test_conflicting_known_references_route_to_review(self) -> None: + resolution = resolve("Maps to 123-456 and also 027-555.", KNOWN) + self.assertEqual(resolution.outcome, ResolutionOutcome.conflicting_references) + self.assertEqual(resolution.cre_ids, ("123-456", "027-555")) + + def test_repeated_single_reference_still_resolves(self) -> None: + resolution = resolve("027-555 is cited twice: 027-555.", KNOWN) + self.assertEqual(resolution.outcome, ResolutionOutcome.resolved) + self.assertEqual(resolution.cre_ids, ("027-555",)) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/librarian/section_validator_test.py b/application/tests/librarian/section_validator_test.py new file mode 100644 index 000000000..a8daeadbd --- /dev/null +++ b/application/tests/librarian/section_validator_test.py @@ -0,0 +1,182 @@ +"""Tests for the C.0 input boundary (section_validator). + +Table-driven over every rejection class plus the happy paths for both +upstream shapes (knowledge_queue row and RFC KnowledgeItem envelope). +Asserts the boundary never leaks a raw Pydantic ValidationError. +""" + +import unittest + +from pydantic import ValidationError + +from application.utils.librarian.section_validator import ( + EmptyTextError, + MalformedKnowledgeItemError, + NotKnowledgeError, + Section, + SectionValidationError, + UnsupportedLanguageError, + section_from_knowledge_item, + section_from_queue_row, +) + + +def valid_queue_row(**overrides) -> dict: + row = { + "id": "4a8c1b2e-1d2f-4e3a-9b4c-5d6e7f8a9b0c", + "source_repo": "OWASP/ASVS", + "source_path": "4.0/en/0x11-V2-Authentication.md", + "source_commit_sha": "abc123def456789012345678901234567890abcd", + "text": "Verify that user-set passwords are at least 12 characters long.", + "confidence": 0.93, + "llm_label": "KNOWLEDGE", + "llm_reasoning": "clear security requirement", + "created_at": "2026-05-25T02:25:00Z", + "consumed_at": None, + } + row.update(overrides) + return row + + +def valid_knowledge_item(**overrides) -> dict: + item = { + "schema_version": "0.2.0", + "chunk_id": "chk:art:OWASP/ASVS:4.0/en/0x11-V2-Authentication.md:0", + "artifact_id": "art:OWASP/ASVS:4.0/en/0x11-V2-Authentication.md", + "event_id": "evt-001", + "pipeline_run_id": "20260601T020000Z", + "filtered_at": "2026-06-01T02:10:00Z", + "status": "accepted", + "source": { + "type": "github", + "repo": "OWASP/ASVS", + "commit_sha": "abc123def456789012345678901234567890abcd", + "committed_at": "2026-06-01T01:00:00Z", + }, + "locator": { + "kind": "repo_path", + "id": "4.0/en/0x11-V2-Authentication.md", + "path": "4.0/en/0x11-V2-Authentication.md", + }, + "content": { + "text": "Verify that user-set passwords are at least 12 characters long.", + "title_hint": "Password length", + "language": "en", + }, + "filter": { + "stages": [{"name": "llm_relevance", "passed": True}], + "is_security_knowledge": True, + "confidence": 0.93, + }, + } + item.update(overrides) + return item + + +class QueueRowBoundaryTest(unittest.TestCase): + def test_valid_row_builds_section_with_synthesized_identity(self) -> None: + section = section_from_queue_row(valid_queue_row()) + self.assertIsInstance(section, Section) + self.assertEqual( + section.chunk_id, + "chk:OWASP/ASVS@abc123def456789012345678901234567890abcd:" + "4.0/en/0x11-V2-Authentication.md", + ) + self.assertEqual( + section.artifact_id, "art:OWASP/ASVS:4.0/en/0x11-V2-Authentication.md" + ) + self.assertEqual(section.source.repo, "OWASP/ASVS") + self.assertEqual(section.source.committed_at, "2026-05-25T02:25:00Z") + self.assertEqual(section.locator.path, "4.0/en/0x11-V2-Authentication.md") + self.assertEqual(section.language, "en") + + def test_volatile_metadata_not_carried_into_section(self) -> None: + section = section_from_queue_row( + valid_queue_row(llm_reasoning="audit-only rationale") + ) + self.assertFalse(hasattr(section, "llm_reasoning")) + self.assertFalse(hasattr(section, "confidence")) + + def test_rejection_table(self) -> None: + cases = [ + ("empty text", valid_queue_row(text=""), EmptyTextError), + ("whitespace text", valid_queue_row(text=" \n\t "), EmptyTextError), + ("noise label", valid_queue_row(llm_label="NOISE"), NotKnowledgeError), + ( + "uncertain label", + valid_queue_row(llm_label="UNCERTAIN"), + NotKnowledgeError, + ), + ( + "missing field", + {k: v for k, v in valid_queue_row().items() if k != "source_repo"}, + MalformedKnowledgeItemError, + ), + ( + "wrong type", + valid_queue_row(confidence="very sure"), + MalformedKnowledgeItemError, + ), + ("not a mapping", "just a string", MalformedKnowledgeItemError), + ] + for name, row, expected_error in cases: + with self.subTest(name): + with self.assertRaises(expected_error): + section_from_queue_row(row) + + def test_never_leaks_raw_pydantic_error(self) -> None: + try: + section_from_queue_row({"id": "x"}) + except SectionValidationError as exc: + self.assertNotIsInstance(exc, ValidationError) + self.assertIsInstance(exc.__cause__, ValidationError) + else: + self.fail("expected SectionValidationError") + + +class KnowledgeItemBoundaryTest(unittest.TestCase): + def test_valid_item_builds_section(self) -> None: + section = section_from_knowledge_item(valid_knowledge_item()) + self.assertEqual( + section.chunk_id, "chk:art:OWASP/ASVS:4.0/en/0x11-V2-Authentication.md:0" + ) + self.assertEqual(section.title_hint, "Password length") + self.assertEqual(section.language, "en") + + def test_missing_language_defaults_to_english(self) -> None: + item = valid_knowledge_item() + del item["content"]["language"] + self.assertEqual(section_from_knowledge_item(item).language, "en") + + def test_regional_english_variant_is_accepted(self) -> None: + item = valid_knowledge_item() + item["content"]["language"] = "en-GB" + self.assertEqual(section_from_knowledge_item(item).language, "en-GB") + + def test_rejection_table(self) -> None: + rejected = valid_knowledge_item( + status="rejected", + content=None, + rejection={"reason_code": "NOT_SECURITY"}, + ) + unsupported_lang = valid_knowledge_item() + unsupported_lang["content"]["language"] = "fr" + whitespace_text = valid_knowledge_item() + whitespace_text["content"]["text"] = " " + malformed = valid_knowledge_item() + del malformed["source"] + + cases = [ + ("status rejected", rejected, NotKnowledgeError), + ("unsupported language", unsupported_lang, UnsupportedLanguageError), + ("whitespace text", whitespace_text, EmptyTextError), + ("missing source", malformed, MalformedKnowledgeItemError), + ] + for name, item, expected_error in cases: + with self.subTest(name): + with self.assertRaises(expected_error): + section_from_knowledge_item(item) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/librarian/__init__.py b/application/utils/librarian/__init__.py index 379c131d1..e0581f28a 100644 --- a/application/utils/librarian/__init__.py +++ b/application/utils/librarian/__init__.py @@ -9,7 +9,11 @@ C -> graph : LinkProposal (confident auto-link, status=linked) C -> D : ReviewItem (low-confidence / flagged, routed to HITL) -Week 1-1 scope: contracts + config + tests only. No linking logic yet. +Scope so far: + W1 (C.-1): contracts + config + eval harness + golden dataset. + W2 (C.0): input boundary — SectionValidator (validate/adapt without + re-normalizing text) and ExplicitLinkResolver (fail-safe + explicit-link resolution). No retrieval/ranking logic yet. Vendored RFC JSON schemas live under ``_rfc_schemas/``. They are pinned to upstream/owasp-graph @ 2b1437987768d5ed20fe9ee721ab9a898c4b84af (PR #734). diff --git a/application/utils/librarian/explicit_link_resolver.py b/application/utils/librarian/explicit_link_resolver.py new file mode 100644 index 000000000..4caf660f3 --- /dev/null +++ b/application/utils/librarian/explicit_link_resolver.py @@ -0,0 +1,73 @@ +"""Module C.0.5 — deterministic explicit-CRE fast path (Week 2). No ML. + +If a section's text already cites a CRE id (``ddd-ddd``, plain or inside an +``opencre.org/cre/`` link), resolve it directly against the set of known +CRE ids and bypass retrieval entirely. The fail-safe rule from the proposal: +unknown or mutually conflicting references never auto-link — they fall +through to human review; only a single, known reference resolves. + +Gate (PR 2): 100% correctness on the explicit golden-dataset slice. Any +regression here is a merge blocker. + +The known-id set is injected (``Container[str]``) so this module stays +dependency-free: the harness seeds it from the golden dataset today; the +DB-backed registry of real ``cre.external_id`` values arrives with the +retriever (W3). +""" + +import re +from dataclasses import dataclass +from enum import Enum +from typing import Container, List, Tuple + +# Word boundaries keep e.g. "1234-567" and "027-5555" from partially matching. +CRE_ID_RE = re.compile(r"\b\d{3}-\d{3}\b") + + +class ResolutionOutcome(str, Enum): + # No CRE reference in the text — continue to the semantic path (C.1+). + no_reference = "no_reference" + # Exactly one known CRE id — deterministic auto-link, skip retrieval. + resolved = "resolved" + # Reference(s) found but none/some are known CRE ids — route to review. + unknown_reference = "unknown_reference" + # Multiple distinct known ids in one section — ambiguous, route to review. + conflicting_references = "conflicting_references" + + +@dataclass(frozen=True) +class Resolution: + outcome: ResolutionOutcome + # Known ids, deduped, in order of first appearance. Non-empty iff some + # reference resolved; for conflicting outcomes these become the + # ReviewItem's suggested_links. + cre_ids: Tuple[str, ...] + # References that matched the CRE-id pattern but are not known ids. + unknown_refs: Tuple[str, ...] + + +def extract_cre_refs(text: str) -> List[str]: + """All CRE-id-shaped references in the text, deduped, in order.""" + seen = set() + refs = [] + for match in CRE_ID_RE.findall(text): + if match not in seen: + seen.add(match) + refs.append(match) + return refs + + +def resolve(text: str, known_cre_ids: Container[str]) -> Resolution: + """Deterministically resolve explicit CRE references in one section.""" + refs = extract_cre_refs(text) + if not refs: + return Resolution(ResolutionOutcome.no_reference, (), ()) + + known = tuple(ref for ref in refs if ref in known_cre_ids) + unknown = tuple(ref for ref in refs if ref not in known_cre_ids) + + if unknown: + return Resolution(ResolutionOutcome.unknown_reference, known, unknown) + if len(known) > 1: + return Resolution(ResolutionOutcome.conflicting_references, known, ()) + return Resolution(ResolutionOutcome.resolved, known, ()) diff --git a/application/utils/librarian/knowledge_source.py b/application/utils/librarian/knowledge_source.py index be7f97f65..76ad493a1 100644 --- a/application/utils/librarian/knowledge_source.py +++ b/application/utils/librarian/knowledge_source.py @@ -6,7 +6,6 @@ from each row at processing time (master guide §1.2). """ -import json from abc import ABC, abstractmethod from typing import Iterator diff --git a/application/utils/librarian/schemas.py b/application/utils/librarian/schemas.py index d1395f013..d632e2859 100644 --- a/application/utils/librarian/schemas.py +++ b/application/utils/librarian/schemas.py @@ -103,7 +103,7 @@ class KnowledgeContent(BaseModel): text: str = Field(min_length=1) title_hint: Optional[str] = None keywords: Optional[List[str]] = None - language: Optional[str] = None + language: Optional[str] = "en" class FilterStage(BaseModel): @@ -277,6 +277,8 @@ class KnowledgeQueueItem(BaseModel): fields so B can extend the row without breaking C. """ + model_config = ConfigDict(extra="ignore") + id: str source_repo: str source_path: str diff --git a/application/utils/librarian/section_validator.py b/application/utils/librarian/section_validator.py new file mode 100644 index 000000000..610a91788 --- /dev/null +++ b/application/utils/librarian/section_validator.py @@ -0,0 +1,180 @@ +"""Module C.0 — the deterministic input boundary (Week 2). + +Validates what enters the Librarian and adapts it into the internal +``Section`` the retrieval pipeline consumes. This layer does **no text +normalization**: the RFC (PR #734) assigns text cleanup to Module A +(normalize + chunk) with Module B's sanitizer on top, so ``text`` is +contractually clean by the time it reaches C. Re-cleaning here would +silently drift C from what A hashed and B classified. + +Two entry points, one per upstream shape: + +- ``section_from_queue_row`` — Module B's reduced ``knowledge_queue`` row + (master guide §1.2). The RFC identity fields C needs downstream are + synthesized from the row:: + + artifact_id = "art:{source_repo}:{source_path}" + chunk_id = "chk:{source_repo}@{source_commit_sha}:{source_path}" + + This chunk_id format differs from Module B's ``ChangeRecord`` + (``chk:art:{repo}:{path}:{index}``); align the two when the live + B->C pipeline wiring lands (W8). Not blocking W2 — no shared consumer yet. + +- ``section_from_knowledge_item`` — the full RFC ``KnowledgeItem`` + envelope (fixtures today; the live B->C path lands W8). + +Volatile / audit-only metadata (``llm_reasoning``, ``filtered_at``, +``pipeline_run_id``, filter stages) is intentionally not carried into +``Section`` — downstream stages must never key a decision on it. + +Every rejection is a typed ``SectionValidationError`` subclass; raw +Pydantic ``ValidationError`` never escapes this module. +""" + +from dataclasses import dataclass +from typing import Any, Dict, Optional, Union + +from pydantic import ValidationError + +from application.utils.librarian.schemas import ( + KnowledgeItem, + KnowledgeQueueItem, + KnowledgeStatus, + Locator, + LocatorKind, + SourceRef, + SourceType, +) + +KNOWLEDGE_LABEL = "KNOWLEDGE" + +# MVP scope: golden dataset and CRE hub vectors are English-only. +_SUPPORTED_PRIMARY_LANGUAGES = frozenset({"en"}) +_DEFAULT_LANGUAGE = "en" + + +class SectionValidationError(ValueError): + """Base class for every typed rejection at the C.0 boundary.""" + + +class MalformedKnowledgeItemError(SectionValidationError): + """Input does not match its contract (wraps Pydantic validation failure).""" + + +class EmptyTextError(SectionValidationError): + """Text is empty or whitespace-only — nothing to retrieve against.""" + + +class UnsupportedLanguageError(SectionValidationError): + """Language is outside the supported set (English-only MVP).""" + + +class NotKnowledgeError(SectionValidationError): + """Item was not accepted as security knowledge upstream; C must not link it.""" + + +@dataclass(frozen=True) +class Section: + """What the C.1+ pipeline consumes: identity + text + provenance, nothing else.""" + + chunk_id: str + artifact_id: str + text: str + title_hint: Optional[str] + language: str + source: SourceRef + locator: Locator + + +def _require_text(text: str) -> str: + if not text or not text.strip(): + raise EmptyTextError("section text is empty or whitespace-only") + return text + + +def _require_language(language: Optional[str]) -> str: + if language is None: + return _DEFAULT_LANGUAGE + primary = language.split("-", 1)[0].lower() + if primary not in _SUPPORTED_PRIMARY_LANGUAGES: + raise UnsupportedLanguageError( + f"unsupported language {language!r}; supported: " + f"{sorted(_SUPPORTED_PRIMARY_LANGUAGES)}" + ) + return language + + +def section_from_queue_row( + row: Union[KnowledgeQueueItem, Dict[str, Any]], +) -> Section: + """Validate one knowledge_queue row and adapt it to a Section. + + Raises a SectionValidationError subclass on any rejection. + """ + if not isinstance(row, KnowledgeQueueItem): + try: + row = KnowledgeQueueItem.model_validate(row) + except ValidationError as exc: + raise MalformedKnowledgeItemError(str(exc)) from exc + + if row.llm_label != KNOWLEDGE_LABEL: + raise NotKnowledgeError( + f"llm_label={row.llm_label!r}; only {KNOWLEDGE_LABEL!r} rows may be linked" + ) + _require_text(row.text) + + # B's reduced row has no commit timestamp; created_at (B's classification + # time) is the best available provenance until the live B->C path lands. + source = SourceRef( + type=SourceType.github, + repo=row.source_repo, + commit_sha=row.source_commit_sha, + committed_at=row.created_at, + ) + locator = Locator( + kind=LocatorKind.repo_path, + id=row.source_path, + path=row.source_path, + ) + return Section( + chunk_id=f"chk:{row.source_repo}@{row.source_commit_sha}:{row.source_path}", + artifact_id=f"art:{row.source_repo}:{row.source_path}", + text=row.text, + title_hint=None, + language=_DEFAULT_LANGUAGE, + source=source, + locator=locator, + ) + + +def section_from_knowledge_item( + item: Union[KnowledgeItem, Dict[str, Any]], +) -> Section: + """Validate one RFC KnowledgeItem envelope and adapt it to a Section. + + Raises a SectionValidationError subclass on any rejection. + """ + if not isinstance(item, KnowledgeItem): + try: + item = KnowledgeItem.model_validate(item) + except ValidationError as exc: + raise MalformedKnowledgeItemError(str(exc)) from exc + + if item.status != KnowledgeStatus.accepted: + raise NotKnowledgeError( + f"status={item.status.value!r}; only 'accepted' items may be linked" + ) + # status=accepted guarantees content (enforced by the KnowledgeItem model). + assert item.content is not None + _require_text(item.content.text) + language = _require_language(item.content.language) + + return Section( + chunk_id=item.chunk_id, + artifact_id=item.artifact_id, + text=item.content.text, + title_hint=item.content.title_hint, + language=language, + source=item.source, + locator=item.locator, + ) diff --git a/requirements.txt b/requirements.txt index 836025197..7fa6f6ba1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -66,7 +66,7 @@ pyasn1 pyasn1-modules pycodestyle pycparser -pydantic>=2,<3 +pydantic>=2.4.0,<3 pyee pyflakes PyGithub diff --git a/scripts/evaluate_librarian.py b/scripts/evaluate_librarian.py index 53a72263a..aac3effed 100644 --- a/scripts/evaluate_librarian.py +++ b/scripts/evaluate_librarian.py @@ -1,11 +1,17 @@ #!/usr/bin/env python -"""Module C regression harness — Week 1 skeleton. +"""Module C regression harness — Week 2: C.0 deterministic input boundary. -Loads + validates the golden dataset, applies the TRACT hub-firewall (on by -default), and prints per-slice counts. No linking yet: predictions are empty, so -this only proves the dataset, scorer, and firewall wire together. Later weeks -plug the C.0 -> C.4 pipeline in where ``predict()`` is stubbed below, and swap -the stub hub for the real CRE vector hub. +On top of the W1 skeleton (golden dataset + scorer + TRACT hub-firewall), +the harness now runs every golden row through the C.0 boundary: + +1. SectionValidator — each row is adapted to a synthetic knowledge_queue row + and must validate into an internal ``Section``; the harness prints the + validation pass rate per slice. +2. ExplicitLinkResolver — sections citing a CRE id resolve deterministically + (no ML); the explicit slice is gated at 100% correctness. + +The semantic path (retriever W3, cross-encoder W4) is still stubbed: rows +without an explicit reference yield no predictions. """ import argparse @@ -13,15 +19,29 @@ import os import sys from collections import Counter -from typing import List +from typing import List, Set # Bootstrap project root onto sys.path so this runs as a standalone script. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from application.utils.librarian.config_loader import load_config +from application.utils.librarian.explicit_link_resolver import ( + ResolutionOutcome, + resolve, +) from application.utils.librarian.hub_firewall import HubRep, firewall, leaks from application.utils.librarian.schemas import GoldenDatasetRow from application.utils.librarian.scoring import score_case +from application.utils.librarian.section_validator import ( + Section, + SectionValidationError, + section_from_queue_row, +) + +# Harness-only synthetic provenance: golden rows are not queue rows, so we +# synthesize the minimum B-shaped row needed to exercise the C.0 boundary. +_SYNTHETIC_SHA = "0" * 40 +_SYNTHETIC_CREATED_AT = "2026-06-01T00:00:00Z" def load_dataset(path: str) -> List[GoldenDatasetRow]: @@ -30,21 +50,53 @@ def load_dataset(path: str) -> List[GoldenDatasetRow]: return [GoldenDatasetRow.model_validate(row) for row in raw] +def queue_row_from_golden(row: GoldenDatasetRow) -> dict: + """Adapt a golden row into the knowledge_queue shape C.0 validates.""" + standard = row.input.source_standard.value if row.input.source_standard else "OTHER" + return { + "id": row.id, + "source_repo": f"golden/{standard}", + "source_path": row.provenance.section_path or "unknown.md", + "source_commit_sha": _SYNTHETIC_SHA, + "text": row.input.text, + "confidence": 0.99, + "llm_label": "KNOWLEDGE", + "created_at": _SYNTHETIC_CREATED_AT, + "consumed_at": None, + } + + def build_stub_hub(rows: List[GoldenDatasetRow]) -> List[HubRep]: - # W1 stub: the golden standards are already linked into OpenCRE, so seed the - # hub from their own text. This is the leakage the firewall must strip. + # The golden standards are already linked into OpenCRE, so seed the hub + # from their own text. This is the leakage the firewall must strip. # W3 replaces this with the real CRE vector hub. return [HubRep(row.id, row.input.text) for row in rows] -def predict(row: GoldenDatasetRow, hub: List[HubRep]) -> List[str]: - # W1 stub: no retriever/ranker yet. Returns no predictions. +def known_cre_ids(rows: List[GoldenDatasetRow]) -> Set[str]: + # Harness registry stub: every CRE id the golden set references is a real + # cre.external_id (see provenance). W3 swaps in the DB-backed id set. + ids: Set[str] = set() + for row in rows: + ids.update(row.expected.cre_ids or []) + if row.input.explicit_cre_ref: + ids.add(row.input.explicit_cre_ref) + return ids + + +def predict(section: Section, registry: Set[str], hub: List[HubRep]) -> List[str]: + # C.0.5: deterministic explicit path. Unknown/conflicting references route + # to review (no auto-link), so they predict nothing here. + resolution = resolve(section.text, registry) + if resolution.outcome == ResolutionOutcome.resolved: + return list(resolution.cre_ids) + # Semantic path (C.1 retriever + C.2 ranker) lands W3/W4. return [] def main(argv: List[str]) -> int: cfg = load_config() - parser = argparse.ArgumentParser(description="Module C eval harness (W1 skeleton)") + parser = argparse.ArgumentParser(description="Module C eval harness (W2: C.0)") parser.add_argument("--dataset", required=True, help="path to golden_dataset.json") parser.add_argument("--slice", help="only evaluate this slice") parser.add_argument("--limit", type=int, help="cap number of rows") @@ -52,7 +104,7 @@ def main(argv: List[str]) -> int: parser.add_argument("--top_k_retrieval", type=int, default=cfg.top_k_retrieval) parser.add_argument("--top_k_rerank", type=int, default=cfg.top_k_rerank) parser.add_argument( - "--dry_run", action="store_true", help="no writes (always true in W1)" + "--dry_run", action="store_true", help="no writes (always true pre-W8)" ) parser.add_argument( "--no_hub_firewall", @@ -68,26 +120,54 @@ def main(argv: List[str]) -> int: rows = rows[: args.limit] hub = build_stub_hub(rows) + registry = known_cre_ids(rows) firewall_on = not args.no_hub_firewall per_slice = Counter(r.slice.value for r in rows) + validated_per_slice: Counter = Counter() correct = 0 stripped = 0 + explicit_total = 0 + explicit_correct = 0 for row in rows: - hub_view = firewall(row.input.text, hub) if firewall_on else hub - if firewall_on and leaks(row.input.text, hub): + try: + section = section_from_queue_row(queue_row_from_golden(row)) + except SectionValidationError: + continue # rejected at the boundary; counts against the pass rate + validated_per_slice[row.slice.value] += 1 + + hub_view = firewall(section.text, hub) if firewall_on else hub + if firewall_on and leaks(section.text, hub): stripped += 1 - if score_case(row.expected.cre_ids or [], predict(row, hub_view)): + case_correct = score_case( + row.expected.cre_ids or [], predict(section, registry, hub_view) + ) + if case_correct: correct += 1 + if row.slice.value == "explicit": + explicit_total += 1 + if case_correct: + explicit_correct += 1 print(f"loaded {len(rows)} golden rows from {args.dataset}") + print("validation pass rate (C.0 boundary):") for slice_name in sorted(per_slice): - print(f" {slice_name:<14} {per_slice[slice_name]}") + passed = validated_per_slice[slice_name] + total = per_slice[slice_name] + print(f" {slice_name:<14} {passed}/{total} ({passed / total:.0%})") print( f"hub-firewall: {'ON' if firewall_on else 'OFF'}; " f"stripped {stripped} leaking hub entries" ) - print(f"correct (W1 stub, no predictions): {correct}/{len(rows)}") + if explicit_total: + gate_ok = explicit_correct == explicit_total + print( + f"explicit slice (C.0.5 resolver): {explicit_correct}/{explicit_total} " + f"— gate 100%: {'PASS' if gate_ok else 'FAIL'}" + ) + if not gate_ok: + return 1 + print(f"correct overall (semantic path still stubbed): {correct}/{len(rows)}") return 0