diff --git a/README.md b/README.md index ee7a83b..d91c513 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ When you run `code2llm ./ -f all`, the following files are created: |------|--------|---------|--------------| | `evolution.toon.yaml` | **YAML** | **πŸ“‹ Refactoring queue** - Prioritized improvements | 0 refactoring actions needed | | `map.toon.yaml` | **YAML** | **πŸ—ΊοΈ Structural map + project header** - Modules, imports, exports, signatures, stats, alerts, hotspots, trend | Project architecture overview | +| `projections.json` | **JSON** | **πŸ”Ž Bounded symbol catalog** (`code2llm.projection/v1`) for intent retrieval / external indexers β€” deterministic, digested, capped at 2048 records | Reuse contract without analyzer internals | ### πŸ€– LLM-Ready Documentation diff --git a/code2llm/cli_exports/formats.py b/code2llm/cli_exports/formats.py index 3385d9f..cef1335 100644 --- a/code2llm/cli_exports/formats.py +++ b/code2llm/cli_exports/formats.py @@ -30,6 +30,8 @@ EvolutionExporter, READMEExporter, ProjectYAMLExporter, + ProjectionsExporter, + PROJECTION_FILENAME, ToonViewGenerator, ContextViewGenerator, ArticleViewGenerator, @@ -195,6 +197,15 @@ def _export_simple_formats(args, result, output_dir: Path, formats): if args.verbose: print(f" - JSON: {filepath}") + if "projections" in formats: + exporter = ProjectionsExporter() + filepath = output_dir / PROJECTION_FILENAME + t0 = time.monotonic() + exporter.export(result, str(filepath)) + elapsed = time.monotonic() - t0 + if args.verbose: + print(f" - PROJECTIONS: {filepath} ({elapsed:.2f}s)") + def _export_yaml(args, result, output_dir: Path): """Export YAML with optional split/separated modes.""" diff --git a/code2llm/cli_exports/orchestrator_constants.py b/code2llm/cli_exports/orchestrator_constants.py index 21c6e3f..7ea3b66 100644 --- a/code2llm/cli_exports/orchestrator_constants.py +++ b/code2llm/cli_exports/orchestrator_constants.py @@ -20,6 +20,7 @@ "planfile": "planfile-tickets.yaml", "readme": "README.md", "project-yaml": "project.yaml", + "projections": "projections.json", } # Files produced per format in dry-run preview @@ -33,6 +34,7 @@ "yaml": ["analysis.yaml"], "json": ["analysis.json"], "readme": ["README.md"], + "projections": ["projections.json"], } # Human-readable labels @@ -47,6 +49,7 @@ "planfile": "PLANFILE (ticket suggestions)", "readme": "README (documentation)", "project-yaml": "PROJECT-YAML (single source of truth)", + "projections": "PROJECTIONS (intent retrieval)", } # --------------------------------------------------------------------------- diff --git a/code2llm/cli_parser.py b/code2llm/cli_parser.py index f372019..78a4aef 100644 --- a/code2llm/cli_parser.py +++ b/code2llm/cli_parser.py @@ -97,7 +97,7 @@ def create_parser() -> argparse.ArgumentParser: default="toon", help=( "Output formats: toon,map,flow,context,code2logic,yaml,json,mermaid," - "evolution,calls,png,project-yaml,planfile,all (default: toon)" + "evolution,calls,png,project-yaml,planfile,projections,all (default: toon)" ), ) diff --git a/code2llm/exporters/__init__.py b/code2llm/exporters/__init__.py index 5a3500d..d30a9fc 100644 --- a/code2llm/exporters/__init__.py +++ b/code2llm/exporters/__init__.py @@ -9,6 +9,7 @@ - READMEExporter β†’ README.md (documentation of all files) - YAMLExporter β†’ analysis.yaml - JSONExporter β†’ analysis.json + - ProjectionsExporter β†’ projections.json (bounded symbol catalog for intent retrieval) - MermaidExporter β†’ *.mmd Registry API: @@ -33,6 +34,13 @@ # Standard exporters (auto-registered via @export_format decorator) from .json_exporter import JSONExporter +from .projections_exporter import ( + MAX_RECORDS, + PROJECTION_FILENAME, + PROJECTION_SCHEMA, + ProjectionsExporter, + build_projection, +) from .yaml_exporter import YAMLExporter from .mermaid_exporter import MermaidExporter from .context_exporter import ContextExporter diff --git a/code2llm/exporters/projections_exporter.py b/code2llm/exporters/projections_exporter.py new file mode 100644 index 0000000..3127633 --- /dev/null +++ b/code2llm/exporters/projections_exporter.py @@ -0,0 +1,139 @@ +"""Bounded, versioned projections of code2llm analysis for intent retrieval. + +Emits ``projections.json``: a deterministic, capped catalog of source +symbols with provenance (kind, symbol, file, line) and content digests. +Downstream consumers (e.g. semcod/search) index it without parsing the +full analysis or importing analyzer internals β€” the reuse contract is +this artifact, not the Python API (PLF-035 / issue #25). + +Contract invariants: + - ``schema`` pins the record shape; ``producer.version`` pins the + analyzer. Bumping either invalidates downstream caches by equality + of the embedded digests alone. + - Byte-deterministic for identical analysis results: canonical JSON + (sorted keys), records sorted by (file, line, kind, symbol), no + timestamps inside records. ``generated_at`` is optional and never + contributes to the digest. + - Bounded: at most ``MAX_RECORDS`` records; symbol and file strings + truncated at ``FIELD_LIMIT`` characters; ``stats.truncated`` flags + an overflow so consumers can refuse silently-stale catalogs. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from code2llm import __version__ + +from .base import BaseExporter, export_format + +PROJECTION_SCHEMA = "code2llm.projection/v1" +PROJECTION_FILENAME = "projections.json" +MAX_RECORDS = 2048 +FIELD_LIMIT = 200 + +_KIND_ORDER = {"module": 0, "class": 1, "function": 2} + + +def _canonical(obj: Any) -> str: + return json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + + +def _digest(obj: Any) -> str: + return "sha256:" + hashlib.sha256(_canonical(obj).encode("utf-8")).hexdigest() + + +def _bounded(value: str) -> str: + return value[:FIELD_LIMIT] + + +def _record(kind: str, symbol: str, file: str, line: Optional[int]) -> Dict[str, Any]: + record: Dict[str, Any] = {"kind": kind, "symbol": _bounded(symbol), "file": _bounded(file)} + if line is not None: + record["line"] = int(line) + record["digest"] = _digest(record) + return record + + +def _items(value: Any) -> Iterable[Any]: + """Yield info objects from either a Dict[str, Info] or a list.""" + if isinstance(value, dict): + return value.values() + return value or [] + + +def _iter_records(result: Any) -> Iterable[Dict[str, Any]]: + modules = _items(getattr(result, "modules", None)) + classes = _items(getattr(result, "classes", None)) + functions = _items(getattr(result, "functions", None)) + for module in modules: + yield _record("module", module.name, module.file, None) + for cls in classes: + yield _record("class", cls.qualified_name or cls.name, cls.file, cls.line) + for func in functions: + yield _record("function", func.qualified_name or func.name, func.file, func.line) + + +def build_projection( + result: Any, + *, + max_records: int = MAX_RECORDS, + generated_at: Optional[str] = None, +) -> Dict[str, Any]: + """Build the projection document from an :class:`AnalysisResult`. + + Duck-typed on purpose: only ``modules``/``classes``/``functions`` + (``name``/``qualified_name``/``file``/``line``) are read, so tests and + alternative frontends can feed lightweight structures. + """ + records: List[Dict[str, Any]] = list(_iter_records(result)) + records.sort(key=lambda r: (r["file"], r.get("line", 0), _KIND_ORDER[r["kind"]], r["symbol"])) + truncated = len(records) > max_records + if truncated: + records = records[:max_records] + + stats = getattr(result, "stats", None) or {} + document: Dict[str, Any] = { + "schema": PROJECTION_SCHEMA, + "producer": {"tool": "code2llm", "version": __version__}, + "project": { + "path": _bounded(str(getattr(result, "project_path", "") or "")), + "analysis_mode": str(getattr(result, "analysis_mode", "") or ""), + }, + "stats": { + "files": int(stats.get("files", 0) or 0), + "lines": int(stats.get("lines", 0) or 0), + "symbols": len(records), + "truncated": truncated, + }, + "records": records, + "digest": _digest(records), + } + if generated_at: + document["generated_at"] = generated_at + return document + + +@export_format( + "projections", + description="Bounded symbol projections for intent retrieval (projections.json)", + extension=".json", +) +class ProjectionsExporter(BaseExporter): + """Export the deterministic projections catalog.""" + + def export( + self, + result: Any, + output_path: str, + generated_at: Optional[str] = None, + **kwargs: Any, + ) -> Optional[Path]: + document = build_projection(result, generated_at=generated_at) + path = self._ensure_dir(output_path) + payload = json.dumps(document, sort_keys=True, ensure_ascii=False, indent=1) + "\n" + self._write_text(path, payload) + return path diff --git a/tests/test_projections_exporter.py b/tests/test_projections_exporter.py new file mode 100644 index 0000000..08ad93c --- /dev/null +++ b/tests/test_projections_exporter.py @@ -0,0 +1,130 @@ +"""Contract tests for the bounded intent-retrieval projection (PLF-035 / #25). + +The projection is a reuse contract consumed by external indexers +(semcod/search): determinism, provenance, bounds and version stamping +are load-bearing, not incidental. +""" +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from code2llm import __version__ +from code2llm.exporters.projections_exporter import ( + FIELD_LIMIT, + MAX_RECORDS, + PROJECTION_SCHEMA, + ProjectionsExporter, + build_projection, +) + + +def _fake_result(modules=(), classes=(), functions=(), stats=None, path="proj", mode="standard"): + def mod(name, file): + return SimpleNamespace(name=name, file=file) + + def cls(name, file, line): + return SimpleNamespace(name=name, qualified_name=f"mod.{name}", file=file, line=line) + + def fn(name, file, line): + return SimpleNamespace(name=name, qualified_name=f"mod.{name}", file=file, line=line) + + return SimpleNamespace( + modules=[mod(*m) for m in modules], + classes=[cls(*c) for c in classes], + functions=[fn(*f) for f in functions], + stats=stats or {"files": 3, "lines": 100}, + project_path=path, + analysis_mode=mode, + ) + + +def test_schema_and_producer_version_are_pinned(): + doc = build_projection(_fake_result()) + assert doc["schema"] == PROJECTION_SCHEMA + assert doc["producer"] == {"tool": "code2llm", "version": __version__} + + +def test_records_carry_source_and_symbol_provenance(): + doc = build_projection( + _fake_result( + modules=[("core", "pkg/core/__init__.py")], + classes=[("Engine", "pkg/core/engine.py", 12)], + functions=[("run", "pkg/core/engine.py", 40)], + ) + ) + by_kind = {r["kind"]: r for r in doc["records"]} + assert by_kind["module"] == { + "kind": "module", + "symbol": "core", + "file": "pkg/core/__init__.py", + "digest": by_kind["module"]["digest"], + } + assert by_kind["class"]["symbol"] == "mod.Engine" + assert by_kind["class"]["file"] == "pkg/core/engine.py" + assert by_kind["class"]["line"] == 12 + assert by_kind["function"]["line"] == 40 + assert all(r["digest"].startswith("sha256:") for r in doc["records"]) + + +def test_deterministic_bytes_and_digest(): + result = _fake_result( + classes=[("B", "b.py", 2), ("A", "a.py", 1)], + functions=[("z", "a.py", 9), ("a", "b.py", 1)], + ) + first = build_projection(result) + second = build_projection(result) + assert first == second + assert first["digest"] == second["digest"] + # Record order is canonical: (file, line, kind, symbol). + order = [(r["file"], r.get("line", 0), r["kind"], r["symbol"]) for r in first["records"]] + assert order == sorted(order) + # generated_at never leaks into the digest. + stamped = build_projection(result, generated_at="2026-10-23T00:00:00Z") + assert stamped["digest"] == first["digest"] + + +def test_bounds_cap_records_and_flag_truncation(): + functions = [(f"f{i}", "m.py", i + 1) for i in range(MAX_RECORDS + 5)] + doc = build_projection(_fake_result(functions=functions)) + assert len(doc["records"]) == MAX_RECORDS + assert doc["stats"]["truncated"] is True + assert doc["stats"]["symbols"] == MAX_RECORDS + small = build_projection(_fake_result(functions=functions[:4])) + assert small["stats"]["truncated"] is False + + +def test_long_fields_are_truncated(): + long_name = "x" * (FIELD_LIMIT + 50) + doc = build_projection(_fake_result(functions=[(long_name, long_name + ".py", 1)])) + record = doc["records"][0] + assert len(record["symbol"]) == FIELD_LIMIT + assert len(record["file"]) == FIELD_LIMIT + + +def test_exporter_writes_canonical_json(tmp_path: Path): + result = _fake_result(classes=[("E", "e.py", 3)]) + exporter = ProjectionsExporter() + out = tmp_path / "projections.json" + path = exporter.export(result, str(out)) + assert path == out and out.exists() + body = out.read_text(encoding="utf-8") + again = exporter.export(result, str(tmp_path / "again.json")) + assert again.read_text(encoding="utf-8") == body + assert body.endswith("\n") + + +def test_registry_lists_projections(): + from code2llm.exporters import EXPORT_REGISTRY, get_exporter + + assert "projections" in EXPORT_REGISTRY + assert get_exporter("projections") is ProjectionsExporter + + +def test_cli_help_mentions_projections(capsys): + from code2llm.cli_parser import create_parser + + with pytest.raises(SystemExit) as exc: + create_parser().parse_args(["--help"]) + assert exc.value.code == 0 + assert "projections" in (capsys.readouterr().out or "")