From 295b0586cfe5b52acb8305696277f65d61ca67c8 Mon Sep 17 00:00:00 2001 From: TillQuandel Date: Sat, 18 Jul 2026 17:17:26 +0200 Subject: [PATCH] =?UTF-8?q?feat(orchestrator):=20--book-mode=20=E2=80=94?= =?UTF-8?q?=20Planung=20je=20Hauptkapitel,=20Extraktion=20global=20(#346)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neues CLI-Flag --book-mode (mutually exclusive mit dem deprecated --by-chapter). Kapitel partitionieren NUR die Planung, nie die Extraktion — Notes synthetisieren weiterhin über Kapitelgrenzen (Kern-Kontrakt). Ablauf: Planner je Hauptkapitel (nur bei nutzbarem Outline-Split, sonst transparenter Normalpfad-Fallback mit Diagnose) → Kandidaten mit index-alignierten Kapitel-Keys → secondary_mentions kapitelübergreifend dedupliziert in related_mentions → globale dedup_concept_candidates → Skip-Action-Filter vor dem Cap → Budget min(n_Kapitel x 12, 120) → wortanteil-balancierter cap_candidates_balanced → EIN globaler Extractor-Lauf über den Volltext. Background-Extractor aus (wie by-chapter). _run_planner_stage als reiner 1:1-Refactor aus _plan_and_extract (Planner-Call + Halluzinations-Filter); Normal-/by-chapter-Verhalten unverändert. config: BOOK_MODE_CONCEPTS_PER_CHAPTER=12, BOOK_MODE_MAX_TOTAL=120 (ENV-Escape-Hatch). Closes #346. --- generative/config.py | 7 + generative/orchestrator.py | 200 ++++++++++++++-- generative/tests/test_book_mode_wiring.py | 268 ++++++++++++++++++++++ generative/ui_strings.py | 23 +- 4 files changed, 482 insertions(+), 16 deletions(-) create mode 100644 generative/tests/test_book_mode_wiring.py diff --git a/generative/config.py b/generative/config.py index 68fe80f..0289c66 100644 --- a/generative/config.py +++ b/generative/config.py @@ -152,6 +152,13 @@ # Outline-Split zu Slivern degeneriert (Titel-Marker-Fehlmap) → Normalpfad. MIN_CHAPTER_SEGMENT_WORDS = int(os.getenv("ATOMIC_AGENT_MIN_CHAPTER_SEGMENT_WORDS", "400")) +# Hybrid-Buchplanung (#346, --book-mode): Kandidaten-Budget für den globalen Cap. +# Formel beim Aufrufer: min(n_Kapitel × BOOK_MODE_CONCEPTS_PER_CHAPTER, +# BOOK_MODE_MAX_TOTAL). Kalibrierungs-Startwerte (Plan v4 §Kalibrierung), NICHT gegen +# ein Gold-Set validiert — ENV nur als Escape-Hatch. +BOOK_MODE_CONCEPTS_PER_CHAPTER = int(os.getenv("ATOMIC_AGENT_BOOK_MODE_CONCEPTS_PER_CHAPTER", "12")) +BOOK_MODE_MAX_TOTAL = int(os.getenv("ATOMIC_AGENT_BOOK_MODE_MAX_TOTAL", "120")) + # Backlog: nicht verdrahtet — Einlösung = Kosten-Cap-Feature, Maintainer-Entscheid MAX_TOKENS_PER_RUN = 500_000 diff --git a/generative/orchestrator.py b/generative/orchestrator.py index 219621a..9335e40 100644 --- a/generative/orchestrator.py +++ b/generative/orchestrator.py @@ -108,6 +108,8 @@ def _reconfigure_streams_utf8() -> None: MODEL_LLM_DEDUP, MAX_CHUNKS_SHORT_DOC, MAX_PAGES_SHORT_DOC, + BOOK_MODE_CONCEPTS_PER_CHAPTER, + BOOK_MODE_MAX_TOTAL, REDUNDANT_SIBLING_COSINE_THRESHOLD, ENABLE_FAITHFULNESS_GATE, TITLE_PRESENCE_COSINE_THRESHOLD, @@ -163,7 +165,7 @@ def _extract_primary_authors(citation: CitationMeta | None) -> list[str]: return authors -def _background_extractor_by_chapter_skip_line(gate_enabled: bool) -> str | None: +def _background_extractor_by_chapter_skip_line(gate_enabled: bool, mode: str = "--by-chapter") -> str | None: """#102: Sichtbarkeits-Fix für den --by-chapter-Pfad. Stage 4.5 (Background- Extractor) läuft dort NIE (Background-Calls würden sich pro Kapitel multiplizieren) — bewusste Kosten-Entscheidung (Variante a), keine @@ -173,10 +175,14 @@ def _background_extractor_by_chapter_skip_line(gate_enabled: bool) -> str | None Gibt die Log-Zeile nur zurück wenn das Gate an ist — sonst doppelt- verwirrend, weil dann ohnehin nichts liefe (analog zum Single-Doc- else-Zweig, der ebenfalls nur bei Bedarf meldet). + + `mode` (#346): --book-mode teilt dieselbe Kosten-Logik (Background pro Kapitel + zu teuer) und meldet über denselben Kanal — Default bleibt --by-chapter + (Bestandsaufrufer unverändert). """ if not gate_enabled: return None - return "[4.5/7] Background-Extractor: übersprungen im --by-chapter-Modus (bewusst — Kosten pro Kapitel)" + return f"[4.5/7] Background-Extractor: übersprungen im {mode}-Modus (bewusst — Kosten pro Kapitel)" _RESCUE_WINDOW_WORDS = 1200 # #308: Fenster-Rescue-Größe (3x window_words=400) @@ -2219,6 +2225,37 @@ def _build_citation( return citation +def _run_planner_stage( + *, + plan_text: str, + hall_text: str, + relevance_profile: dict, + citation: CitationMeta | None, + source_name: str, + n_chunks: int, + hall_ellipsis: str, +) -> ConceptPlan: + """Planner-Call + Halluzinations-Filter — 1:1-Extrakt aus `_plan_and_extract` (#346). + + Reiner Refactor, identisches Verhalten: `planner.run` auf `plan_text`, + `filter_hallucinated` gegen `hall_text`, Diagnose-Print mit `hall_ellipsis`. + Der `--book-mode`-Pfad ruft diese Stufe je Hauptkapitel auf + (plan_text=hall_text=Kapiteltext); `_plan_and_extract` nutzt sie für Normal- und + by-chapter-Pfad. Der Konzept-Cap (`cap_actionable_concepts`) bleibt bewusst beim + Aufrufer — book-mode fährt statt dessen einen wortanteil-balancierten Cap. + """ + primary_authors = _extract_primary_authors(citation) + with _span("Planner", pdf=source_name, n_chunks=n_chunks): + plan = planner.run(plan_text, relevance_profile, primary_authors=primary_authors) + plan, hallucinated = planner.filter_hallucinated(plan, hall_text) + if hallucinated: + print( + f" {len(hallucinated)} halluzinierte Konzepte verworfen: " + f"{', '.join(hallucinated[:3])}{hall_ellipsis if len(hallucinated) > 3 else ''}" + ) + return plan + + @dataclasses.dataclass(frozen=True) class _PlanExtractResult: """Ergebnis EINER `_plan_and_extract`-Invokation (ein Text-Scope). @@ -2277,15 +2314,15 @@ def _plan_and_extract( - Ellipsis (Div. 6): `hall_ellipsis` bewahrt die pfad-eigene Schreibweise der Halluzinations-Zeile ("..." by-chapter / "…" Normalpfad). """ - primary_authors = _extract_primary_authors(citation) - with _span("Planner", pdf=source_name, n_chunks=n_chunks): - plan = planner.run(plan_text, relevance_profile, primary_authors=primary_authors) - plan, hallucinated = planner.filter_hallucinated(plan, hall_text) - if hallucinated: - print( - f" {len(hallucinated)} halluzinierte Konzepte verworfen: " - f"{', '.join(hallucinated[:3])}{hall_ellipsis if len(hallucinated) > 3 else ''}" - ) + plan = _run_planner_stage( + plan_text=plan_text, + hall_text=hall_text, + relevance_profile=relevance_profile, + citation=citation, + source_name=source_name, + n_chunks=n_chunks, + hall_ellipsis=hall_ellipsis, + ) if runtime_config is not None: plan.concepts, _capped = cap_actionable_concepts(plan.concepts, cap_budget) if _capped: @@ -2370,6 +2407,112 @@ def _plan_and_extract( ) +def _run_book_mode( + *, + text: str, + chunks: list, + relevance_profile: dict, + existing_concepts: dict, + citation: CitationMeta | None, + tag_whitelist: list, + runtime_config, + source_name: str, +) -> tuple[list, dict, int, list, list]: + """Hybrid-Buchplanung (#346, --book-mode): Kapitel partitionieren NUR die Planung. + + Ablauf: Planner je Hauptkapitel (`_run_planner_stage` auf `chunk.text`) → + Kandidaten mit index-alignierten Kapitel-Keys sammeln (Key = `chunk.title`), + secondary_mentions kapitelübergreifend dedupliziert in `related_mentions` → + globale `dedup_concept_candidates` → Skip-Action-Kandidaten heraus (wie der + actionable-Filter des Normalpfads, PR-2-Review) → Budget + `min(n_Kapitel × BOOK_MODE_CONCEPTS_PER_CHAPTER, BOOK_MODE_MAX_TOTAL)` → + wortanteil-balancierter `cap_candidates_balanced` → EIN globaler Extractor-Lauf + über den **Volltext** (`extract_text=text` — Notes synthetisieren über + Kapitelgrenzen, das ist der Kern-Kontrakt). + + Background-Extractor läuft hier NIE (wie by-chapter, #102). Rückgabe: + (drafts, concept_map, dropped, extractor_failures, related_mentions). + """ + n_chapters = len(chunks) + print(f"[4-5/7] Planner (Buch-Modus): {n_chapters} Hauptkapitel einzeln planen, dann global extrahieren") + _skip_line = _background_extractor_by_chapter_skip_line(ENABLE_BACKGROUND_EXTRACTOR, mode="--book-mode") + if _skip_line: + print(_skip_line) + + candidates: list = [] + chapter_keys: list = [] + chapter_word_counts: dict = {} + related_mentions: list[str] = [] + source_title = "" + source_summary = "" + + for i, chunk in enumerate(chunks, 1): + key = chunk.title # deterministischer Kapitel-Key aus der Outline + chapter_word_counts.setdefault(key, len(chunk.text.split())) + preview = chunk.title[:60] + ("..." if len(chunk.title) > 60 else "") + print(f"\n[4/7] Kapitel {i}/{n_chapters}: {preview}") + if not chunk.text.strip(): + print(" Leerer Chunk, uebersprungen") + continue + plan = _run_planner_stage( + plan_text=chunk.text, + hall_text=chunk.text, + relevance_profile=relevance_profile, + citation=citation, + source_name=source_name, + n_chunks=n_chapters, + hall_ellipsis="…", + ) + if not source_title: + source_title, source_summary = plan.source_title, plan.source_summary + for c in plan.concepts: + if c.origin == "secondary_mention": + if c.title not in related_mentions: + related_mentions.append(c.title) + continue + candidates.append(c) + chapter_keys.append(key) + + # Globale Dedup über Kapitelgrenzen. Survivor sind Original-Objekte → Kapitel-Key + # per id() rückverfolgbar (index-aligned zur deduplizierten Liste rekonstruieren). + deduped, n_dupes = dedup_concept_candidates(candidates) + key_by_id = {id(c): k for c, k in zip(candidates, chapter_keys)} + deduped_keys = [key_by_id[id(c)] for c in deduped] + + # Skip-Action-Kandidaten VOR dem Cap heraus (PR-2-Review): sie dürfen keinen + # Budget-Slot belegen. secondary_mentions sind bereits oben ausgeschleust. + actionable_pairs = [(c, k) for c, k in zip(deduped, deduped_keys) if c.action != "skip"] + actionable = [c for c, _ in actionable_pairs] + actionable_keys = [k for _, k in actionable_pairs] + + budget = min(n_chapters * BOOK_MODE_CONCEPTS_PER_CHAPTER, BOOK_MODE_MAX_TOTAL) + kept, capped = cap_candidates_balanced(actionable, budget, chapter_word_counts, actionable_keys) + print( + f" [book-mode] {len(candidates)} Rohkandidaten → {len(deduped)} nach Dedup " + f"({n_dupes} Duplikate) → {len(kept)} nach Cap " + f"(Budget {budget}, {len(capped)} über Budget verworfen)" + ) + + merged_plan = ConceptPlan(source_title, source_summary, kept) + + print(f"\n[5/7] Extractor: {len(kept)} Konzepte global über den Volltext…") + with _span("Extractor", pdf=source_name, n_concepts=len(kept)): + drafts, concept_map, dropped, failures = asyncio.run( + run_extractors_per_concept( + text, + merged_plan, + existing_concepts, + citation=citation, + tag_whitelist=tag_whitelist, + background_map={}, + related_mentions=related_mentions, + max_concurrent_calls=(runtime_config.max_concurrent_calls if runtime_config is not None else None), + ) + ) + print(f" {len(drafts)} Draft-Notes extrahiert") + return drafts, concept_map, dropped, failures, related_mentions + + def _run_extraction_stages( args, source_path: Path, runtime_config=None ): # main() übergibt immer einen RuntimeConfig; None = kein Runtime-Config / Capping deaktiviert @@ -2437,7 +2580,17 @@ def _run_extraction_stages( f"Ohne eingebettete Lesezeichen ist --by-chapter für dieses PDF ungeeignet." ) if len(chunks) > LARGE_DOC_THRESHOLD and not getattr(args, "by_chapter", False): - print(f" [WARN] {len(chunks)} Chunks - großes Dokument. Erwäge --by-chapter für Bücher.") + print(f" [WARN] {len(chunks)} Chunks - großes Dokument. Erwäge --by-chapter/--book-mode für Bücher.") + # #346: book-mode braucht einen echten Outline-Split (>1 Kapitel). Ohne nutzbare + # Outline transparenter Fallback auf den Normalpfad + sichtbare Diagnose (der + # Feedback-Kanal für reale Outline-Verteilungen bei Fremd-Beständen, Plan v4). + _book_mode_requested = getattr(args, "book_mode", False) + _book_mode_active = _book_mode_requested and _split_source == "outline" and len(chunks) > 1 + if _book_mode_requested and not _book_mode_active: + print( + f" [book-mode] Kein nutzbarer Kapitel-Split (Quelle: {_split_source}, " + f"{len(chunks)} Segment(e)) — transparenter Fallback auf den Normalpfad." + ) acronym_dict = acronym_fix.extract_acronym_pairs(text) if acronym_dict: print( @@ -2550,7 +2703,20 @@ def _run_extraction_stages( background_map: dict = {} related_mentions: list[str] = [] - if getattr(args, "by_chapter", False) and len(chunks) > 1: + if _book_mode_active: + # --- Schritt 4+5: Hybrid-Buchplanung (#346) — Planung je Kapitel, Extraktion + # global über den Volltext. Background-Extractor aus (wie by-chapter, #102). + drafts, concept_map, dropped_total, extractor_failures, related_mentions = _run_book_mode( + text=text, + chunks=chunks, + relevance_profile=relevance_profile, + existing_concepts=existing_concepts, + citation=citation, + tag_whitelist=tag_whitelist, + runtime_config=runtime_config, + source_name=source_path.name, + ) + elif getattr(args, "by_chapter", False) and len(chunks) > 1: # --- Schritt 4+5: Planner + Extractor kapitelweise --- print("[4-5/7] Planner + Extractor: Kapitel einzeln verarbeiten") # #102: Background-Extractor läuft hier bewusst NICHT — Sichtbarkeit @@ -2771,7 +2937,13 @@ def main(argv: list[str] | None = None): ap.add_argument("--source", default=None, help=msg("orch.arg.source")) ap.add_argument("--doi", default=None, help=msg("orch.arg.doi")) ap.add_argument("--dry-run", action="store_true", help=msg("orch.arg.dry_run")) - ap.add_argument("--by-chapter", action="store_true", help=msg("orch.arg.by_chapter")) + # #346: --by-chapter (deprecated) und --book-mode partitionieren beide die + # Kapitel — aber --by-chapter partitioniert auch die EXTRAKTION (verliert + # kapitelübergreifende Synthese), --book-mode nur die Planung. Gegenseitig + # ausschließend, damit kein widersprüchlicher Doppelmodus entsteht. + _mode_group = ap.add_mutually_exclusive_group() + _mode_group.add_argument("--by-chapter", action="store_true", help=msg("orch.arg.by_chapter")) + _mode_group.add_argument("--book-mode", action="store_true", help=msg("orch.arg.book_mode")) ap.add_argument("--no-llm", action="store_true", help=msg("orch.arg.no_llm")) ap.add_argument("--target-tag", default=None, help=msg("orch.arg.target_tag")) ap.add_argument("--llm-fallback", action="store_true", help=msg("orch.arg.llm_fallback")) diff --git a/generative/tests/test_book_mode_wiring.py b/generative/tests/test_book_mode_wiring.py new file mode 100644 index 0000000..caf2d25 --- /dev/null +++ b/generative/tests/test_book_mode_wiring.py @@ -0,0 +1,268 @@ +"""#346: --book-mode — Planung je Hauptkapitel, Extraktion global. + +Kern-Kontrakt (drei Text-Scopes, explizit getestet): +- Normalpfad extrahiert aus dem Volltext (plant auf Overview). +- --by-chapter extrahiert je Kapitel aus chunk.text (partitioniert auch die Extraktion). +- --book-mode plant lokal je Hauptkapitel, extrahiert EINMAL global über den Volltext + (Notes können über Kapitelgrenzen synthetisieren — das ist der Kern-Kontrakt). + +Muster: test_by_chapter_background_extractor_skip.py (Stub der Stage-1–5-Senken, +Integration über _run_extraction_stages). +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from generative import orchestrator +from generative.schemas.atomic_note import ConceptItem, ConceptPlan, QualityReport + + +def _chunk(title: str, text: str, source: str) -> SimpleNamespace: + return SimpleNamespace(title=title, text=text, source=source) + + +def _concept(title: str, *, priority: str = "high", action: str = "create", origin: str = "primary") -> ConceptItem: + return ConceptItem(title=title, priority=priority, chapter="", action=action, origin=origin) + + +def _install(monkeypatch, chunks, plans, *, gate: bool = False): + """Stubt die Stage-1–5-Senken und gibt (calls, full_text) zurück. + + plans: dict plan_text -> Konzeptliste (planner.run liefert daraus). full_text ist + der von pdf_to_text zurückgegebene Volltext (Kern-Assertion des Extractor-Scopes). + """ + pc = orchestrator.pdf_chunker + full_text = "VOLLTEXT_WORT " * 80 + monkeypatch.setattr(pc, "pdf_to_text", lambda *_a, **_k: full_text) + monkeypatch.setattr(pc, "split_by_chapters", lambda *_a, **_k: chunks) + monkeypatch.setattr( + pc, "pdf_metadata", lambda *_a, **_k: {"Author": "Autor", "Year": "2020", "Title": "Titel", "Pages": "300"} + ) + monkeypatch.setattr(pc, "extract_overview", lambda *_a, **_k: "Überblick") + monkeypatch.setattr(orchestrator.acronym_fix, "extract_acronym_pairs", lambda *_a, **_k: {}) + monkeypatch.setattr( + orchestrator.context_builder, + "build_relevance_profile", + lambda *_a, **_k: {"existing_concepts": [], "tag_whitelist": []}, + ) + monkeypatch.setattr(orchestrator.context_builder, "build_concept_links", lambda *_a, **_k: {}) + monkeypatch.setattr( + orchestrator.quality, + "check_quality", + lambda **_kw: QualityReport(peer_reviewed=None, citation_count=None, retracted=False, flags=[]), + ) + + calls: dict = {"planner": [], "extract": []} + + def _planner_run(plan_text, _profile, **_kw): + calls["planner"].append(plan_text) + return ConceptPlan("Titel", "Summary", list(plans.get(plan_text, []))) + + monkeypatch.setattr(orchestrator.planner, "run", _planner_run) + monkeypatch.setattr(orchestrator.planner, "filter_hallucinated", lambda plan, _text: (plan, [])) + + async def _extract(full, plan, *_a, **_k): + calls["extract"].append( + SimpleNamespace( + full_text=full, + concepts=list(plan.concepts), + related=list(_k.get("related_mentions") or []), + ) + ) + return ([], {}, 0, []) # #210: 4. Rückgabewert = extractor_failures + + monkeypatch.setattr(orchestrator, "run_extractors_per_concept", _extract) + monkeypatch.setattr(orchestrator, "ENABLE_BACKGROUND_EXTRACTOR", gate) + return calls, full_text + + +def _args(**over) -> SimpleNamespace: + base = dict(book_mode=False, by_chapter=False, dry_run=True, doi=None, llm_fallback=False) + base.update(over) + return SimpleNamespace(**base) + + +# --- Kern-Kontrakt: drei Text-Scopes ------------------------------------------- + + +def test_normal_path_extracts_from_full_text(monkeypatch): + chunks = [_chunk("Abschnitt 1", "abschnitt eins " * 40, "words")] + calls, full_text = _install(monkeypatch, chunks, {"Überblick": [_concept("Konzept X")]}) + + orchestrator._run_extraction_stages(_args(), Path("paper.pdf"), None) + + assert len(calls["planner"]) == 1 + assert calls["planner"][0] == "Überblick" # Normalpfad plant auf Overview + assert len(calls["extract"]) == 1 + assert calls["extract"][0].full_text == full_text # Extractor über den Volltext + + +def test_by_chapter_extracts_from_chunk_text(monkeypatch): + chunks = [ + _chunk("Kapitel 1", "KAPITEL_EINS " * 40, "outline"), + _chunk("Kapitel 2", "KAPITEL_ZWEI " * 40, "outline"), + ] + plans = {chunks[0].text: [_concept("A")], chunks[1].text: [_concept("B")]} + calls, _ = _install(monkeypatch, chunks, plans) + + orchestrator._run_extraction_stages(_args(by_chapter=True), Path("book.pdf"), None) + + # Ein Planner- UND ein Extractor-Call je Kapitel; extrahiert aus chunk.text (nicht Volltext). + assert len(calls["planner"]) == 2 + assert len(calls["extract"]) == 2 + assert {c.full_text for c in calls["extract"]} == {chunks[0].text, chunks[1].text} + + +def test_book_mode_plans_locally_extracts_globally(monkeypatch): + """Kern-Regressionstest: book-mode plant je Kapitel, extrahiert EINMAL mit Volltext.""" + chunks = [ + _chunk("Kapitel 1", "kap eins " * 40, "outline"), + _chunk("Kapitel 2", "kap zwei " * 40, "outline"), + ] + plans = { + chunks[0].text: [_concept("Konzept A"), _concept("Konzept B")], + chunks[1].text: [_concept("Konzept C")], + } + calls, full_text = _install(monkeypatch, chunks, plans) + + orchestrator._run_extraction_stages(_args(book_mode=True), Path("book.pdf"), None) + + assert len(calls["planner"]) == 2 # Planner je Hauptkapitel + assert len(calls["extract"]) == 1 # EIN globaler Extractor-Lauf + assert calls["extract"][0].full_text == full_text # … mit dem Volltext + titles = {c.title for c in calls["extract"][0].concepts} + assert titles == {"Konzept A", "Konzept B", "Konzept C"} # Kandidaten aus beiden Kapiteln + + +# --- Fallback / Wiring --------------------------------------------------------- + + +def test_book_mode_without_outline_falls_back_to_normal(monkeypatch, capsys): + chunks = [ + _chunk("Abschnitt 1", "a " * 40, "heuristic"), + _chunk("Abschnitt 2", "b " * 40, "heuristic"), + ] + calls, full_text = _install(monkeypatch, chunks, {"Überblick": [_concept("Konzept X")]}) + + orchestrator._run_extraction_stages(_args(book_mode=True), Path("nooutline.pdf"), None) + + # Kein Outline-Split → transparenter Normalpfad: Planner genau 1x auf Overview. + assert len(calls["planner"]) == 1 + assert calls["planner"][0] == "Überblick" + assert len(calls["extract"]) == 1 + assert calls["extract"][0].full_text == full_text + assert "book-mode" in capsys.readouterr().out.lower() # sichtbare Diagnose + + +def test_planner_calls_equal_chapters_and_candidates_within_budget(monkeypatch): + monkeypatch.setattr(orchestrator, "BOOK_MODE_CONCEPTS_PER_CHAPTER", 2) + monkeypatch.setattr(orchestrator, "BOOK_MODE_MAX_TOTAL", 100) + chunks = [ + _chunk("Kapitel 1", "eins " * 60, "outline"), + _chunk("Kapitel 2", "zwei " * 20, "outline"), + ] + plans = { + chunks[0].text: [_concept(f"K1-{i}") for i in range(3)], + chunks[1].text: [_concept(f"K2-{i}") for i in range(3)], + } + calls, _ = _install(monkeypatch, chunks, plans) + + orchestrator._run_extraction_stages(_args(book_mode=True), Path("book.pdf"), None) + + assert len(calls["planner"]) == 2 # Planner-Calls == Kapitelzahl + budget = min(2 * 2, 100) # = 4 + assert len(calls["extract"][0].concepts) == budget # Kandidaten ≤ Budget (hier == 4) + + +def test_book_mode_filters_skip_actions_before_cap(monkeypatch): + """Skip-action-Kandidaten VOR dem Cap heraus — sie erreichen den Extractor nie.""" + chunks = [ + _chunk("Kapitel 1", "eins " * 40, "outline"), + _chunk("Kapitel 2", "zwei " * 40, "outline"), + ] + plans = { + chunks[0].text: [_concept("Keep A"), _concept("Skip Me", action="skip")], + chunks[1].text: [_concept("Keep B")], + } + calls, _ = _install(monkeypatch, chunks, plans) + + orchestrator._run_extraction_stages(_args(book_mode=True), Path("book.pdf"), None) + + titles = {c.title for c in calls["extract"][0].concepts} + assert titles == {"Keep A", "Keep B"} + assert "Skip Me" not in titles + + +def test_book_mode_dedups_secondary_mentions_across_chapters(monkeypatch): + chunks = [ + _chunk("Kapitel 1", "eins " * 40, "outline"), + _chunk("Kapitel 2", "zwei " * 40, "outline"), + ] + plans = { + chunks[0].text: [_concept("Primär A"), _concept("Wilson", origin="secondary_mention")], + chunks[1].text: [_concept("Primär B"), _concept("Wilson", origin="secondary_mention")], + } + calls, _ = _install(monkeypatch, chunks, plans) + + orchestrator._run_extraction_stages(_args(book_mode=True), Path("book.pdf"), None) + + # Sekundär-Erwähnung nicht als primäres Konzept extrahiert … + assert {c.title for c in calls["extract"][0].concepts} == {"Primär A", "Primär B"} + # … sondern EINMAL (kapitelübergreifend dedupliziert) in related_mentions. + assert calls["extract"][0].related.count("Wilson") == 1 + + +def test_book_mode_skips_background_extractor(monkeypatch): + """Background-Extractor läuft im book-mode nie — auch bei aktivem Gate (#102-Analogie).""" + chunks = [ + _chunk("Kapitel 1", "eins " * 40, "outline"), + _chunk("Kapitel 2", "zwei " * 40, "outline"), + ] + plans = {chunks[0].text: [_concept("A")], chunks[1].text: [_concept("B")]} + calls, _ = _install(monkeypatch, chunks, plans, gate=True) + + def _boom(*_a, **_k): + raise AssertionError("Background-Extractor darf im book-mode nie laufen") + + monkeypatch.setattr(orchestrator.background_extractor, "run", _boom) + + orchestrator._run_extraction_stages(_args(book_mode=True), Path("book.pdf"), None) + + assert len(calls["extract"]) == 1 # kein Crash → Background-Extractor nie aufgerufen + + +def test_book_mode_and_by_chapter_are_mutually_exclusive(): + with pytest.raises(SystemExit): + orchestrator.main(["--source", "x.pdf", "--by-chapter", "--book-mode"]) + + +# --- PR-2-Nachzügler (Mistral-Review): cap_candidates_balanced-Randfälle -------- + + +def test_cap_budget_zero_keeps_only_secondary_mentions(): + concepts = [ + _concept("A", origin="primary"), + _concept("B", origin="primary"), + _concept("S", origin="secondary_mention"), + ] + keys = ["Kap1", "Kap1", "Kap1"] + + kept, capped = orchestrator.cap_candidates_balanced(concepts, 0, {"Kap1": 100}, keys) + + assert [c.title for c in kept] == ["S"] # nur secondary_mention überlebt + assert {c.title for c in capped} == {"A", "B"} + + +def test_cap_missing_chapter_word_count_no_crash(): + """Kapitel-Key ohne chapter_word_counts-Eintrag → get(k, 0)-Fallback, kein KeyError.""" + concepts = [_concept("A"), _concept("B"), _concept("C")] + keys = ["Kap1", "Kap2", "Kap2"] # keiner in chapter_word_counts + + kept, capped = orchestrator.cap_candidates_balanced(concepts, 2, {}, keys) + + assert len(kept) == 2 # deterministisch gekappt statt Crash + assert len(kept) + len(capped) == 3 diff --git a/generative/ui_strings.py b/generative/ui_strings.py index e4fa776..8265f63 100644 --- a/generative/ui_strings.py +++ b/generative/ui_strings.py @@ -121,8 +121,27 @@ def msg(key: str, **fmt: object) -> str: "de": "Kein Schreiben in Vault", }, "orch.arg.by_chapter": { - "en": "Run planner and extractor chapter by chapter (for large books)", - "de": "Planner und Extractor kapitelweise ausführen (für große Bücher)", + "en": ( + "Deprecated (use --book-mode): run planner AND extractor chapter by chapter " + "— partitions extraction too, losing cross-chapter synthesis (for large books)" + ), + "de": ( + "Deprecated (nutze --book-mode): Planner UND Extractor kapitelweise — " + "partitioniert auch die Extraktion, verliert kapitelübergreifende Synthese " + "(für große Bücher)" + ), + }, + "orch.arg.book_mode": { + "en": ( + "Book mode: plan per main chapter (denser concepts), extract globally over the " + "full text so notes keep cross-chapter synthesis. Needs a usable PDF outline; " + "falls back to the normal path otherwise." + ), + "de": ( + "Buch-Modus: Planung je Hauptkapitel (dichtere Konzepte), Extraktion global über " + "den Volltext, damit Notes kapitelübergreifend synthetisieren. Braucht eine " + "nutzbare PDF-Outline; sonst Fallback auf den Normalpfad." + ), }, "orch.arg.no_llm": { "en": (