From 85b0f6ac61d89d669d8d2e6d4ea2d0ceb636680c Mon Sep 17 00:00:00 2001 From: jay Date: Fri, 29 May 2026 10:54:00 -0700 Subject: [PATCH 01/17] fix title generation --- backend/orchestrator.py | 16 +++++++++------- backend/report_exporter.py | 4 +++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/backend/orchestrator.py b/backend/orchestrator.py index 7c31755..34c99d2 100644 --- a/backend/orchestrator.py +++ b/backend/orchestrator.py @@ -1797,7 +1797,7 @@ async def synthesize(self) -> AsyncIterator[ResponseMessage]: "- A short section title\n" "- One sentence describing what the section will cover\n\n" "Return a JSON object:\n" - '{"sections": [{"title": "...", "description": "..."}, ...]}' + '{"report_title": "...", "sections": [{"title": "...", "description": "..."}, ...]}' ) outline_response = await self.agents.report.run( @@ -1822,6 +1822,7 @@ async def synthesize(self) -> AsyncIterator[ResponseMessage]: # Guard against None/empty content — both produce a JSONDecodeError or # TypeError that the except block would swallow silently. sections: List[Dict[str, str]] = [] + report_title: str = query try: raw = outline_response.content or "" if raw.startswith("```"): @@ -1831,7 +1832,9 @@ async def synthesize(self) -> AsyncIterator[ResponseMessage]: elif "```" in raw: raw = raw.split("```")[1].split("```")[0].strip() if raw: - sections = json.loads(raw).get("sections", []) + parsed = json.loads(raw) + sections = parsed.get("sections", []) + report_title = parsed.get("report_title", query) else: logger.warning( "[Orchestrator] Outline response was empty after retry; using defaults" @@ -2103,7 +2106,7 @@ async def synthesize(self) -> AsyncIterator[ResponseMessage]: ] doc_parts: List[str] = [ - f"RESEARCH REPORT", + f"RESEARCH REPORT: {report_title}", f"Query: {query}", "", ] @@ -2147,14 +2150,13 @@ async def synthesize(self) -> AsyncIterator[ResponseMessage]: try: from report_exporter import export_report_pdf - await export_report_pdf(document, query, self.session_id) + await export_report_pdf(document, query, report_title, self.session_id) except Exception as _pdf_exc: logger.warning("[Orchestrator] PDF export error: %s", _pdf_exc) # ── Extract structured metadata for downstream consumers ──────────── - # title: query is the ground truth; report header carries no LLM-derived - # title in the multi-pass flow, so use the query directly. - report_title = query + # title was derived from the outline prompt during Phase A; falls back + # to the original query if the LLM did not supply one. # key_findings: content of the first section whose title contains # "finding" (case-insensitive), empty string if no such section exists. diff --git a/backend/report_exporter.py b/backend/report_exporter.py index b6d83de..c631ff7 100644 --- a/backend/report_exporter.py +++ b/backend/report_exporter.py @@ -229,6 +229,7 @@ def footer(self) -> None: async def export_report_pdf( document: str, query: str, + title: str = "", session_id: Optional[str] = None, ) -> Optional[Path]: """ @@ -244,7 +245,8 @@ async def export_report_pdf( try: reports_dir = _get_reports_dir() date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") - slug = _query_to_slug(query) + slug_base = title or query + slug = _query_to_slug(slug_base) sid_part = (session_id or "cli")[:8] filename = f"{date_str}_{sid_part}_{slug}.pdf" output_path = reports_dir / filename From 346095ccb139ec0e1d0cf06fddd7c361faf84cc6 Mon Sep 17 00:00:00 2001 From: jay Date: Fri, 29 May 2026 13:31:58 -0700 Subject: [PATCH 02/17] have neo4j data be stored in the repo instead of being a managed volume --- docker-compose-full.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-full.yml b/docker-compose-full.yml index 1e03186..0cf9e36 100644 --- a/docker-compose-full.yml +++ b/docker-compose-full.yml @@ -58,7 +58,7 @@ services: - NEO4J_PLUGINS=["apoc"] - NEO4J_server_memory_heap_max__size=1G volumes: - - neo4j_data:/data + - ./data/neo4j_data:/data restart: unless-stopped networks: - research-network @@ -103,7 +103,7 @@ services: networks: - research-network healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:9393/health" ] + test: ["CMD", "curl", "-f", "http://localhost:9393/health"] interval: 5s timeout: 3s retries: 10 @@ -120,7 +120,7 @@ services: networks: - research-network healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:9292/health" ] + test: ["CMD", "curl", "-f", "http://localhost:9292/health"] interval: 5s timeout: 3s retries: 10 @@ -139,7 +139,7 @@ services: networks: - research-network healthcheck: - test: [ "CMD", "curl", "-f", "http://localhost:9191/health" ] + test: ["CMD", "curl", "-f", "http://localhost:9191/health"] interval: 5s timeout: 3s retries: 10 From 9da8f1de4335b65330d5feb0249b6efbc7e64409 Mon Sep 17 00:00:00 2001 From: jay Date: Fri, 29 May 2026 13:33:59 -0700 Subject: [PATCH 03/17] update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0641d3d..9fa330d 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,5 @@ backend/logs/sessions/*.json # Generated reports and retrieved docs data/reports/* data/benchmarks/* -data/documents/* \ No newline at end of file +data/documents/* +data/neo4j_data/* \ No newline at end of file From 080c96b327ba10cf2a56b38b1b1c0189034b7817 Mon Sep 17 00:00:00 2001 From: jay Date: Fri, 29 May 2026 13:34:20 -0700 Subject: [PATCH 04/17] remove memory file --- .../memory/MEMORY.md | 708 ------------------ 1 file changed, 708 deletions(-) delete mode 100644 .github/agents/Agent-Developer-Assistant/memory/MEMORY.md diff --git a/.github/agents/Agent-Developer-Assistant/memory/MEMORY.md b/.github/agents/Agent-Developer-Assistant/memory/MEMORY.md deleted file mode 100644 index d8268f9..0000000 --- a/.github/agents/Agent-Developer-Assistant/memory/MEMORY.md +++ /dev/null @@ -1,708 +0,0 @@ -# Agent Memory — Research Assistant - -## 2026-04-23 (Legacy research_agent.py / advanced_features.py cleanup) - -- **Dead code removed:** Only 3 members of `ResearchAgent` are reachable from live code: `__init__`, `chat()`, `_select_model_for_step()`. The entire plan/execute/synthesize pipeline (~750 lines) was dead code — fully replaced by `orchestrator.py`. All removed cleanly. -- **Files reduced:** `research_agent.py` 960→143 lines; `advanced_features.py` ~300→25 lines; `optimization.py` ~180 lines of dead classes removed. -- **AdvancedResearchAgent:** Now a thin passthrough subclass with only `__init__`. All its methods (`_execute_step_parallel`, `_assess_source_credibility`, `_deep_scrape`, `_filter_relevant_links`, `_cross_reference_sources`, `_generate_follow_up_questions`) were dead — Orchestrator supersedes all of them. -- **optimization.py:** `AsyncCache`, `BatchProcessor`, `OptimizedResearchAgent` were dead utility classes — removed. `import` of `Message` from `research_agent` fixed to come from `model_backend` directly. -- **api_server.py unchanged:** The import chain is `api_server → advanced_features → research_agent` — preserved. `/chat` endpoint still calls `agent.chat()`; `/mcp/tools` still uses `agent.mcp_servers`. -- **All 122 unit tests pass** after cleanup. - -## 2026-04-22 (Session resume after hard page refresh) - -- **Root cause:** `activeSessionIdRef` and `researchConvIdRef` are in-memory `useRef`s — lost on hard refresh. `handleReconnected` was only called on WebSocket re-connections, not on the initial connection after a refresh. Backend fully supports replay but frontend never sent the `resume` message on initial connect. -- **Fix — frontend only, no backend changes needed:** - 1. `useWebSocket.ts`: Added `onInitialConnect` callback option (fires on first `ws.onopen`; `onReconnected` fires on subsequent opens). Both stored in refs for stability. - 2. `App.tsx`: Both `useRef`s now pre-populated via `sessionStorage.getItem()` as initial value — safe because WS `onopen` is async and all React `useEffect`s complete before the network handshake. - 3. `SS_SESSION_ID` / `SS_CONV_ID` sessionStorage keys written on `session_created` + `handleSendMessage`, cleared on `report`, `research_stopped`, `plan_denied`, `error`. - 4. `handleInitialConnect`: reads sessionStorage and sends `{ type: 'resume', session_id }` if stored session exists. - 5. **Replay deduplication:** `replayCountRef = useRef(0)`. Set to `event_count` from `session_resumed`. In message loop, all non-control events decrement counter and set `isReplayedEvent = true` while > 0. `addMessageToConv`, `initFromPlan`, and `addSynthesisNode` guarded with `if (!isReplayedEvent)`. Graph state transitions are NOT guarded — they're idempotent. -- **Key patterns:** `onInitialConnect` does NOT need the 150ms delay that `onReconnected` has — backend is already up on page load. `session_resumed` reconnect notification message is always added (not guarded) since it's a new message. `error` handler always clears sessionStorage since errors during a resume leave no valid session to return to. - -## 2026-04-15 (Knowledge Graph Schema Redesign — Typed Nodes, Relations, Claims, Documents) - -- **Full rewrite of `long_term_memory.py`**: Replaced single `:Entity` node type with 7 typed Neo4j labels (`:Person`, `:Organization`, `:Technology`, `:Concept`, `:Event`, `:Location`, `:Metric`). Each type has its own uniqueness constraint, vector index, and type-specific properties (e.g., Person.affiliation, Technology.version, Metric.value+unit). -- **Typed relationship labels**: Replaced catch-all `:RELATES_TO` with 18 typed labels (CAUSES, ENABLES, PREVENTS, REQUIRES, PART_OF, USES, PRODUCES, COMPETES_WITH, AFFILIATED_WITH, AUTHORED_BY, FUNDED_BY, PRECEDED_BY, OCCURRED_AT, ASSERTS, SUPPORTS, REFUTES, MENTIONS, RELATES_TO as fallback). `classify_relation()` maps free-form verb phrases to typed labels via compiled regex patterns — deterministic, no LLM call. -- **`:Claim` nodes**: Individual assertions with confidence score, status (supported/disputed/unverified/retracted), linked to entities via ASSERTS edges and optionally to Documents via SUPPORTS edges. Deduplication by vector similarity. -- **`:Document` nodes**: Subsume old `:Source` node with richer properties (url, title, content_summary, doc_type, credibility_score, domain). Unique constraint on URL. `store_source()` and `link_to_source()` preserved as deprecated wrappers. -- **Cross-type entity dedup**: `upsert_entity()` now searches ALL entity type vector indexes via UNION ALL Cypher. When a near-duplicate exists under a different type, the more specific type wins (Person > Organization > Technology > Event > Location > Metric > Concept). Type upgrade changes the Neo4j label in-place. -- **`_create_schema()` now creates 23 Neo4j operations**: 2 Memory + 14 entity types (7 constraints + 7 vector indexes) + 2 Community + 2 Claim + 3 Document (2 constraints + 1 vector index). -- **`recall_graph_context()` enhanced**: Now includes CLAIMS section (up to 5 claims from top 3 seed entities). Type-specific entity details in recall output (affiliation, tech_category, maturity, date ranges, metric values). Typed relationship labels shown in KNOWN RELATIONSHIPS. -- **Orchestrator extraction prompt expanded**: Now requests type-specific properties per entity type + explicit relationship labels from the typed vocabulary + claims array. Storage loop stores claims, documents, and link_document_to_entity. -- **3 new API endpoints**: `GET /graph/claims`, `PATCH /graph/claims/{claim_id}`, `GET /graph/documents`. `GET /graph/entities` gains `node_type` filter parameter. Prune message includes orphaned claims count. -- **Migration script**: `backend/migrations/migrate_entity_to_typed.py` — idempotent, dry-run by default. Converts :Entity→typed labels, :Source→:Document, reclassifies RELATES_TO edges via classify_relation(). -- **All 57 smoke tests + 122 unit tests pass.** -- **Key pattern for dynamic Cypher in AST smoke tests**: When Cypher constraint/index names are generated dynamically (f-strings with variables), the literal strings won't appear in AST source inspection. Smoke tests must check for the loop structure or variable names rather than literal output strings. - -## 2026-04-10 (Pipeline Data-Flow Audit — 9 Fixes for Comprehensive Research) - -- **Root cause of thin reports:** Traced full Search→Analyst→QA→Synthesis pipeline. Found 9 data-flow bottlenecks where information was lost or insufficiently passed between agents. All fixed. -- **Config defaults increased:** `max_iterations` 3→5 (SearchAgent gets more tool-call loops per step), `distill_max_chars` 2000→4000 (less aggressive compression of raw tool results before RAG storage), `step_summary_max_chars` 800→1500 (outline phase sees richer step summaries), `section_draft_top_k` 6→10 (more RAG evidence per synthesis section), `analyst_top_k` 8→10 (analyst sees more step-scoped chunks). -- **`_extract_sources()` fixed (8-session-old bug):** Was `@staticmethod` parsing only SearchAgent's `final_message` JSON — chronically empty because LLM rarely returns clean JSON. Now an instance method with two tiers: (1) parse `final_message` JSON if valid, (2) collect all unique `source_url` values from `_search_store._chunks`. REFERENCES section and source citations in reports should now populate reliably. -- **Analyst cross-step context enriched:** Cross-step RAG retrieval `top_k` increased 3→5. **New:** Prior step structured claims and tensions now passed to each analyst as `prior_claims_block` — analysts can triangulate against curated analytical output from earlier steps, not just raw text chunks. Last 20 claims + last 5 tensions. -- **Synthesis section drafting enriched (biggest impact):** Each section draft now receives: (a) RAG evidence chunks (increased to 10), (b) **all structured claims** from `_analyst_output`, (c) **all identified tensions** between sources, (d) **unresolved contradictions** from QA. System prompt updated to explicitly instruct novel insight generation from cross-source synthesis. Analyst fallback truncation 3000→6000 chars. -- **Step summary generation expanded:** Input claims 10→15, tensions 3→5, notes 300→500 chars. Prompt now requests 8 bullets (was 5) and explicitly asks for tension coverage. -- **Distillation input expanded:** Raw text fed to distillation LLM increased from 8000→12000 chars. -- **Smoke tests: 62 total** (was 57). 5 new: `test_config_increased_defaults`, `test_extract_sources_collects_from_rag_chunks`, `test_analyst_receives_prior_claims`, `test_section_drafting_receives_structured_claims`, `test_step_summary_expanded_input`. All pass. -- **`.env` updated:** `MAX_ITERATIONS=5` (was 3). -- **Key insight:** The biggest single improvement is passing structured claims/tensions to section drafting. Previously, the ReportComposer only had RAG chunks (raw evidence) or truncated analyst JSON — it had to re-derive insights from scratch. Now it has direct access to curated analytical work, which is the bridge for generating novel insights. - -## 2026-04-10 (Knowledge Graph Phase 1–5 Enhancement + Graph Pruning) - -- **Phase 1 — Core primitives**: Added `Source` node type (url, title, credibility_score uniqueness constraint on `url`). Temporal props (`last_confirmed`, `confirmation_count`) on both Entity nodes and RELATES_TO edges. `stats()` now returns 6 keys (entities, relationships, communities, contradictions, sources, hierarchies). -- **Phase 2 — New relationship types**: `IS_A` (hierarchy), `CONTRADICTS` (between entities with linked rel references), `SOURCED_FROM` (entity → Source). 13 new async KG methods: `store_hierarchy`, `store_contradiction`, `find_contradictions`, `store_source`, `link_to_source`, `get_provenance`, `recent_entities`, `recent_relationships`, `session_diff`, `find_paths`, `find_common_neighbors`, `decay_confidence`, `prune`. -- **`store_relationship()` now deduplicates**: queries for existing (source, target, relation_type) triple; if found: merges (max confidence, appended evidence, incremented confirmation_count, returns `merged: True`); new edges get `confirmation_count: 1`. -- **`find_entities()` gains `include_hierarchy` param**: `OPTIONAL MATCH (node)-[:IS_A*1..3]->(anc)` returns `ancestors` list. -- **`recall_graph_context()` enhanced**: `min_confidence`, `include_contradictions`, `include_provenance` params; optional KNOWN CONTRADICTIONS + SOURCE PROVENANCE sections. -- **`prune(min_confidence, max_age_days, dry_run=True)`**: 3-pass: stale RELATES_TO → orphan Entity nodes → dangling CONTRADICTS. Safe default `dry_run=True`. -- **`decay_confidence(half_life_days=30)`**: Batch Cypher `exp(-0.693 * days_old / half_life)`, floor at 0.01. -- **Phase 3 — Orchestrator integration**: `_graph_mutations_since_community_update` counter; dynamic entity_limit in `_recall_memories` (based on `graph.stats()` entity count); enriched extraction prompt requests `parent_type`, `source_url`, `contradicts_prior`; context injection via `recall_graph_context` before each extraction; IS_A + provenance wiring post-store; smart community gate (only runs if `mutations >= config.graph_community_min_mutations`); `decay_confidence` called post-synthesis. -- **Config**: Two new fields: `confidence_decay_half_life` (default 30), `graph_community_min_mutations` (default 3). -- **Phase 4 — API endpoints**: 9 new `/graph/*` REST endpoints in `api_server.py` (stats, entities, relationships, communities, contradictions, provenance, paths, session/{id}, prune). `POST /graph/prune` is destructive; all others are GET. -- **Phase 5 — Smoke tests**: 9 new test functions (all static). All 57 tests pass. -- **CRITICAL neo4j driver 5.x pattern**: When Cypher returns a relationship object and `.data()` is called, neo4j driver 5.x serializes it as a 4-tuple `(start_id, end_id, type_name, properties_dict)` — NOT a dict. `rec["rel"].get(...)` will raise `'tuple' object has no attribute 'get'`. **Always extract relationship properties by name in Cypher** (`rel.prop AS prop`) instead of `RETURN rel`. This applies to any query where a relationship edge is returned as a bare column. - -## 2026-04-01 (ChromaDB → Neo4j Migration) - -- **Full migration of `long_term_memory.py` from ChromaDB to Neo4j.** All ChromaDB internals replaced with Neo4j async driver (`AsyncGraphDatabase`). Public API (`AsyncLongTermMemory` class + `KnowledgeGraph`) preserved — callers unchanged except constructor args. -- **Neo4j native graph model:** Entities, relationships, and communities are now first-class Neo4j primitives (`:Entity` nodes, `:RELATES_TO` edges, `:Community` nodes + `:MEMBER_OF` edges) instead of flat ChromaDB collections with JSON metadata. Relationship traversal uses Cypher variable-length paths instead of iterative Python BFS. -- **Vector indexes replace HNSW collections:** Three Neo4j vector indexes (`memory_embedding_idx`, `entity_embedding_idx`, `community_embedding_idx`) with 384 dimensions and cosine similarity function. Created via `_create_schema()` on init. -- **`_NoOpEmbeddingFunction` deleted.** Was needed for ChromaDB's embedding function interface. Neo4j vector indexes are optional — if embeddings disabled, zero-vectors are stored but similarity search is meaningless (same degradation behavior). -- **Natively async:** Neo4j driver is fully async (`AsyncGraphDatabase.driver`) — all `asyncio.to_thread()` wrappers from the ChromaDB era are gone. `close()` method added for clean driver shutdown (called in `api_server.py` lifespan shutdown). -- **Config changes:** `chroma_persist_dir`/`chroma_collection_name` → `neo4j_uri`/`neo4j_user`/`neo4j_password`/`neo4j_database`/`neo4j_embedding_dimensions`. -- **Docker:** `neo4j:5.26-community` service added with APOC plugin, healthcheck via `cypher-shell`, named volume `neo4j_data`. -- **Migration script:** `scripts/migrate_chroma_to_neo4j.py` — one-time, idempotent (uses MERGE). Migrates flat memories, entities, relationships, and communities in order. -- **Files modified:** `long_term_memory.py` (full rewrite), `config.py`, `api_server.py`, `cli.py`, `smoke_test.py`, `requirements.txt`, `docker-compose.yml`, `docker-compose-full.yml`, `README.md`, `docs/RAG_NOTES.md`, `docs/MCP_SERVERS.md`. -- **Files created:** `scripts/migrate_chroma_to_neo4j.py`. -- **Smoke tests:** All pass. Integration tests skip gracefully when Neo4j is unavailable. - -## 2026-03-25 (Community update loop + synthesis UK spelling gap) - -- **Synthesis step guard misses UK spelling:** `_SYNTHESIS_SIGNALS` had "synthesize all" (US 'z') but not "synthesise all" (UK/AU 's'). LLM used "Synthesise all findings from steps 1–9" in the plan — bypassed the guard, ran the full Search→Analyst→QA pipeline as an extra step. Fixed: added "synthesise all" and "compile and synthesise" to `_SYNTHESIS_SIGNALS`. Lesson: always add both spellings for synthesis/compile variants. -- **`_store_community_sync` triggered `_NoOpEmbeddingFunction` on every write:** Called `_communities_col.add()` / `.update()` without `embeddings=`. ChromaDB called the NoOp EF per document → DEBUG log spam that looked like a loop; communities stored with zero-vectors (useless for cosine queries). Fix: pre-compute embedding in async `update_communities` loop, pass `vector` to `_store_community_sync`, which now uses `embeddings=[vector]` when available. -- **`update_communities` had no cap on sequential LLM calls:** With 9 research steps × N entities/relationships, could produce many clusters each costing a sequential LLM call. Ran ~2.5 min post-report. Fixed: `max_communities: int = 10` param added; loop is `clusters[:max_communities]`. -- **Post-synthesis timing pattern:** Session job stays alive until `synthesize()` generator fully consumed — community LLM calls happen after the `report` event, so "Cancelling 1 running job" at shutdown doesn't mean the report wasn't delivered. -- **Smoke tests: 47 total** (unchanged count; updated `test_synthesis_step_detection` with UK-spelling cases; `test_orchestrator_community_detection_post_synthesis` now checks `max_communities` param exists). - -## 2026-03-25 (Synthesis step leak — prevent final step from running Search→Analyst→QA) - -- **Root cause:** The Root planning agent (despite CRITICAL CONSTRAINT in system prompt) would frequently include a final "synthesis" step in the research plan (e.g. "Synthesize all findings and produce a comprehensive report"). `execute()` routes all plan steps through `_run_step()`, which runs the full Search→Analyst→QA pipeline — wasting compute and generating duplicate content that then gets processed again by `synthesize()`. -- **Fix 1 (preventive): `_root_system_prompt()`** — Updated the parallelism rules section to remove the example mentioning "a synthesis step", and added an explicit `CRITICAL CONSTRAINT` paragraph instructing the Root NOT to include synthesis/report-generation steps. Real synthesis is always handled by the ReportComposer after all research steps complete. -- **Fix 2 (safeguard): `_is_synthesis_step(step)` static method** — Added to `Orchestrator`. Returns `True` when `step.description` matches any phrase in `_SYNTHESIS_SIGNALS` (19 entries like "synthesize all", "final synthesis", "generate a final report", etc.). Precision-tuned: "synthesize findings from X" (legitimate research) does NOT match; "synthesize all findings" (report step) does. -- **Fix 3 (safeguard): `_run_step()` synthesis guard** — Added check right after the existing dedup check, before the `try:` block. If `_is_synthesis_step()` returns `True`, the step is marked `COMPLETED` with a skip message, emits `step_complete` with `skipped_synthesis: True` flag, and returns immediately. Works for both sequential and parallel step execution paths. -- **Frontend update: `App.tsx` `step_complete` case** — If `data?.skipped_synthesis` is `True`, chat content shows "Skipped step: [description] — report synthesis is handled automatically by the ReportComposer." instead of "Completed step: [name]". -- **Smoke tests: 47 total** (was 46). Added `test_synthesis_step_detection`: tests 19 synthesis descriptions are caught, 9 legitimate research descriptions are not, and asserts presence of `CRITICAL CONSTRAINT` and `skipped_synthesis` in source. All 47 pass. -- **Pattern:** Two-layer defense is the right approach for LLM instruction failures — update the system prompt (layer 1), add a code-level guard that catches instruction violations (layer 2). The code guard is the final authority; the prompt update reduces frequency. - -## 2026-03-25 (Outline retry + structured report metadata) - -- **Outline retry:** Phase A calls `self.agents.report.run()` for the outline JSON. 65/67 observed failures were empty responses (not malformed JSON). Added a single retry immediately after the first call when `content.strip()` is falsy — before the parse block. The parse/fallback logic is unchanged; the retry just gives the model a second chance before falling through to defaults. -- **Structured report event metadata:** `report` WebSocket event `data` dict now has three new top-level fields: `title` (= `query`; no LLM-derived title exists in the multi-pass assembly flow), `sources` (normalised list of `{title, url}` from `all_sources`), `key_findings` (content string of the first drafted section whose title contains "finding", empty string if absent). Extracted from already-computed data — zero extra LLM calls. -- **Pattern:** The `_DEFAULT_SYSTEM_PROMPT` full-report format is intentionally NOT used in the multi-pass flow (Phase B bypass from previous session). The report header in `doc_parts` is always `"RESEARCH REPORT"` with no LLM-derived title line, so `query` is the canonical title. -- **Frontend changes required:** `src/types/conversation.ts` `SynthesisData` gained `key_findings?: string` and `sources?: Array<{title: string; url: string}>`. `App.tsx` `report` case updated to use `data?.title ?? 'Research Report'` (was hardcoded `'Research Report'`) and wire `key_findings` + `sources` into the stored message. Rendering components (`SynthesisMessage`, `ReportViewer`, `ResearchReportDocument`) needed no changes — they already read `synthesis.title` with a fallback. -- **Benchmark artifact:** `data/benchmarks/ANALYSIS-1.md` created — formatted run #4 analysis capturing all findings plus the three fixes applied this session (section duplication, outline retry, structured metadata), with open issues for the next run. - -## 2026-03-25 (Section duplication bug — synthesize() Phase B) - -- **Root cause:** `ReportComposer._DEFAULT_SYSTEM_PROMPT` always instructs the LLM to produce a full multi-section report (RESEARCH REPORT → EXECUTIVE SUMMARY → KEY FINDINGS → NOVEL INSIGHTS → RECOMMENDATIONS → KNOWLEDGE GAPS → REFERENCES). Phase B of `synthesize()` calls `self.agents.report.run()` once per section with a user-prompt saying "Draft ONLY the '{sec_title}' section" — but the system prompt overrides this. For generic section names (especially "Analysis"), the LLM produces a full report structure, embedding all section headers in the content. The outer `doc_parts` assembly then prepends each section title again, creating duplicates. -- **Fix:** In Phase B, replaced `self.agents.report.run()` with a direct `self.agents.report.model.generate()` call using a minimal single-section system prompt. This bypasses `_DEFAULT_SYSTEM_PROMPT` without touching any class interfaces. One-line conceptual change, contained entirely within `synthesize()`. -- **Affected:** 100% of benchmark reports showed duplication. EXECUTIVE SUMMARY, KEY FINDINGS, RECOMMENDATIONS, KNOWLEDGE GAPS appeared twice; NOVEL INSIGHTS and REFERENCES appeared once (inside the "Analysis" section LLM output) + once in outer assembly. -- **Pattern:** `_DEFAULT_SYSTEM_PROMPT` is designed for single full-report calls (old synthesis style). Phase A/B/C multi-pass architecture was added later but Phase B's calls are incompatible with that full-report system prompt. Whenever `SubAgent.run()` is used for sub-task drafting, always verify the system prompt won't generate the entire parent artifact. - -## 2026-03-24 (web_scrape main-content heuristic expansion) - -- **`_extract_main_content()` rewritten as a 3-stage pipeline.** Stage 1 (semantic elements) unchanged. Stage 2 (new): positive class/id pattern match against `_CONTENT_PATTERNS` (`\b(content|article|post|entry|story|prose)\b`) with a link-density guard (<0.5) to skip tag clouds and nav menus that happen to have a `content` class. Stage 3 (improved): replaced raw `len(get_text())` with `_content_score()` composite: `(text_len + p_count * 50) * (1 - link_density)`. Paragraph count bonus generalises the old character-count heuristic; link-density penalty eliminates nav menus as false positives. -- **Two new helper functions:** `_link_density(el)` (ratio of `` text to total text, 0–1) and `_content_score(el)` (composite). Both guard against empty elements (no ZeroDivisionError). Both importable and independently tested. -- **`_CONTENT_PATTERNS` constant** added alongside `_BOILERPLATE_PATTERNS`. `\b` word-boundary anchors prevent partial-match false positives (e.g. `"account"` does not match `"content"`). -- **Stage 3 score threshold** is 300 composite units (same number as the old char threshold, but now effectively much stricter for link-dense navs and lenient for paragraph-rich articles). -- **8 new tests** added to `mcp/web_scrape/tests.py` (18 total, was 10). All 18 pass. Run as `python3 tests.py` from `mcp/web_scrape/`. - -## 2026-03-24 (Benchmark Run #3 — Shutdown race condition + cancel-of-complete fix) - -- **33/100 tasks succeeded, 40 failed, 27 never ran.** Root cause was NOT OOM this time — a graceful `Shutting down` at 22:43:52 (likely accidental `make restart-all` or SIGTERM) cancelled 21 active sessions, and subsequent connections hit a shutdown race condition. -- **Three cascading failure modes identified:** (1) 10 "Research session was cancelled" — active sessions killed by `session_manager.shutdown()`, (2) 10 `'NoneType' object has no attribute 'start_job'` — `app.state.session_manager` nulled out during shutdown Step 6 while new WS connections still arriving, (3) 4 "Server not fully initialised" + 16 hard connection failures during/after restart. -- **11 of 21 cancelled sessions already had a `report` event** in their replay log — synthesis was done, the cancel arrived during post-report cleanup status events or the `finally` block. These were false cancellations. -- **Fix 1: Shutdown race condition.** Added `app.state.shutting_down = False` flag during lifespan startup. `_shutdown()` sets it to `True` as its very first action, before cancelling any jobs. WS handler guard now checks `getattr(app_state, "shutting_down", False)` alongside the existing `agent_pool` check. New sessions during shutdown get a clean "Server not fully initialised" error immediately instead of crashing on null `session_manager`. -- **Fix 2: Cancel of already-complete sessions.** `_run_session`'s `CancelledError` handler now checks `any(e.get("type") == "report" for e in s.replay_log)`. If a report event was already emitted, the handler sets `s.state = "complete"` instead of `"cancelled"` and does NOT emit the spurious error message. The benchmark client already received the report via the drain — it just needs the session to not emit a contradicting error afterward. -- **Concurrency note:** `run-bench.sh` was at `--concurrency 10` (commit `1c391b6`). Previous runs crashed at 10 (OOM). Should be lowered for next run, though this run's failure was NOT OOM. -- **Smoke tests: 45 total** (was 43). Added `test_shutdown_guard_rejects_during_shutdown` + `test_cancel_preserves_complete_sessions`. All pass. - -## 2026-03-24 (Session concurrency limiter for benchmark scalability) - -- **Backend-side session semaphore added.** `ResearchSessionManager` now accepts `max_concurrent_sessions` (default 10 via `MAX_CONCURRENT_SESSIONS` env var). When set >0, `start_job()` wraps the coroutine in a `_gated()` wrapper that acquires the semaphore before executing. Excess jobs are registered and start immediately as asyncio tasks, but block on semaphore acquisition — their plan generation can still proceed, the gate is on the execute+synthesize pipeline. Set to 0 to disable. -- **Docker memory raised 6GB→10GB** in both `docker-compose.yml` and `docker-compose-full.yml`. With `release_memory()` + the session semaphore capping concurrent pipelines, 10GB provides comfortable headroom for 10 concurrent shallow sessions. -- **Run #3 retrospective:** The 4.5-hour run at concurrency=10 did NOT OOM — `release_memory()` fix from Run #2 is working. The failure was entirely from the external shutdown + the two race conditions now fixed. -- **Smoke tests: 46 total** (was 45). Added `test_session_manager_concurrency_semaphore`. All pass. - -## 2026-03-24 (Azure Terraform Audit + Fixes) - -- **Critical nginx bug fixed:** `src/nginx.conf` hard-coded `host.docker.internal:9999`/`9191` — unusable in Azure Container Apps. `NGINX_BACKEND_URL` env var was set in Terraform but never consumed. Fixed by converting nginx.conf to use `${NGINX_BACKEND_URL}` and `${NGINX_FILE_HANDLER_URL}` placeholders; `src/Dockerfile` now copies conf to `/etc/nginx/templates/default.conf.template` so nginx's native envsubst mechanism substitutes at startup. `NGINX_FILE_HANDLER_URL` added to frontend Container App env in Terraform. -- **Four missing persistent volumes added to backend:** `logs` (session checkpoints — the P0 data-loss vector), `config` (model_settings.json persistence), `instructions` (self-optimization RESEARCH-METHODS.md). Each gets its Azure File Share + environment-level storage resource + volume block + mount. `LOG_DIR=/app/logs` env var now explicit in Container App. -- **AWS/Bedrock variables added:** `aws_base_url` + `aws_api_key` variables in `variables.tf`, wired to backend env. BedrockBackend uses OpenAI-compatible endpoint (not boto3), so only these two variables are needed. -- **Terraform resource count increased:** was ~15 resources, now ~19 (3 new file shares + 3 new env-level storage mounts). -- **`docs/` bundled in backend image (2026-03-24 follow-up):** Build context lifted from `./backend` to `.` (repo root). `backend/Dockerfile` now uses `COPY backend/ .` + `COPY docs /app/docs`. Root `.dockerignore` added to exclude `src/`, `mcp/`, `infra/`, `.git/`, etc. Both compose files updated (`context: .`, `dockerfile: backend/Dockerfile`). Azure README build command updated to `docker build -f backend/Dockerfile -t ... .`. Local dev unaffected: the `./docs:/app/docs` bind-mount in compose shadows the baked-in copy at runtime. - -## 2026-03-24 (Benchmark Run #2 — Five fixes for complete run survivability) - -- **95/100 tasks failed (5 succeeded then server OOM-crashed).** Same root cause family as Run #1 but at `concurrency=3`. 3GB Docker limit still insufficient: completed sessions held full Orchestrator state (RAG store, claims, step summaries) in memory with no eager cleanup. After 5 completions + 3 in-progress, memory exceeded limit → container OOM-killed → 92 tasks got instant "did not receive a valid HTTP response" during restart. -- **Three compounding problems identified:** - 1. **No session memory cleanup** — completed sessions held heavy Orchestrator buffers until TTL eviction (1 hour). `release_memory()` added to Orchestrator, called in `_run_session` finally block. - 2. **Resume logic bug** — `load_existing_results()` collected ALL task IDs (success + failure). Failed tasks were never retried on `--resume`. Fixed: only successful results (with `article` field) are preserved; output file rewritten to drop failures. - 3. **No connection retry** — transient server unavailability (OOM restart) caused permanent task failure. Added exponential backoff retry (5 attempts, 15s base) for `ConnectionClosed`/`WebSocketException`/`OSError`. -- **Additional fixes:** Docker memory limit raised 3GB→6GB. Default concurrency in `run-bench.sh` lowered 3→2. Execution event logging changed from INFO + full JSON dump to DEBUG + type-only (massive IO/memory reduction). -- **Smoke tests: 43 total** (was 42). Added `test_orchestrator_release_memory` (AST check + api_server wiring). All pass. - -## 2026-03-23 (Benchmark Run #1 — OOM crash at concurrency=10) - -- **96/100 benchmark tasks failed.** Root cause: Docker container OOM-killed while ~10 sessions were in synthesis simultaneously. `concurrency=10` (default) caused all active sessions to hit the memory-intensive synthesis phase at once. No graceful shutdown logged (0 `Session removed`, 0 `Shutting down` entries) — confirms hard kill. -- **All 9 persisted sessions had `state=executing` with last event `research_complete`.** Every session completed all steps successfully but died during synthesis. The benchmark evaluator requires a `report` event — none were emitted. -- **Diagnosis clues for future OOM:** (1) 0 `Session removed` entries, (2) no shutdown/cancel log messages, (3) NDJSON shows a fresh `Logging initialized` mid-stream with only a 4s gap from last activity, (4) all session checkpoints frozen at same phase. -- **Fix:** re-run with `--concurrency 3 --resume`. Long-term fix: consider a synthesis semaphore in the backend to cap concurrent synthesis operations. - -## 2026-03-23 (Model Settings UI) - -- **New feature: frontend model configuration.** "Model Settings" sub-panel added to Sidebar settings tab → opens `ModelSettings.tsx` with Provider section + 4 per-agent sections (Root/Search/Analyst/QA). -- **BUG FIXED: `_select_model()` used module-level `config` not `self.config`.** The orchestrator had a scoping bug: `_select_model()` read the global config singleton (from `config.py`) instead of `self.config` (the per-session config). Fixed by `cfg = self.config` at top of method. Cross-session config updates would have been silently ignored prior to this fix. -- **Per-agent model overrides:** `root_model_override`, `search_model_override`, `analyst_model_override`, `qa_model_override` added to `Config`. `_select_model()` checks these before heavy/light fallback. UI shows each agent with a separate model ID field. -- **Sampling params wired through:** `root/search/analyst/qa_temperature`, `root/search/analyst/qa_top_p`, `root/search/analyst/qa_max_tokens` added to Config. Temperature + max_tokens forwarded to all `_root_backend.generate()` calls and through `SubAgent.run()` to the backend. `top_p` is stored in Config but NOT forwarded (backends don't have it in their generate() signatures). -- **Config persistence:** `POST /config` writes `config/model_settings.json` (atomic via `.tmp` rename). Lifespan loads and overlays these on startup. Pattern mirrors `user_mcp_servers.json` in same directory. -- **Hot reload:** `POST /config` rebuilds `model_backend` + `agent_pool` in-memory without process restart. New research sessions pick up new config immediately. -- **Smoke test:** `test_select_model_reads_config` updated to check `cfg.xxx` pattern (was `config.xxx`). All 42 tests pass. - -## 2026-03-23 (Docs Viewer UI Feature) - -- **New feature: in-app documentation browser.** "Docs" button in top nav bar opens a full-screen modal (`DocsViewer.tsx`) that loads all `.md` files from `docs/` via two new backend REST endpoints (`GET /project-docs`, `GET /project-docs/{filename}`). -- **GOTCHA: FastAPI reserves `/docs` for Swagger UI.** Any route named `/docs` is silently intercepted — this is not a 404, it returns the Swagger HTML page. Always use a different prefix for custom doc-serving endpoints. Route was renamed to `/project-docs`. -- **GOTCHA: `docs/` not inside the Docker build context.** `backend/Dockerfile` context is `./backend` — sibling directories like `docs/` must be bind-mounted as a volume. All three compose files needed `- ./docs:/app/docs`. The `_DOCS_DIR` path must use `os.path.dirname(__file__)` (resolves to `/app` inside the container) not `../docs`. -- **Pattern used:** mirrors the Research Methods viewer already in `Sidebar.tsx` — `react-markdown` + `remark-gfm` + Tailwind `prose` classes, same light/dark styling. No new npm deps needed. -- **Security note:** `GET /docs/{filename}` validates with `os.path.basename` + `.md` extension check to block path traversal. `filename` must not contain `/` or `\`. -- **Fragment wrapper:** `App.tsx` return was changed from a bare `
` to `<>...` fragment so `DocsViewer` can be a sibling (portal-style overlay) to the main layout div without nesting it inside. -- **New files:** `src/components/DocsViewer.tsx`. New API client methods: `listDocs()`, `getDoc(filename)`. - -## 2026-03-23 (S8 Bug-Fix Session — 3 issues from pipeline analysis) - -- **P1 FIXED: NDJSON timestamp `%f` bug.** `_NdjsonFormatter` was calling `self.formatTime()` which delegates to `time.strftime()` — that does NOT expand `%f`. Every timestamp had the literal string `%f` instead of microseconds. Fix: overrode `formatTime()` in `_NdjsonFormatter` using `datetime.fromtimestamp(record.created, tz=timezone.utc).strftime(datefmt or _ISO_FMT)`. Sub-second ordering now works. -- **P2 ALREADY FIXED (step_complete logging):** S8 analysis reported `step_complete` not logged at INFO, but commit `19e84d8` had already added `logger.info("[Step %d] complete (qa_retries=%d, contradictions=%d)", ...)` at the end of `_run_step`. The smoke test `test_orchestrator_step_level_logging` already validates this. No code change needed. -- **P2 FIXED: Confidence field inconsistency.** `AnalystAgent._DEFAULT_SYSTEM_PROMPT` had no `confidence` field in the claim schema — the model produced it non-deterministically. Added rule #6 to system prompt: "For EVERY claim, set confidence: corroborated / partially_corroborated / single_source" and added the field to the JSON schema. Downstream `record_event` call already reads `c.get("confidence", "")` — no other changes needed. -- **Smoke tests: 42 total** (was 40). Added `test_analyst_prompt_confidence_field` + extended `test_ndjson_formatter_valid_json` with `%f`-literal assertion and decimal-point check. All 42 pass. - -## 2026-03-24 (Auto PDF export on report completion) - -- **New feature: every report is auto-exported to `data/reports/` as a PDF.** Works for both API/frontend mode and CLI headless mode since the hook is in `orchestrator.synthesize()`. -- **Hook location:** `orchestrator.py` synthesize() — right after `document` is assembled and `context.save_step("final_report", ...)`, before yielding the `report` event. Pattern: `from report_exporter import export_report_pdf; await export_report_pdf(document, query, self.session_id)`. Wrapped in `try/except` so export failure never interrupts the pipeline. -- **`backend/report_exporter.py`:** new module. `_generate_pdf()` (sync, called via `asyncio.to_thread`) uses `fpdf2` to render sections. `export_report_pdf()` is the async entry point. File naming: `{YYYY-MM-DD}_{session_id[:8]}_{title_slug}.pdf`. `REPORTS_DIR` env var controls destination; fallback is `Path(__file__).parent.parent / "data" / "reports"` (resolves to `/data/reports` locally). -- **fpdf2 gotcha:** In fpdf2 2.7+, `multi_cell()` has `new_x=XPos.RIGHT` as default (changed from LMARGIN). All `multi_cell` calls must explicitly pass `new_x="LMARGIN", new_y="NEXT"` or the x coordinate will be at the right margin, causing "Not enough horizontal space" on the next call. -- **`fpdf2==2.8.7`** added to `requirements.txt`. Installed in venv. -- **Docker:** `REPORTS_DIR=/app/data/reports` env var added + `./data/reports:/app/data/reports` volume bind-mount added to both `docker-compose.yml` and `docker-compose-full.yml` under the `research-assistant` service. -- **`data/reports/` pre-existed** with old frontend-generated PDFs — the directory was already in the repo. -- All 46 smoke tests pass unchanged. - -## Working Practices - -- **Log analysis: always use temp Python scripts** instead of heredoc or inline shell approaches. Python scripts handle multiline JSON, malformed records, and complex aggregations far more reliably than shell pipelines or awk/jq. Create the script with `create_file`, run it, then `rm` it when done. - -## 2026-03-22 (Session 8 Analysis — 33678dbd, Shallow Mode, 14 Steps, Quantum Computing) - -- **Report killed again mid-synthesis — 4th occurrence in 8 sessions.** Same failure pattern: `make restart-all` during synthesis window. ~2.5 min into synthesis (outline fallback generated, Phase B started, 1 section drafted) when shutdown triggered. Session state: `cancelled`. Checkpoint was successfully persisted — persistence fix works but doesn't auto-recover. -- **Cross-session report completion: 2/8** (S2, S7 only). No deep-mode session has ever produced a report. -- **NEW BUG: NDJSON timestamp `%f` never resolves.** `_ISO_FMT = "%Y-%m-%dT%H:%M:%S.%fZ"` used in `_NdjsonFormatter.formatTime()` — but `time.strftime()` does NOT support `%f` (that's `datetime.strftime` only). All timestamps show literal `%f` instead of microseconds. Present since S7 but not previously caught. Fix: override `formatTime()` to use `datetime.fromtimestamp(record.created, tz=timezone.utc).strftime(_ISO_FMT)`. -- **Outline parse failure recurred** despite the `raw = content or ""` guard fix. `char 0` error means string was empty. S7 succeeded, S8 failed — confirms non-deterministic (model-dependent). May need to retry the LLM call once on empty response rather than falling back silently. -- **Orchestrator logs `step_start` at INFO but not `step_complete`** — asymmetric logging gap introduced in the S6→S7 fix. All 14 step_complete events only exist in `api_server` WS relay log, not orchestrator logger. -- **Analyst claim volume up 44%** (203 vs 141 in S7). Avg 14.5 claims/step. Data quality self-assessment excellent (14/14 steps include critical notes). But confidence field inconsistently populated — Steps 1,4 have per-claim labels; Steps 2-14 mostly empty strings. -- **Scrape failure rate tripled** (81% success vs 97% in S7). Quantum computing targets more paywalled/ephemeral sources. 5× HTTP 403, 3× HTTP 404. -- **`_step_sources` still empty — 8th consecutive session.** Broken `_extract_sources()` data path. -- **Step summaries 13/14 truncated at 800 chars.** Same as S7 (14/15). Config-level issue. -- **New cosmetic error: MCP anyio cancel scope mismatch on shutdown.** `streamable_http.terminate_session()` called from different task than scope was created. MCP SDK issue, not ours. - -## 2026-03-22 (Session 7 Analysis — af45bde5, Shallow Mode, 15 Steps, FIRST COMPLETE REPORT) - -- **First complete 15-step report produced.** 32,100 chars, 6 sections (Executive Summary, Key Findings, Novel Insights, Recommendations, Knowledge Gaps, References). Same query as S6 ("LLM/SLM architecture research in 2026") but shallow depth. All synthesis phases completed: outline generation, 6 section drafts (28,503 chars), cross-reference (8 contradictions flagged). -- **Shallow mode 15-step total time: ~48 min** (20 min execute + 28 min synthesis). Deep mode (S6) spent 57 min on execute alone and never produced a report. Synthesis is ~58% of total runtime — Phase B (6 Bedrock generate calls) is the bottleneck. -- **NDJSON fix validated in production:** 100% parse rate (434/434 valid JSON records). Was 28% in S6. Orchestrator events now appear: 34 records captured (was 0 in S6). Both logging fixes confirmed working. -- **`_step_sources` has NEVER worked across 7 sessions.** `_extract_sources()` parses SearchAgent's `final_message` for JSON with `sources` key — agent doesn't emit that format. All 15 steps show 0 sources in checkpoint. Synthesis falls back to `_analyst_output.claims` which contains source references. Low impact but technically broken data path. -- **Step summary truncation at 800 chars.** 14/15 summaries hit the 800-char cap exactly. May be losing synthesis context. Worth investigating if raising limit improves report quality. -- **Outline parse succeeded (first time since S2).** No fallback to default 5-section outline. Either the hardening fix or model behavior randomness. Non-deterministic. -- **New error type: tool name confusion.** Model called `arxiv.org/abs/2502.06807` as a tool name instead of passing URL to `scrape_url`. Pipeline recovered. Single occurrence. -- **Step 14 (gap-fill) produced 0 claims.** Analyst correctly identified insufficient sources rather than fabricating. Suggests gap-fill mechanism needs prompt tuning or alternative source strategy. -- **Cross-session report completion: 2/7.** S2 (shallow, simple) and S7 (shallow, complex). No deep-mode session has ever produced a report. Synthesis durability remains the constraint for deep mode. - -## 2026-03-22 (Logging + Observability Fix — 3 issues) - -- **Root cause of NDJSON corruption:** `%(message)r` in the log format string produces Python repr output (single-quoted strings) — NOT valid JSON. `json.loads` fails on ~72% of records. Only records whose messages happen to contain single quotes get double-quote outer delimiters from repr, accidentally producing valid JSON. Fixed with `_NdjsonFormatter` class using `json.dumps()`. -- **File handler was DEBUG-mode-only:** NDJSON file handler guarded by `debug_mode`, so production INFO-mode runs never wrote a file at all. Removed the guard; file handler now always enabled at INFO level. No more DEBUG noise from mcp_client in the file. -- **Outline parse hardened:** Previous code did `json.loads(outline_response.content)` — could receive None (TypeError) or empty string after code-fence stripping (JSONDecodeError). Fixed: `raw = outline_response.content or ""` + `if raw:` guard before json.loads. Code-fence stripping was already present. -- **Orchestrator events added at INFO level:** step_start, step_complete, research_complete, QA convergence/budget decisions now emit `logger.info()` at key pipeline milestones. Previously these were only yielded as ResponseMessage WebSocket events. Now they're captured by the NDJSON file handler for post-run analysis. -- **smoke_test.py now at 40 tests** (was 37): Added `test_ndjson_formatter_valid_json`, `test_outline_parse_handles_empty_content`, `test_orchestrator_step_level_logging`. - -## 2026-03-22 (Log Analysis — Session fb6f2833, Deep Mode, 15 Steps) - -- **Largest plan executed to date:** 15-step deep-mode plan on "LLM/SLM architecture design in 2026" using Bedrock (`claude-sonnet-4-6` / `claude-haiku-4-5`). All 15 steps completed. 454 tool calls (309 search, 142 scrape, 3 file). ~57 min execution wall-clock. -- **Report killed by Docker restart — 3rd time in 6 sessions.** Synthesis started at 14:55:30 (outline + Phase B drafting), Docker container restarted at 14:55:35 — **5 second window**. Appears to be a `make restart-all` command issued while synthesis was running. Session state frozen at `executing`. -- **Session persistence P0 fix validated but insufficient.** Checkpoint was successfully written to `logs/sessions/fb6f2833-...json` with full synthesis state (15 step summaries, 15 source maps, 182 contradictions). However, recovery requires a client to send `resume` — user stepped away and never reconnected. Need auto-recovery on server restart without client trigger. -- **97% scrape success rate** — best ever (138/142). Previous best was 75% (S3). Academic/tech queries target open sources. Only 4 failures: openai.com (2), lovechip.com, marktechpost.com. -- **New error type: Bedrock content filter block** at 14:08:16. One model generation rejected. Pipeline recovered — all 15 steps completed despite it. -- **Outline parse failure recurred** (3rd occurrence: S2, S5, S6). Still not fixed in `orchestrator.py`. Falls back to generic 5-section outline. The code-fence stripping fix works in `optimization.py` but was never applied to the outline generation path. -- **NDJSON logging fundamentally broken:** Only 28% of 4,767 lines parsed as valid JSON. Scraped content with embedded newlines destroys the format. Orchestrator events go only to in-memory ring buffer, not persisted. Critical for post-run analysis. -- **182 contradictions across 15 steps** (~12/step avg). Highest QA activity observed. All convergence checks passed. -- **Deep-mode runtime scales linearly with steps:** 15 steps = ~57 min vs. 10 steps = ~24.5 min. Updated estimate: ~3.8 min/step for deep mode. -- **Cross-session report completion: 1/6.** Only S2 (shallow, tutorial query) produced a complete report. S1/S3/S5/S6 killed during synthesis. S4 failed at execution (GCP bug). Synthesis durability is the existential issue. -- **Zero code-level bugs** — third consecutive session with no internal errors. - -## 2026-03-21 (Session Persistence — P0 Fix) - -- **Root cause:** `session_store.py` was purely in-memory; Docker restart during the ~1–3 min synthesis window destroyed all findings. 2 of 5 sessions lost reports this way. -- **Fix:** Three-file change — no new dependencies, no new infra. - - `session_store.py`: Added `persist()` and `load_checkpoint()` methods. Sessions directory: `logs/sessions/{session_id}.json` (respects `LOG_DIR` env var, same pattern as `observability.py`). Atomic write via `.tmp` rename. `SessionStore.__init__` now accepts optional `sessions_dir` param. - - `orchestrator.py`: Added `synthesis_checkpoint()` (serializes `_analyst_output`, `_step_summaries`, `_step_sources`, `_contradictions`, query) and `restore_synthesis_state(checkpoint)` (reconstructs those fields from dict). RAG store chunks are NOT persisted — synthesis falls back to `_analyst_output` if the store is empty. - - `api_server.py`: (1) `_run_session()` writes synthesis checkpoint after `execute()` completes, before `synthesize()` starts — the critical window. Also writes final state in `finally`. (2) `_recover_session_from_checkpoint()` new module-level async function: loads checkpoint, creates fresh orchestrator, restores synthesis state, re-runs `synthesize()` as a tracked background job. (3) `resume` handler: tries `load_checkpoint()` before returning "session not found". -- **Recovery flow on reconnect:** client sends `resume` → backend hits in-memory miss → loads disk checkpoint → if `state == "executing"` + `synthesis_checkpoint` present + no `report` event in replay_log → re-runs synthesis → client receives full replay + streaming synthesis. If state is "complete"/"error" → replay-only, no synthesis re-run. -- **3 new smoke tests added** (37 total): `test_session_store_persistence`, `test_orchestrator_synthesis_checkpoint`, `test_api_server_recovery_function`. All 37 pass. -- **Disk file lifecycle:** Written at `research_complete` (checkpoint), overwritten at session end (final state). Files NOT auto-deleted after completion — low-priority maintenance concern. - -## 2026-03-21 (Log Analysis — Session 82608f76, Deep Mode) - -- **First deep-mode run.** Query: Iran conflict continuation, deep depth. Models: `claude-sonnet-4-6` (heavy) / `claude-haiku-4-5` (light) on Bedrock. **Pipeline completed successfully (24.5 min), but report lost to Docker restart during synthesis.** -- **Zero code-level bugs** — first session with no internal errors. All fixes from sessions 1–4 hold: NoneType scrape, dict-slice, analyst_notes type, GCP backend. Scrape 401/403 errors (31) are all external — paywalled news sites. -- **Convergence check validated at scale:** 10 retries used out of 27 possible (63% reduction). Every step except step 8 triggered exactly 1 retry then broke on convergence. Step 10 used 2 retries (contradiction count increased 8→9→11). Source disagreement reclassification working — step 8 had 8 contradictions but 0 actionable. -- **Tool volume:** 219 web_search + 72 scrape_url + 1 list_files = 292 total. Scrape success rate 57% (41/72) — lower than session 3 (75%) because geopolitical queries target more paywalled sources. -- **Docker restart killed synthesis:** `research_complete` at 21:09:21, outline parse failed at 21:10:24, server restart at 21:10:37 (~76s gap). User's computer slept → Docker killed container. In-memory session store lost all findings. -- **P0 issue confirmed: session persistence.** 2 of 5 sessions lost reports to mid-synthesis interruptions. `session_store.py` must persist to disk or ChromaDB. -- **Outline parse failure recurred** (same as session 2) — LLM returns JSON wrapped in markdown code fences. Need same code-fence-stripping fix as `optimization.py`. -- **Deep mode runtime estimate update:** Actual 24.5 min vs. estimated 60–90 min. Convergence check invalidates the pre-check extrapolation. Updated estimate: 20–35 min for 10-step plans. -- **Error trend (normalized):** S1: 384 scrape errors (NoneType+403), S2: ~10, S3: 10, S4: 0, S5: 31 (all 403, 0 code bugs). Code reliability is now stable; remaining errors are purely from external site access restrictions. -- **Cross-session report completion: 1/5** — only session 2 (shallow, LLM/SLM topic) produced a complete report. Sessions 1, 3, 5 killed during synthesis. Session 4 never reached synthesis (GCP backend failure). Synthesis durability is the critical path. - -## 2026-03-21 (Self-Optimization Run #2 — Analysis) - -- **Second self-optimization run produced 14 changes** to RESEARCH-METHODS.md (+110 lines). All grounded in observed failures from sessions 1–4. No content removed. -- **High-confidence wins:** Agent Temporal Anchoring Rule (fixes Gemini refusal P0), Gap-Triggered Re-search Obligation (fixes silent gap-skipping), Coverage Notes Convention (formalizes analyst output format), Zero-Result Escalation extended to agent refusals, Minimum Completeness Threshold for memory storage. -- **Aspirational but needs code changes:** Importance Scoring Rubric (code uses fixed `importance=7`), Authoritative URL Pinning (code doesn't detect canonical URLs), Partial Session Recovery (orchestrator doesn't implement the 3-consecutive-failure halt). -- **Prompt size creep risk:** Methods doc grew from ~190→~300 lines (~6K tokens). Injected into every planning prompt unconditionally. After 3–4 more optimization runs, may start diluting query context. Consider relevance-filtered injection or a hard line cap. -- **`_develop_new_research_methods()` JSON parse failed** — LLM returned JSON wrapped in markdown code fences (`` ```json ``` ``). `json.loads()` choked. The `new_methods` list was empty; all 14 changes came from the free-form "generate updated doc" phase compensating. Fix: strip code fences before parsing. One-line fix in `optimization.py`. -- **Domain-specific methods (Governance Maturity Check, Concentration Check) are positive but narrow** — useful for protocol/ecosystem research, noise for other query types. Currently injected into all plans unconditionally. -- **Prediction for next Bedrock run:** slightly better coverage completeness, marginal increase in step count (10–12 vs 9–10), ~10–15% longer wall-clock. No effect on error rate (those are code-level fixes). -- **Prediction for next Gemini run:** 50/50 odds Gemini still refuses sensitive current-events queries — refusal appears to be model-level safety, not prompt-sensitivity. Methods doc adds a second reinforcement layer on top of the hardcoded system prompt fix. - -## 2026-03-21 (GCP Backend Fixes + Config Model Selection) - -- **`GCPVertexAIBackend.generate()` fixed** (`model_backend.py`): Now converts raw `ChatCompletionMessageFunctionToolCall` SDK objects to internal `ToolCall(name=tc.function.name, parameters=json.loads(tc.function.arguments), ...)` — same pattern as BedrockBackend. Also fixed the early-return on empty content: now checks `not content and not raw_tool_calls` so tool-only responses (content=None) are not silently dropped. -- **Error handler double-fault fixed** (`orchestrator.py` `_run_search`): Error log now uses `getattr(tool_call, "name", "")` instead of `tool_call.name`. Previously if the AttributeError was on `.name`, the error handler itself re-raised a second AttributeError. -- **`_select_model()` now reads from `config.*` fields** (`orchestrator.py`): All hardcoded model strings removed. Reads `config.{backend}_heavy_model` / `config.{backend}_light_model`. Same logic preserved (heavy-or-fallback → heavy model; explicitly-light → light model). -- **Heavy/light model config fields added** (`config.py`): Added `openai_heavy_model`/`openai_light_model`, `azure_heavy_model`/`azure_light_model`, `aws_heavy_model`/`aws_light_model`, `gcp_heavy_model`/`gcp_light_model`, `ollama_heavy_model`/`ollama_light_model`. All respect env vars: `{BACKEND}_HEAVY_MODEL` / `{BACKEND}_LIGHT_MODEL`. Defaults match previous hardcoded values. Existing `*_model` fields kept for backward compat. -- **SearchAgent system prompt updated** (`orchestrator.py`): Added a rule at the top of the Rules section: "You have real-time web search and scraping tools — use them. Never refuse based on training cutoff date." Intended to fix Gemini's refusal pattern where it classified current-events queries as "future/hypothetical". -- **`openai._base_client` and `openai.resources` suppressed to WARNING** (`observability.py`): These two loggers were generating 68% of log volume (1,075/1,583 records) at DEBUG level. Adding them to the silenced-libs list is the minimal fix. -- **5 new smoke tests added** (`smoke_test.py`): `test_gcp_backend_converts_tool_calls`, `test_error_handler_no_double_fault`, `test_config_heavy_light_model_fields`, `test_select_model_reads_config`, `test_search_agent_prompt_tool_use_instruction`. All 34 tests pass. - -## 2026-03-21 (Log Analysis — Session 88953a2b, GCP Vertex AI) - -- **First GCP Vertex AI run.** Query: same Iran conflict continuation, shallow depth. Models: `gemini-2.5-pro` (heavy) / `gemini-2.5-flash` (light) via `_select_model()`. **Total research failure — zero useful data retrieved.** -- **P0 Bug: `GCPVertexAIBackend.generate()` returns raw SDK tool call objects.** Unlike Bedrock (which converts to `ToolCall(name=tc.function.name, ...)`), the GCP backend passes `response.choices[0].message.tool_calls` through unmodified. These are `ChatCompletionMessageFunctionToolCall` objects with `.function.name` / `.function.arguments`, not `.name` / `.parameters`. Causes `AttributeError` at orchestrator line 2362 (tool call logging) and 2422 (error handler — double fault). **Steps 1 and 3 failed.** Fix: add same `ToolCall()` conversion as Bedrock. -- **P0 Behavioral: Gemini 2.5 Flash refuses current-events queries.** All 5 analyst outputs were meta-refusals ("hypothetical future scenario", "unable to research fictional events"). The model classified "Iran conflict in 2026" as speculative/future despite it being a real current event. Claude Sonnet on the identical query produced 200+ real claims with sources. Likely a combination of training cutoff interpretation + geopolitical topic sensitivity in Gemini's safety layer. -- **Error handler double-fault:** `_run_search` except block at line 2422 does `tool_call.name` in the error log message — if the original error was `.name` not existing, the error handler itself raises a second `AttributeError`. Needs `getattr(tool_call, 'name', '')`. -- **Config gap confirmed:** `config.py` has single `gcp_model` field but `_select_model()` hardcodes `gemini-2.5-pro`/`gemini-2.5-flash`. Same hardcoding pattern for all backends. Need `{backend}_heavy_model` / `{backend}_light_model` env vars per backend so users can override via config. -- **Positive: WebSocket stability confirmed** — 0 disconnects (was 3 in 171108ba). The 150ms delay fix is working. Clean shutdown (0 crash errors). All 3 MCP servers connected. Structured analyst_notes working. NoneType scrape bug still eliminated. -- **Error trend: 105 → 37 → 14 → 2.** Remaining issues are integration/model-level, not core pipeline bugs. -- **`openai._base_client` logger noise:** 1,075 of 1,583 records (68%) from SDK HTTP client debug logging. Consider setting to WARNING. - -## 2026-03-21 (P1 Fixes from Session 171108ba Analysis) - -- **WebSocket resume timing fix:** `useWebSocket.ts` `onopen` handler now delays 150ms before calling `onReconnectedRef.current?.()` on reconnects. Prevents `session=none` race condition where `resume` message was sent before the backend WS handler was ready to process it. -- **`synthesis_progress` event added:** Backend (`orchestrator.py`) now emits `type="synthesis_progress"` with `{section_index, section_title, total_sections}` data before each section draft in Phase B, replacing the old `status` message. Frontend (`App.tsx`) handles `synthesis_progress`: updates `setCurrentStatus` to show "Drafting section N of M: Title…" and re-enables `isResearching(true)` on section 1 so the spinner reappears during synthesis (between `research_complete` and `report`). The `report` event continues to call `setIsResearching(false)`. -- **Why `isResearching(true)` on synthesis_progress section 1:** After `research_complete`, spinner disappears. Synthesis can take 2–3 min. Setting `isResearching(true)` re-shows the progress indicator. The ResearchControlBar also re-appears (pause/stop are harmless during synthesis — the pipeline is done but the WS session is still open). - -## 2026-03-21 (Log Analysis — Session 171108ba, Shallow Depth) - -- **Second shallow run; first after `str(notes)[:300]` fix and web scraper NoneType fix.** Query: "Research the current ongoing conflict in Iran", shallow depth. **~16.6 min wall-clock** (longer than 6ce9428a's ~12 min due to heavier geopolitical query + 6-min cross-reference step). -- **Both P0 bugs from analytics/2 CONFIRMED FIXED:** (1) `_generate_step_summary` dict-slice → 0 step failures. (2) NoneType scrape bug → 0 occurrences (was 28). Total errors: 14 (down from 37). -- **Frontend bug discovered — `analyst_notes.slice` TypeError:** `GraphView.tsx:385` calls `.slice()` on `analyst_notes` assuming string, but backend sends dict. Caused React crash → WebSocket disconnect. **FIXED:** Added `typeof` check + `JSON.stringify` fallback in GraphView and updated StepResultViewer type. -- **WebSocket disconnect/reconnect: 3 cycles (13:45:08–13:45:58).** First two reconnects failed to establish session (`session=none`). Third succeeded after ~37s total backoff. Resume protocol has a timing issue — `resume` message may fire before backend is ready. Recommendation: add 100–200ms delay after `open` or implement `resume_ack`. -- **Server killed mid-synthesis:** Only 2/5 report sections drafted (Executive Summary + Key Findings). `"type": "report"` event never emitted. Synthesis was still actively generating when the server was manually shut down at 13:51:02. Not a code bug — operational issue. -- **HTTP 403 is now the dominant scrape failure mode** (10 occurrences, all news/institutional sites: Britannica, NYT, France24, Reuters, BMJ, EU Council, UK Parliament). Query-topic-dependent — geopolitical queries target more gated sources. -- **Error trend: 105 → 37 → 14** across 3 sessions. Core reliability bugs are resolved. Remaining errors are external (HTTP 403s) and cosmetic (shutdown crash). -- **Analyst quality for geopolitical queries is exceptional:** Step 8 (cross-reference) produced a 4-tier source reliability assessment, 30% completeness score, explicit single-source flagging, and conditional prediction models based on unresolved key claims (Khamenei death confirmation, JCPOA status). -- **Shallow depth wall-clock stabilizing at 12–17 min** depending on query complexity. Geopolitical queries run longer due to more search/scrape activity per step. - -## 2026-03-20 (Log Analysis — Session 6ce9428a, Shallow Depth) - -- **First full run after QA loop rework + GraphRAG + research depth.** Query: "LLM/SLM architecture research 2025–2026", shallow depth (QA loop skipped). **~12 min wall-clock vs ~98 min for previous moderate-depth run — 88% reduction.** -- **`_generate_step_summary` dict-slice bug:** `orchestrator.py:3073` does `notes[:300]` but `notes` is now a dict (structured analyst notes). Raises `KeyError: slice(None, 300, None)`. Caused step 1 to fail. Fix: `str(notes)[:300]`. -- **NoneType scrape bug is proportionally WORSE:** 28 errors / 45 calls = 62% failure rate (vs. previous 81/166 = 49%). Absolute count is down because of fewer scrape calls in shallow mode, but the underlying bug is hit more frequently. -- **GraphRAG extraction working:** 146 entities, 151 relationships across 9 completed steps. Entity counts correlate with claim complexity. -- **LTM claim persistence working:** 77 claims + 5 RAG chunks persisted to ChromaDB. -- **`paper_server` MCP not running:** Configured but connection fails. Analyst flagged "no primary research papers — only survey abstracts" as critical gap. Paper server absence is a likely contributor. -- **Outline parse failure during synthesis:** JSON parse returned empty, fell back to defaults. Single occurrence — may be transient LLM formatting issue. -- **Shallow depth produced 138 claims, 37 tensions across 10 steps.** Comparable analytical quality to the previous moderate run that spent 3x compute on non-converging QA retries. -- **Runtime estimate update:** shallow depth = ~12–15 min for 10-step plans. Previous estimate of 15–25 min was conservative. - -## 2026-03-20 (Benchmark Runtime Estimates — v1, from log analysis) - -**Context:** Derived from `backend/logs/backend.ndjson` (2026-03-20 session). Single query "MCP adoption 2026", model=`claude-sonnet-4-6` on AWS Bedrock, moderate depth. Query started 19:49:59, server killed at 20:47:26 (~57 min, no `report` event seen — still running). Prior session `fed60e8d` at moderate depth ran ~98 min total. Both data points suggest moderate depth tasks run 55–100 min. - -**Per-task wall-clock estimates (baseline v1):** - -| Depth | QA retries | Estimate | Basis | -|---|---|---|---| -| `shallow` | 0 (QA skipped) | 15–25 min | Extrapolated: ~38% of moderate (no QA loop, no LoopAgent calls) | -| `moderate` | ≤2 | 40–70 min | Log evidence: 57 min still running, prior ~98 min (pre-convergence-fix) | -| `deep` | ≤3 + enhanced prompts | 60–90 min | Extrapolated from moderate + ~30% overhead | - -**Full bench (100 tasks) projections at `--concurrency 3`:** - -| Depth | Estimate | -|---|---| -| `shallow` | ~8–10 hours | -| `moderate` | ~20–25 hours | -| `deep` | ~35–45 hours | - -**Dominant bottleneck:** AWS Bedrock Sonnet latency (P90=48.3s, max=125.9s from prior session). Not Python/async overhead. - -**Key caveat:** No completed `shallow` run observed yet. The 15–25 min estimate is extrapolated from moderate — update once a shallow run with report event is logged. - -**Script default set to:** `--depth shallow --concurrency 3` (as of 2026-03-20). - ---- - -## 2026-03-20 (Log Analysis — Pipeline Bottleneck Discovery) - -- **QA loop never converges for research topics with inherent nuance.** Session `fed60e8d` (10-step "MCP adoption 2026"): 8/9 active steps exhausted 2/2 retry budget, contradiction counts *increased* across retries (e.g. step 2: 5→6→8). The LoopAgent correctly identifies real source disagreements, but the loop mechanism converts this into wasted compute. Estimated ~36 extra Sonnet + ~54 extra Haiku calls (~3x per step) with zero measurable quality improvement. -- **`scrape_url` NoneType bug**: 303 errors from `'NoneType' object has no attribute 'get'` in `mcp/web_scrape/main.py`. BS4 processing pipeline has an unguarded `.get()` on a None return from `find()`. The generic `except Exception` catches it but strips the URL — can't identify which pages trigger it. Need URL in error message + defensive null-checks. -- **Scrape error distribution**: 303 NoneType, 41 Request Error, 19x 403, 10x 429, 8x 404, 3 connection failures = 384 total failures across session. -- **Sonnet latency is severe**: P90=48.3s, max=125.9s (>2 min). 21 of 87 Sonnet calls exceeded 30s. Total LLM wait time ~88 min out of ~98 min wall clock. -- **Analyst self-awareness is high quality**: AnalystAgent correctly identified source quality gradient, corroboration sparsity (only 7/32 claims verified by 2+ sources), temporal contamination (2025 HN source cited as 2026 evidence), and the "enthusiasm/adoption gap" as the most analytically significant pattern. These notes are embedded in the QA audit payloads (logged as DEBUG messages in the LLM request bodies). -- **`_analyst_recommendations` feedback loop may not be working**: Analyst explicitly recommended "targeted searches for OpenAI tool-calling vs MCP technical comparisons" in `coverage_notes`, but subsequent search queries didn't incorporate these. Need to audit how `extra_context` is forwarded to `_run_search()`. -- **RAG growth**: 34,079 chunks in one session (381 add ops, avg 89/step, max 217/step). Needs pruning/limits. -- **Convergence check needed**: If contradiction count doesn't decrease between QA retries, should break immediately (like the existing data-poverty check for chunk count). - -## 2026-03-20 (QA Loop Rework — Convergence + Type-Based Routing) - -- **`Contradiction` dataclass now has `contradiction_type`** field: `factual_error`, `temporal_mismatch`, `source_disagreement`, `insufficient_evidence`, `unknown`. Added to `to_dict()`. -- **LoopAgent classifies every contradiction by type.** Only `factual_error` / `temporal_mismatch` / `unknown` are actionable (trigger targeted re-search). -- **`source_disagreement` contradictions are injected as tensions** directly into `analyst_result` (topic/source_a/position_a/source_b/position_b schema). No re-search triggered. Propagates to ReportComposer as synthesis input — turns a bug into a feature. -- **`insufficient_evidence` contradictions become analyst recommendations** forwarded to subsequent steps' SearchAgent (`_analyst_recommendations`). -- **Convergence check added:** `prev_actionable_count` (None on first pass, skips check on first retry). If `len(actionable) >= prev_actionable_count` fires, breaks immediately with "not converging" message. Fixes the 5→6→8 explosion pattern from session `fed60e8d`. -- **Loop exit order:** clean break → type split + inject source_disagreements → no-actionable break → convergence check → budget check → increment → re-search → re-analyst → merge → continue. -- **Default `research_depth` changed from `"moderate"` to `"shallow"`** in 4 places: `Orchestrator.__init__`, `_make_orchestrator()` in `api_server.py`, WS handler fallback, `App.tsx` useState. -- All 29 smoke tests pass. - -## 2026-03-20 (Research Depth Setting — UI + Backend) - -- Added `research_depth` parameter to `Orchestrator.__init__` (replaces the old `max_qa_retries` parameter). `max_qa_retries` is now derived from depth: shallow=0, moderate=2, deep=3. -- **shallow**: Phase 3 (LoopAgent QA loop) is skipped entirely; pipeline is Search → Analyst only. `qa_retry_count` initialized to 0 before the if/else so `record_event` always has a valid value. -- **moderate**: standard behaviour (Search → Analyst → QA, up to 2 retries). -- **deep**: injected depth-hint prompts into `_run_search` (specific queries, primary sources, multiple independent sources per claim) and `_run_analyst` (require 2+ sources per claim, surface ALL tensions). QA up to 3 retries. -- `api_server.py`: `_make_orchestrator()` now accepts `research_depth` kwarg; `query` handler extracts `raw.get("research_depth", "moderate")` and validates against the allowed set before passing to orchestrator. -- Frontend: `ResearchDepth` type exported from `Sidebar.tsx`. Settings tab replaced `