From 1b1001ba282843260ac606873e1477768c7a2500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Sat, 30 May 2026 21:02:59 -0400 Subject: [PATCH 1/2] fix(insights-agent): synthesizer dropped specialist findings An end-to-end run surfaced this: the cost_analyst worker produced the full answer ("AWS spend was 8662.07 USD ...") but the final answer was just a trailing caveat ("Your final bill may differ."). The synthesizer was fed the workers' contributions as prior *AIMessages*, so the model treated the answer as already given and only appended a short follow-up, dropping the numbers. Fix: _synthesis_input collects the user question plus the specialists' findings into a single human turn the model answers fresh, instead of replaying worker AIMessages. Adds offline regression tests (unit on _synthesis_input, plus a graph-level assertion that the synthesizer's input is a human turn carrying the finding, not a replayed assistant turn). --- .../src/insights_agent/graph/supervisor.py | 34 ++++++++++- insights-agent/tests/test_supervisor.py | 58 ++++++++++++++++++- 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/insights-agent/src/insights_agent/graph/supervisor.py b/insights-agent/src/insights_agent/graph/supervisor.py index 33a248b..474b650 100644 --- a/insights-agent/src/insights_agent/graph/supervisor.py +++ b/insights-agent/src/insights_agent/graph/supervisor.py @@ -125,7 +125,13 @@ def decide(state: SupervisorState) -> str: return state["route"] if state["route"] in WORKER_NAMES else "synthesize" async def synthesize(state: SupervisorState) -> dict[str, Any]: - resp = await llm.ainvoke([SystemMessage(_SYNTHESIZE_PROMPT), *state["messages"]]) + # Present the specialists' findings as *material to synthesize from* in a + # human turn — not as prior assistant turns. If we replayed the worker + # AIMessages directly, the model treats the answer as already given and + # only appends a tiny follow-up, dropping the actual numbers. + resp = await llm.ainvoke( + [SystemMessage(_SYNTHESIZE_PROMPT), _synthesis_input(state["messages"])] + ) return {"messages": [resp]} graph = StateGraph(SupervisorState) @@ -146,6 +152,32 @@ async def synthesize(state: SupervisorState) -> dict[str, Any]: return graph.compile() +def _synthesis_input(messages: Sequence[BaseMessage]) -> HumanMessage: + """Build the synthesizer's input: the user question plus the specialists' + findings, as a single human turn the model answers fresh.""" + question = "" + for m in messages: + if isinstance(m, HumanMessage): + question = _stringify_content(m.content) + break + + findings: list[str] = [] + for m in messages: + if isinstance(m, AIMessage) and getattr(m, "name", None) in WORKER_NAMES: + text = _stringify_content(m.content).strip() + if text and text != "(no findings)": + findings.append(f"[{m.name}] {text}") + findings_block = "\n\n".join(findings) if findings else "(no specialist findings)" + + return HumanMessage( + content=( + f"User question:\n{question}\n\n" + f"Specialist findings:\n{findings_block}\n\n" + "Write the final answer to the user now." + ) + ) + + async def ask_supervisor(graph: Any, question: str) -> AgentResult: """Run one question through the supervisor graph and return a compact result.""" state: dict[str, Any] = await graph.ainvoke( diff --git a/insights-agent/tests/test_supervisor.py b/insights-agent/tests/test_supervisor.py index 53448af..c06c120 100644 --- a/insights-agent/tests/test_supervisor.py +++ b/insights-agent/tests/test_supervisor.py @@ -14,7 +14,7 @@ from langchain_core.callbacks import CallbackManagerForLLMRun from langchain_core.embeddings import DeterministicFakeEmbedding from langchain_core.language_models import BaseChatModel -from langchain_core.messages import AIMessage, BaseMessage +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage from langchain_core.outputs import ChatGeneration, ChatResult from langchain_core.tools import BaseTool from langchain_core.vectorstores import InMemoryVectorStore @@ -26,6 +26,7 @@ FINISH, RunLimits, _run_react, + _synthesis_input, _to_text, ask_supervisor, build_supervisor_graph, @@ -223,6 +224,61 @@ async def test_tool_call_budget_forces_synthesis( await client.aclose() +class TestSynthesisInput: + """Regression for the synthesizer dropping the specialists' findings. + + Worker contributions must reach the synthesizer as *material in a human + turn*, not as prior assistant turns — otherwise the model treats the answer + as already given and emits only a tiny follow-up, losing the numbers. + """ + + def test_includes_question_and_findings_as_human_turn(self) -> None: + messages = [ + HumanMessage(content="How much did I spend on AWS?"), + AIMessage(content="AWS spend was $8662.07 (snapshots).", name=COST_ANALYST), + AIMessage(content="(no findings)", name="concept_expert"), + ] + out = _synthesis_input(messages) + assert isinstance(out, HumanMessage) + text = out.content + assert "How much did I spend on AWS?" in text + assert "$8662.07" in text + assert "[cost_analyst]" in text + # Empty/no-findings contributions are excluded. + assert "(no findings)" not in text + assert "Write the final answer" in text + + def test_no_findings_placeholder(self) -> None: + out = _synthesis_input([HumanMessage(content="hi")]) + assert "(no specialist findings)" in out.content + + +async def test_synthesizer_receives_findings_not_replayed_ai_turns( + client: CloudOracleClient, httpx_mock: HTTPXMock +) -> None: + # End-to-end at the graph level: the synthesizer's input must be a human + # turn carrying the worker's finding, not the worker AIMessage replayed. + httpx_mock.add_response(json=SUMMARY_PAYLOAD) + model = ScriptedChatModel( + script=[ + _route(COST_ANALYST), + _call("cloudoracle_cost_summary", {"start": "2026-05-01", "end": "2026-05-31"}), + _say("AWS spend was $150 in the period."), + _route(FINISH), + _say("Final: you spent $150 on AWS."), + ] + ) + graph = build_supervisor_graph(model, build_tools(client)) + await ask_supervisor(graph, "AWS spend?") + + # last_messages = the synthesize call's input: [System, Human(findings)]. + assert model.last_messages is not None + last = model.last_messages[-1] + assert isinstance(last, HumanMessage) + assert "$150" in last.content + await client.aclose() + + class TestRunReact: async def test_unknown_tool_becomes_observation(self) -> None: model = ScriptedChatModel( From 80091d95e5dd415b381406d0a897e119125cf843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesus=20Nu=C3=B1ez?= Date: Sun, 31 May 2026 16:52:12 -0400 Subject: [PATCH 2/2] docs(readme): add top-level architecture diagram; v3 is done, not current focus --- README.md | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 57dc8ef..862a746 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,31 @@ ![Tests](https://img.shields.io/badge/tests-469%20unit%20%2B%2021%20integration-brightgreen)![Go Version](https://img.shields.io/badge/go-1.25-blue) ![License](https://img.shields.io/badge/license-Apache%20License%202.0-green) -A Go FinOps toolkit that ships in two modes from the same `oracle` binary, with a polyglot agent extension in progress: +**One FinOps toolkit, three surfaces over the same cost data** — audit what you spend, predict what a PR will cost, and ask about both in plain language. + +```mermaid +flowchart LR + SRC["Cloud accounts · AWS · GCP · Azure
+ Terraform plans"] + + subgraph SYS["CloudOracle — one FinOps toolkit"] + direction TB + V1["v1 — Audit
ingest live spend, run rules
→ executive PDF + dashboard"] + V2["v2 — PR check
price a Terraform plan pre-merge
→ GitHub PR cost comment"] + V3["v3 — Insights Agent
ask FinOps questions in plain language
→ natural-language answers"] + end + + SRC --> V1 + SRC --> V2 + V1 -. cost data .-> V3 +``` + +A Go FinOps toolkit spanning three modes — two from the same `oracle` binary, plus a polyglot Python agent extension: - **v1 — Audit existing cloud spend.** Ingest live EC2/RDS/EBS/Lambda inventory from AWS, GCP, or Azure into Postgres, run deterministic rules over it, and produce an executive PDF + dashboard with an LLM-narrated summary. See **[docs/v1-guide.md](docs/v1-guide.md)**. - **v2 — Predict cost impact of a Terraform PR before merge.** Read `terraform show -json plan.tfplan`, look every changing resource up against the AWS Pricing API, and post (or upsert) a Markdown comment on the PR with the net monthly delta, top movers, and a 1–3 sentence LLM narrative. Ships as a GitHub Action and as the `oracle pr-check` subcommand. See **[docs/v2-guide.md](docs/v2-guide.md)**. -- **v3 — Insights Agent.** Polyglot Go + Python extension adding agentic FinOps analysis on top of v1/v2 cost data — a hand-rolled LangGraph supervisor over specialist agents, RAG over a FinOps corpus (pgvector), production guardrails, real billing via AWS Cost Explorer, and a CLI + HTTP surface. **Current focus.** See **[v3 — Insights Agent](#v3--insights-agent-current-focus)** below, **[docs/v3-guide.md](docs/v3-guide.md)**, and **[insights-agent/README.md](insights-agent/README.md)**. +- **v3 — Insights Agent.** Polyglot Go + Python extension adding agentic FinOps analysis on top of v1/v2 cost data — a hand-rolled LangGraph supervisor over specialist agents, RAG over a FinOps corpus (pgvector), production guardrails, real billing via AWS Cost Explorer, and a CLI + HTTP surface. See **[v3 — Insights Agent](#v3--insights-agent)** below, **[docs/v3-guide.md](docs/v3-guide.md)**, and **[insights-agent/README.md](insights-agent/README.md)**. -## v3 — Insights Agent (current focus) +## v3 — Insights Agent A Python sibling of the Go server that lets you ask FinOps questions in natural language. The agent decides which `/api/v1` endpoint to call, fetches