feat(gooddata-eval): capture reasoning_steps through the agentic-CLI path

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -80,8 +80,12 @@ def _dispatch_agentic(
     langfuse: Any,
     run_ts: str,
     model_version_override: str | None,
-) -> None:
-    """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
+) -> list[str] | None:
+    """Call the appropriate evaluate_agentic_* function for the item's test_kind.
+
+    Returns whatever that function returns -- only alert_skill/metric_skill/conversation
+    currently return their reasoning_steps; the rest still return None (unchanged).
+    """
     kind = item.test_kind
     eo = item.expected_output
     lf_kw: _LfKw = {
@@ -93,7 +97,7 @@ def _dispatch_agentic(
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
-        evaluate_agentic_visualization(
+        return evaluate_agentic_visualization(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -103,7 +107,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_metric_skill":
-        evaluate_agentic_metric_skill(
+        return evaluate_agentic_metric_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -113,7 +117,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_alert_skill":
-        evaluate_agentic_alert_skill(
+        return evaluate_agentic_alert_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -126,7 +130,7 @@ def _dispatch_agentic(
         eo_dict = eo if isinstance(eo, dict) else {}
         tool_call = eo_dict.get("tool_call", {})
         expected_args = tool_call.get("function_arguments", eo_dict)
-        evaluate_agentic_search_tool(
+        return evaluate_agentic_search_tool(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -136,7 +140,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_general_question":
-        evaluate_agentic_general_question(
+        return evaluate_agentic_general_question(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -146,7 +150,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_guardrail":
-        evaluate_agentic_guardrail(
+        return evaluate_agentic_guardrail(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -157,7 +161,7 @@ def _dispatch_agentic(
         )
     elif kind == "agentic_conversation":
         fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {}
-        evaluate_agentic_conversation(
+        return evaluate_agentic_conversation(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -202,12 +206,14 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version)
+            reasoning_steps = _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version)
             item_report.pass_at_k = True
             item_report.runs = k
+            item_report.reasoning_steps = reasoning_steps or []
         except AssertionError as exc:
             item_report.pass_at_k = False
             item_report.runs = k
+            item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -6,7 +6,7 @@
 import json
 import os
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 from gooddata_sdk import GoodDataSdk
@@ -232,6 +232,7 @@ class AlertRunResult:
     alert_id: str | None
     eval: AlertEvaluation
     actual_alert_arguments: dict
+    reasoning_steps: list[str] = field(default_factory=list)
 
 
 @dataclass
@@ -355,13 +356,15 @@ def _run_once(conv_id: str) -> AlertRunResult:
             alert_id: str | None = None
             actual_args: dict = {}
             tool_called = False
+            reasoning_steps: list[str] = []
             # conversation_history stores prior turns for GPT-4o context.
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
             current_question = question
 
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
+                reasoning_steps.extend(chat_result.reasoning_steps or [])
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
@@ -395,6 +398,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 alert_id=alert_id,
                 eval=ev,
                 actual_alert_arguments=actual_args,
+                reasoning_steps=reasoning_steps,
             )
         finally:
             if alert_id_to_delete:
@@ -462,8 +466,13 @@ def evaluate_agentic_alert_skill(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
-) -> None:
-    """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
+) -> list[str]:
+    """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
+
+    Returns the best run's reasoning_steps on success; on failure the same list is attached
+    to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception
+    idiom in `ChatClient.ask()`) so callers can retrieve it either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -534,11 +543,14 @@ def evaluate_agentic_alert_skill(
     if not summary.pass_at_k:
         best = summary.best
         ev = best.eval
-        raise AlertSkillAssertionError(
+        exc = AlertSkillAssertionError(
             f"Alert skill assertion failed. strict_pass={ev.strict_pass}. "
             f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, "
             f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, "
             f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, "
             f"recipients_correct={ev.recipients_correct}. "
             f"Actual args: {best.actual_alert_arguments}"
         )
+        exc.reasoning_steps = best.reasoning_steps
+        raise exc
+    return summary.best.reasoning_steps

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -5,7 +5,7 @@
 
 import json
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Literal
 
 from gooddata_sdk import GoodDataSdk
@@ -269,6 +269,7 @@ class ConversationResult:
     full_skill_coverage: bool
     conversation_success: bool
     total_clarification_turns: int
+    reasoning_steps: list[str] = field(default_factory=list)
 
 
 def run_agentic_conversation(
@@ -296,6 +297,7 @@ def run_agentic_conversation(
     # not persist in the (shared) workspace and get reused by a later test. Deferred to
     # the end — a later turn may $ref a metric an earlier turn created.
     created_metric_ids: list[str] = []
+    reasoning_steps: list[str] = []
 
     try:
         if initial_conversation_id is not None:
@@ -318,6 +320,7 @@ def run_agentic_conversation(
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
                 all_tool_calls.extend(chat_result.tool_call_events or [])
+                reasoning_steps.extend(chat_result.reasoning_steps or [])
 
                 if _check_output_present(resolved_turn, chat_result):
                     break
@@ -381,6 +384,7 @@ def run_agentic_conversation(
         full_skill_coverage=full_skill_coverage,
         conversation_success=conversation_success,
         total_clarification_turns=total_clarification_turns,
+        reasoning_steps=reasoning_steps,
     )
 
 
@@ -403,8 +407,14 @@ def evaluate_agentic_conversation(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
-) -> None:
-    """Run conversation evaluation, log to Langfuse, and raise on failure."""
+) -> list[str]:
+    """Run conversation evaluation, log to Langfuse, and raise on failure.
+
+    Returns the conversation's reasoning_steps on success; on failure the same list is
+    attached to the raised exception as ``.reasoning_steps`` (mirrors the
+    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve it
+    either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -478,8 +488,11 @@ def evaluate_agentic_conversation(
 
     if not result.conversation_success:
         failed_turns = [tr for tr in result.turn_results if not tr.skill_success]
-        raise ConversationAssertionError(
+        exc = ConversationAssertionError(
             f"Conversation assertion failed. "
             f"full_skill_coverage={result.full_skill_coverage}. "
             f"Failed turns: {[t.turn_id for t in failed_turns]}"
         )
+        exc.reasoning_steps = result.reasoning_steps
+        raise exc
+    return result.reasoning_steps

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -5,7 +5,7 @@
 
 import os
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 from gooddata_sdk import GoodDataSdk
@@ -108,6 +108,7 @@ class MetricRunResult:
     actual_maql: str
     maql_correct: bool
     total_turns: float
+    reasoning_steps: list[str] = field(default_factory=list)
 
 
 @dataclass
@@ -191,11 +192,13 @@ def _execute_single_metric_run(
     metric_id_to_delete: str | None = None
     turns = 0
     current_question = question
+    reasoning_steps: list[str] = []
 
     try:
         for _iteration in range(max_iterations):
             turns += 1
             chat_result = client.send_message(conversation_id, current_question)
+            reasoning_steps.extend(chat_result.reasoning_steps or [])
             candidate = _extract_metric_result(chat_result.tool_call_events or [])
             if candidate is not None:
                 metric_result = candidate
@@ -217,6 +220,7 @@ def _execute_single_metric_run(
             actual_maql=actual_maql,
             maql_correct=maql_correct,
             total_turns=float(turns),
+            reasoning_steps=reasoning_steps,
         )
     finally:
         if metric_id_to_delete:
@@ -300,8 +304,13 @@ def evaluate_agentic_metric_skill(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
-) -> None:
-    """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
+) -> list[str]:
+    """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure.
+
+    Returns the best run's reasoning_steps on success; on failure the same list is attached
+    to the raised exception as ``.reasoning_steps`` (mirrors the `conversation_id`-on-exception
+    idiom in `ChatClient.ask()`) so callers can retrieve it either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -363,9 +372,12 @@ def evaluate_agentic_metric_skill(
         best = summary.best
         expected_outputs_list: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output]
         candidates_str = "; ".join(repr(c.get("maql", "")) for c in expected_outputs_list)
-        raise MetricSkillAssertionError(
+        exc = MetricSkillAssertionError(
             f"Metric skill assertion failed. "
             f"metric_created={best.metric_created}, maql_correct={best.maql_correct}. "
             f"Expected MAQL (candidates): {candidates_str}. "
             f"Actual MAQL: {best.actual_maql}."
         )
+        exc.reasoning_steps = best.reasoning_steps
+        raise exc
+    return summary.best.reasoning_steps