From 92001ee4844955c45edabdbbc8e62433474cb3a5 Mon Sep 17 00:00:00 2001 From: Robin Date: Mon, 31 Aug 2026 17:13:21 -0700 Subject: [PATCH] trace should capture framework name --- README.md | 7 +- TRACING.md | 40 ++++++++- agentx/integrations/_traced_call.py | 3 + agentx/monitor/client.py | 22 +++++ agentx/tracing/framework_detect.py | 64 ++++++++++++++ agentx/tracing/tracer.py | 48 +++++++++-- tests/test_framework_detect.py | 125 ++++++++++++++++++++++++++++ 7 files changed, 298 insertions(+), 11 deletions(-) create mode 100644 agentx/tracing/framework_detect.py create mode 100644 tests/test_framework_detect.py diff --git a/README.md b/README.md index 3242bce..3abe811 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,12 @@ extra: | LlamaIndex | `pip install "agentx-python[llamaindex]"` | `AgentXLlamaIndexHandler` | | AutoGen | `pip install "agentx-python[autogen]"` | `AgentXAutoGenObserver` | -Or plain Python - wrap any function with `@tracer.trace(...)` and it just works, no framework required. +Or plain Python - wrap any function with `@tracer.trace(...)` and it just works, no framework +required. Tracing is **platform agnostic**: each integration stamps its platform label +automatically, a plain trace auto-detects the one orchestration framework imported in the +process, and `framework="any-name"` labels platforms AgentX has never heard of - the label +drives the dashboard's framework filter and Monitor's Platforms chart. See +[Platform detection](TRACING.md#platform-detection). Running specialist agents in parallel with a `ThreadPoolExecutor`? Wrap each worker body in `tracer.use_span(span)` so their steps land on the parent trace instead of becoming independent traces - see [TRACING.md](TRACING.md) for the full pattern. diff --git a/TRACING.md b/TRACING.md index acc04a6..1ab986d 100644 --- a/TRACING.md +++ b/TRACING.md @@ -97,7 +97,7 @@ with tracer.trace("agent-name", framework="langchain") as span: | Parameter | Type | Required | Description | |---|---|---|---| | `name` | `str` | ✓ | Agent or operation label shown in the UI | -| `framework` | `str` | - | Framework identifier: `"langchain"`, `"crewai"`, `"openai-agents"`, `"anthropic"`, or custom | +| `framework` | `str` | - | Platform label - any string, including custom platform names. Auto-filled when omitted: see [Platform detection](#platform-detection) | | `model` | `str` | - | LLM model used, e.g. `"gpt-4o"`, `"claude-sonnet-4-6"` | | `session_id` | `str` | - | Groups traces from the same user session or thread | | `metadata` | `dict` | - | Arbitrary key-value metadata (not indexed, max 16 KB) | @@ -113,6 +113,44 @@ with tracer.trace("agent-name", framework="langchain") as span: --- +## Platform detection + +Tracing is **platform agnostic**: every trace carries a platform label, and any agent runtime +works. The label resolves in priority order: + +1. **Explicit** - `framework="..."` on `trace()`. Any string is valid, including platforms + AgentX has no integration for: `framework="my-inhouse-runner"` charts and filters like any + built-in name. (The engine folds labels to lowercase, so `"LangChain"` and `"langchain"` + are one platform.) +2. **Integration** - every AgentX integration stamps its literal automatically, no parameter + needed: + + | Integration | Label | + |---|---| + | `AgentXCallbackHandler` (LangChain/LangGraph) | `langchain` | + | `AgentXCrewObserver` | `crewai` | + | `AgentXTracingProcessor` (OpenAI Agents SDK) | `openai-agents` | + | `patch_openai_client` | `openai` | + | `patch_anthropic_client` | `anthropic` | + | `patch_genai_client` | `google-genai` | + | `AgentXADKPlugin` | `google-adk` | + | `AgentXLiteLLMLogger` | `litellm` | + | `AgentXLlamaIndexHandler` | `llamaindex` | + | `AgentXAutoGenObserver` | `autogen` | + | `MoveworksImporter` | `moveworks` | + | `DatabricksTraceImporter` | `databricks` | + +3. **Auto-detection** - a plain `@tracer.trace(...)` with neither of the above looks at which + known orchestration framework is actually imported in the process (LangChain/LangGraph, + CrewAI, LlamaIndex, AutoGen, OpenAI Agents SDK, Google ADK, Semantic Kernel, Haystack, + Pydantic AI, smolagents, DSPy) and labels the span when exactly one is loaded. Ambiguous or + unknown means no label - the trace still ingests fine and buckets as "Other / custom" in the + dashboard, never mislabeled. + +The label powers the Live Traces framework filter and Monitor's **Platforms** chart +(`GET /agent-monitoring/metrics` - `byFramework` buckets, `frameworks` totals, and a +`framework=` filter). + ## Framework examples ### LangChain diff --git a/agentx/integrations/_traced_call.py b/agentx/integrations/_traced_call.py index c06a4e8..8567f74 100644 --- a/agentx/integrations/_traced_call.py +++ b/agentx/integrations/_traced_call.py @@ -143,6 +143,9 @@ def finish_llm_call( input=input_repr, output=output, model=model, + # Stamp the provider literal on a span opened without one (adoption keeps an + # explicit framework= or a framework integration's label winning over this). + framework=framework, input_tokens=input_tokens, output_tokens=output_tokens, cache_read_tokens=cache_read_tokens, diff --git a/agentx/monitor/client.py b/agentx/monitor/client.py index d8b61fb..0b2ed57 100644 --- a/agentx/monitor/client.py +++ b/agentx/monitor/client.py @@ -268,6 +268,28 @@ def kpis(self, window: str = "7d") -> dict: plus deltas vs the prior window and the run-outcome breakdown.""" return self._request("GET", "/kpis", params={"window": window}) + def metrics( + self, + window: str = "1d", + *, + agent: Optional[str] = None, + model: Optional[str] = None, + tool: Optional[str] = None, + framework: Optional[str] = None, + status: Optional[str] = None, + ) -> dict: + """The Monitor metrics grid's data over a window ("1h".."90d") - bucketed spans by kind, + latency percentiles, tokens/cost, tool executions and failures, and platform attribution + (`frameworks` window totals + per-bucket `byFramework` - the Platforms chart). Optional + filters scope every number the way the dashboard's filter chips do; `framework` matches + the platform label traces carry (see TRACING.md's Platform detection), with "other" + selecting unlabeled traffic.""" + params = {"window": window} + for key, value in (("agent", agent), ("model", model), ("tool", tool), ("framework", framework), ("status", status)): + if value is not None: + params[key] = value + return self._request("GET", "/metrics", params=params) + def topics(self, window: str = "7d") -> dict: """The Topics view's data over a window ("24h", "7d", "30d"): LLM-classified themes of sampled production traffic with per-topic counts and sentiment. Empty until Topics is diff --git a/agentx/tracing/framework_detect.py b/agentx/tracing/framework_detect.py new file mode 100644 index 0000000..c4286a9 --- /dev/null +++ b/agentx/tracing/framework_detect.py @@ -0,0 +1,64 @@ +"""Best-effort agent-framework auto-detection (the "platform agnostic" story). + +A span whose framework was neither passed explicitly (``tracer.trace(..., +framework="...")``) nor adopted from a framework integration (callback handler, +observer, patched client - see ``_TraceSpan._captured_framework``) gets labeled +by looking at which known ORCHESTRATION framework is actually imported in this +process. ``sys.modules`` is the signal - imported, not merely installed - so a +machine with ten frameworks pip-installed but one in use still resolves. + +Only unambiguous answers are given: zero or more than one known framework +loaded means ``None``, and the span goes out unlabeled rather than mislabeled. +The user's explicit ``framework=`` always wins, including totally custom names +for platforms this table has never heard of. + +Raw provider SDKs (openai, anthropic, google-genai, ...) are deliberately NOT +in this table: they are transitive dependencies of nearly every framework, so +their presence says nothing about what orchestrates the agent - and their +patched-client integrations already stamp the provider literal on the spans +they create. +""" + +from __future__ import annotations + +import sys +from typing import Optional + +# Top-level module name -> the wire literal the matching integration emits. +# Multiple modules may map to one literal (langgraph is the LangChain family). +_ORCHESTRATOR_MODULES = { + "langchain": "langchain", + "langchain_core": "langchain", + "langgraph": "langchain", + "crewai": "crewai", + "llama_index": "llamaindex", + "autogen": "autogen", + "autogen_agentchat": "autogen", + "agents": "openai-agents", # the OpenAI Agents SDK's import name + "google.adk": "google-adk", + "semantic_kernel": "semantic-kernel", + "haystack": "haystack", + "pydantic_ai": "pydantic-ai", + "smolagents": "smolagents", + "dspy": "dspy", +} + + +def _looks_like_openai_agents_sdk() -> bool: + # "agents" is a name any user package could claim - only trust it when the + # OpenAI Agents SDK's own submodules are loaded alongside it. + return "agents.run" in sys.modules or "agents.tracing" in sys.modules + + +def detect_framework() -> Optional[str]: + """The single unambiguous orchestration framework imported right now, else None.""" + found: set = set() + for module, literal in _ORCHESTRATOR_MODULES.items(): + if module not in sys.modules: + continue + if module == "agents" and not _looks_like_openai_agents_sdk(): + continue + found.add(literal) + if len(found) > 1: + return None + return found.pop() if len(found) == 1 else None diff --git a/agentx/tracing/tracer.py b/agentx/tracing/tracer.py index b6eb24e..20e38ca 100644 --- a/agentx/tracing/tracer.py +++ b/agentx/tracing/tracer.py @@ -14,6 +14,7 @@ from agentx.tracing.ingest_client import IngestClient from agentx.tracing.ci_types import CIRun, CIRunResult, CIRunStatus, CIQuestionScore from agentx.tracing.eval_scope import EVAL_RUN_SOURCE, current_eval_run_id +from agentx.tracing.framework_detect import detect_framework F = TypeVar("F", bound=Callable[..., Any]) @@ -125,6 +126,10 @@ def __init__( # Adopted from a merged child run (e.g. AgentXCallbackHandler) when this span itself # wasn't opened with an explicit framework= - see _merge_child_run below. self._captured_framework: Optional[str] = None + # Best-effort auto-detection (framework_detect.py), resolved once at span open so child + # spans emitted mid-flight carry the same label the root will. Lowest precedence: + # explicit framework= > integration adoption > this. None when ambiguous. + self._detected_framework: Optional[str] = None if framework else detect_framework() self._input_tokens: int = 0 self._output_tokens: int = 0 # Subsets of _input_tokens (not additional tokens) - a prompt-caching write/read, when the @@ -191,7 +196,7 @@ def __exit__(self, exc_type, exc_val, tb): latency_ms=latency_ms, error=self._error, metadata=metadata, - framework=self._framework or self._captured_framework, + framework=self._framework or self._captured_framework or self._detected_framework or detect_framework(), model=self._model or self._captured_model, tool_calls=self.tool_calls or None, session_id=self._session_id, @@ -235,14 +240,18 @@ def _record_llm_call( input: Any = None, output: Any = None, model: Optional[str] = None, + framework: Optional[str] = None, input_tokens: Optional[int] = None, output_tokens: Optional[int] = None, cache_read_tokens: Optional[int] = None, cache_write_tokens: Optional[int] = None, ) -> None: """Record one LLM-call child span (e.g. one patched Anthropic call) under this span - - name left unset so _merge_child_run auto-numbers it "LLM Call N".""" + name left unset so _merge_child_run auto-numbers it "LLM Call N". ``framework`` lets the + patched client stamp its provider literal on a span the user opened without one - the + adoption in _merge_child_run keeps explicit/integration labels winning.""" self._merge_child_run( + framework=framework, execution_steps=[{ "duration_ms": duration_ms, "start_time": start_time, @@ -301,7 +310,7 @@ def child_span( child = _TraceSpan( tracer=self._tracer, name=name, - framework=framework or self._framework or self._captured_framework, + framework=framework or self._framework or self._captured_framework or self._detected_framework, model=model, session_id=self._session_id, ) @@ -397,6 +406,13 @@ def _merge_child_run( under this span). """ with self._merge_lock: + # Adopt framework/model BEFORE emitting child spans: child_span resolves its + # framework from this span's fields, so adopting after the emission loops used to + # send every CrewAI/AutoGen child out unlabeled while only the root got stamped. + if model and not self._captured_model: + self._captured_model = model + if framework and not self._captured_framework: + self._captured_framework = framework for step in [] if not emit_steps else (execution_steps or []): self._child_span_count += 1 self.child_span( @@ -463,10 +479,6 @@ def _merge_child_run( self.input = input if output is not None: self.output = output - if model and not self._captured_model: - self._captured_model = model - if framework and not self._captured_framework: - self._captured_framework = framework if input_tokens: self._input_tokens += input_tokens if output_tokens: @@ -520,7 +532,8 @@ def wrapper(*args, **kwargs): # to be called from inside another active span. span = self._tracer.trace( self.name, metadata=self._metadata, framework=self._framework, model=self._model, - session_id=self._session_id, + session_id=self._session_id, sync=self._sync, monitor=self._monitor, + pattern_ids=self._pattern_ids, agent_id=self._agent_id, span_kind=self._span_kind, ) span.__enter__() try: @@ -542,7 +555,8 @@ async def wrapper(*args, **kwargs): # See _wrap_sync's comment - same "fresh span per call" reasoning applies here. span = self._tracer.trace( self.name, metadata=self._metadata, framework=self._framework, model=self._model, - session_id=self._session_id, + session_id=self._session_id, sync=self._sync, monitor=self._monitor, + pattern_ids=self._pattern_ids, agent_id=self._agent_id, span_kind=self._span_kind, ) span.__enter__() try: @@ -911,6 +925,22 @@ def trace( with client.tracer.trace("support-agent", agent_id="ag_123", sync=True) as span: span.output = call_llm(...) + + ``framework`` names the platform the agent runs on - tracing is platform agnostic, and + this label is how the dashboard's framework filter and Monitor's Platforms chart group + traffic. Three ways it gets set, strongest first: + + 1. **Explicit**: ``framework="langchain"`` - any string works, including platforms + AgentX has no integration for (``framework="my-inhouse-runner"``). + 2. **Integration**: every integration stamps its own literal automatically - + ``langchain``, ``crewai``, ``openai-agents``, ``openai``, ``anthropic``, + ``google-genai``, ``google-adk``, ``litellm``, ``llamaindex``, ``autogen``, + ``moveworks``, ``databricks``. + 3. **Auto-detection**: with neither of the above, the SDK labels the span with the one + known orchestration framework imported in the process (LangChain/LangGraph, CrewAI, + LlamaIndex, AutoGen, OpenAI Agents SDK, Google ADK, Semantic Kernel, Haystack, + Pydantic AI, smolagents, DSPy). Ambiguous (several imported) or unknown -> the span + goes out unlabeled rather than mislabeled. """ return _TraceSpan( tracer=self, diff --git a/tests/test_framework_detect.py b/tests/test_framework_detect.py new file mode 100644 index 0000000..37af118 --- /dev/null +++ b/tests/test_framework_detect.py @@ -0,0 +1,125 @@ +""" +Platform-agnostic framework capture: explicit framework= always wins, integrations stamp their +literal, and - new - the SDK auto-detects the single unambiguous orchestration framework +imported in the process (agentx/tracing/framework_detect.py). Also pins the two capture-gap +fixes: _merge_child_run adopts the framework BEFORE emitting child spans (CrewAI/AutoGen +children used to go out unlabeled), and _record_llm_call forwards the patched client's provider +literal onto a user-opened span that has no label of its own. + +Same harness as test_span_tree.py: only the ingest_client boundary is mocked, so the real +_send/_dispatch/child_span paths run and every wire dict is inspectable. +""" +from __future__ import annotations + +import sys +import types +from unittest.mock import MagicMock + +import pytest + +from agentx.tracing.framework_detect import _ORCHESTRATOR_MODULES, detect_framework +from agentx.tracing.tracer import Tracer + + +def make_tracer() -> Tracer: + return Tracer(ingest_client=MagicMock()) + + +def enqueued_wires(tracer: Tracer) -> list: + return [call.args[0] for call in tracer._client.enqueue.call_args_list] + + +@pytest.fixture() +def clean_modules(monkeypatch): + """Remove every known orchestrator from sys.modules so each test states its own world.""" + for module in list(_ORCHESTRATOR_MODULES) + ["agents.run", "agents.tracing"]: + monkeypatch.delitem(sys.modules, module, raising=False) + return monkeypatch + + +def fake_import(monkeypatch, name: str) -> None: + monkeypatch.setitem(sys.modules, name, types.ModuleType(name)) + + +def test_detects_single_imported_orchestrator(clean_modules): + fake_import(clean_modules, "crewai") + assert detect_framework() == "crewai" + tracer = make_tracer() + with tracer.trace("agent"): + pass + assert enqueued_wires(tracer)[0]["framework"] == "crewai" + + +def test_no_orchestrator_means_no_label(clean_modules): + assert detect_framework() is None + tracer = make_tracer() + with tracer.trace("agent"): + pass + assert "framework" not in enqueued_wires(tracer)[0] + + +def test_ambiguous_imports_stay_unlabeled(clean_modules): + fake_import(clean_modules, "crewai") + fake_import(clean_modules, "llama_index") + assert detect_framework() is None + + +def test_langchain_family_collapses_to_one_literal(clean_modules): + # langgraph + langchain_core together are ONE framework, not an ambiguity. + fake_import(clean_modules, "langchain_core") + fake_import(clean_modules, "langgraph") + assert detect_framework() == "langchain" + + +def test_agents_module_needs_sdk_submodules(clean_modules): + fake_import(clean_modules, "agents") # could be anyone's package named "agents" + assert detect_framework() is None + fake_import(clean_modules, "agents.run") + assert detect_framework() == "openai-agents" + + +def test_explicit_framework_beats_detection(clean_modules): + fake_import(clean_modules, "crewai") + tracer = make_tracer() + with tracer.trace("agent", framework="my-inhouse-runner"): + pass + assert enqueued_wires(tracer)[0]["framework"] == "my-inhouse-runner" + + +def test_children_inherit_detected_framework(clean_modules): + fake_import(clean_modules, "llama_index") + tracer = make_tracer() + with tracer.trace("agent") as span: + span.child_span("step", duration_ms=5) + wires = enqueued_wires(tracer) + assert [w.get("framework") for w in wires] == ["llamaindex", "llamaindex"] + + +def test_merge_child_run_adopts_before_emitting_children(clean_modules): + # The CrewAI/AutoGen shape: root opened with no framework, the merged sub-run carries it. + # Children must go out labeled too - adoption used to happen after emission. + tracer = make_tracer() + with tracer.trace("crew") as span: + span._merge_child_run( + framework="crewai", + execution_steps=[{"duration_ms": 10, "input": "q", "output": "a"}], + ) + wires = enqueued_wires(tracer) + assert len(wires) == 2 + assert all(w.get("framework") == "crewai" for w in wires) + + +def test_record_llm_call_stamps_provider_on_unlabeled_span(clean_modules): + tracer = make_tracer() + with tracer.trace("agent") as span: + span._record_llm_call(duration_ms=7, model="claude-x", framework="anthropic") + root = enqueued_wires(tracer)[-1] + assert root["framework"] == "anthropic" + + +def test_record_llm_call_never_overrides_explicit_framework(clean_modules): + tracer = make_tracer() + with tracer.trace("agent", framework="langchain") as span: + span._record_llm_call(duration_ms=7, model="gpt-x", framework="openai") + root = enqueued_wires(tracer)[-1] + assert root["framework"] == "langchain"