feat: expose recognized text on user-turn spans

livekit-agents/livekit/agents/telemetry/gen_ai.py

@@ -90,6 +90,11 @@ def _text_part(content: str) -> dict[str, Any]:
     return {"type": "text", "content": content}
 
 
+def to_speech_messages(text: str, *, role: str) -> list[dict[str, Any]]:
+    """Text consumed by speech synthesis or produced by speech recognition."""
+    return [{"role": role, "parts": [_text_part(text)]}] if text else []
+
+
 def _message_parts(item: ChatItem) -> list[dict[str, Any]]:
     from ..llm import AudioContent, ImageContent
 

livekit-agents/livekit/agents/voice/audio_recognition.py

@@ -28,7 +28,7 @@
 from ..language import LanguageCode
 from ..log import logger
 from ..stt import SpeechEvent
-from ..telemetry import trace_types, tracer
+from ..telemetry import gen_ai, trace_types, tracer
 from ..types import NOT_GIVEN, NotGivenOr
 from ..utils import aio, is_given
 from ..vad import VADStream
@@ -1783,6 +1783,13 @@ async def _bounce_eou_task(
                         trace_types.ATTR_END_OF_TURN_DELAY: metrics.end_of_turn_delay or 0,
                     }
                 )
+                if user_turn_span.is_recording() and gen_ai.capture_content_enabled():
+                    gen_ai.set_content_attributes(
+                        user_turn_span,
+                        output_messages=gen_ai.to_speech_messages(
+                            self._audio_transcript, role="user"
+                        ),
+                    )
                 if self._stt_request_ids:
                     user_turn_span.set_attribute(
                         trace_types.ATTR_PROVIDER_REQUEST_IDS, self._stt_request_ids
@@ -2005,15 +2012,13 @@ def _ensure_user_turn_span(self, start_time: float | None = None) -> trace.Span:
         if (room_io := self._session._room_io) and room_io.linked_participant:
             _set_participant_attributes(self._user_turn_span, room_io.linked_participant)
 
-        # add STT model/provider attributes
-        if self._stt_model:
-            self._user_turn_span.set_attribute(
-                trace_types.ATTR_GEN_AI_REQUEST_MODEL, self._stt_model
-            )
-        if self._stt_provider:
-            self._user_turn_span.set_attribute(
-                trace_types.ATTR_GEN_AI_PROVIDER_NAME, self._stt_provider
-            )
+        gen_ai.set_request_attributes(
+            self._user_turn_span,
+            operation=None,
+            model=self._stt_model,
+            provider=self._stt_provider,
+            output_type=trace_types.GenAIOutputType.TEXT,
+        )
 
         return self._user_turn_span
 

tests/test_stt_event_traces.py

@@ -12,7 +12,7 @@
 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
 
 from livekit.agents import AgentSession, llm, stt
-from livekit.agents.telemetry import set_tracer_provider, trace_types, tracer
+from livekit.agents.telemetry import gen_ai, set_tracer_provider, trace_types, tracer
 from livekit.agents.voice.audio_recognition import AudioRecognition
 from livekit.agents.voice.endpointing import BaseEndpointing
 
@@ -162,3 +162,35 @@ async def test_transcript_list_is_not_limited_by_span_event_count(
     [events] = _turn_events(exporter)
     assert len(events) == 130
     assert all(event["transcript_length"] == 0 for event in events)
+
+
+@pytest.mark.parametrize("capture", [True, False])
+async def test_committed_transcript_is_the_user_turn_output(
+    recognition: AudioRecognition,
+    exporter: InMemorySpanExporter,
+    monkeypatch: pytest.MonkeyPatch,
+    capture: bool,
+) -> None:
+    monkeypatch.setattr(gen_ai, "_capture_content", capture)
+    recognition._stt_provider = "google"
+    recognition._stt_model = "test-stt"
+    for transcript in ["first turn", "second turn"]:
+        await recognition._on_stt_event(
+            stt.SpeechEvent(
+                type=stt.SpeechEventType.FINAL_TRANSCRIPT,
+                alternatives=[stt.SpeechData(language="en", text=transcript)],
+            )
+        )
+        await _commit(recognition)
+    turns = [span for span in exporter.get_finished_spans() if span.name == "user_turn"]
+    assert len(turns) == 2
+    for turn, transcript in zip(turns, ["first turn", "second turn"], strict=True):
+        assert turn.attributes["gen_ai.provider.name"] == "gcp.gen_ai"
+        assert turn.attributes["gen_ai.request.model"] == "test-stt"
+        assert "gen_ai.input.messages" not in turn.attributes
+        if capture:
+            assert json.loads(turn.attributes["gen_ai.output.messages"]) == [
+                {"role": "user", "parts": [{"type": "text", "content": transcript}]}
+            ]
+        else:
+            assert "gen_ai.output.messages" not in turn.attributes