diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index ef468d9602..699e6f79ee 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -814,6 +814,12 @@ class SPANDATA: Example: 100 """ + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS = "gen_ai.usage.reasoning.output_tokens" + """ + The number of tokens used for reasoning to create the AI output. + Example: 75 + """ + GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" """ The number of tokens in the output. diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index e53851a676..b37820e88b 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -69,6 +69,7 @@ class TokenUsage(NamedTuple): total_tokens: "Optional[int]" cache_read: "Optional[int]" cache_creation: "Optional[int]" + reasoning: "Optional[int]" try: @@ -735,6 +736,7 @@ def _extract_tokens_from_generations( total_total = 0 total_cache_read = None total_cache_creation = None + reasoning = None for gen_list in generations: if not gen_list: @@ -761,18 +763,23 @@ def _extract_tokens_from_generations( continue input_token_details = usage_metadata.get("input_token_details") - if not isinstance(input_token_details, dict): - continue + if isinstance(input_token_details, dict): + if isinstance(input_token_details.get("cache_read"), int): + total_cache_read = (total_cache_read or 0) + input_token_details[ + "cache_read" + ] - if isinstance(input_token_details.get("cache_read"), int): - total_cache_read = (total_cache_read or 0) + input_token_details[ - "cache_read" - ] + if isinstance(input_token_details.get("cache_creation"), int): + total_cache_creation = ( + total_cache_creation or 0 + ) + input_token_details["cache_creation"] - if isinstance(input_token_details.get("cache_creation"), int): - total_cache_creation = (total_cache_creation or 0) + input_token_details[ - "cache_creation" - ] + output_token_details = usage_metadata.get("output_token_details") + if not isinstance(output_token_details, dict): + continue + + if isinstance(output_token_details.get("reasoning"), int): + reasoning = (reasoning or 0) + output_token_details["reasoning"] return TokenUsage( total_input if total_input > 0 else None, @@ -780,6 +787,7 @@ def _extract_tokens_from_generations( total_total if total_total > 0 else None, total_cache_read, total_cache_creation, + reasoning, ) @@ -819,6 +827,7 @@ def _record_token_usage( total_tokens = None cache_read_tokens = None cache_creation_tokens = None + reasoning = None # Legacy that reads provider-specific token information. token_usage = _get_token_usage(response) @@ -838,6 +847,8 @@ def _record_token_usage( cache_read_tokens = token_usage.cache_read if token_usage.cache_creation is not None: cache_creation_tokens = token_usage.cache_creation + if token_usage.reasoning is not None: + reasoning = token_usage.reasoning set_on_span = ( span.set_attribute if isinstance(span, StreamedSpan) else span.set_data @@ -860,6 +871,9 @@ def _record_token_usage( SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, cache_creation_tokens ) + if reasoning is not None: + set_on_span(SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, reasoning) + def _get_request_data( obj: "Any", args: "Any", kwargs: "Any" diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index e0ca232302..08fa52ab48 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -271,6 +271,7 @@ def nonstreaming_multi_candidate_google_genai_model_response(): cached_content_token_count=4, prompt_token_count=10, candidates_token_count=20, + thoughts_token_count=5, total_token_count=30, ), ) @@ -591,7 +592,7 @@ def test_langchain_multi_choice_response( assert len(chat_spans) == 1 assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 25 assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 assert ( @@ -616,7 +617,7 @@ def test_langchain_multi_choice_response( assert len(chat_spans) == 1 assert chat_spans[0]["data"]["gen_ai.usage.input_tokens"] == 10 - assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 20 + assert chat_spans[0]["data"]["gen_ai.usage.output_tokens"] == 25 assert chat_spans[0]["data"]["gen_ai.usage.total_tokens"] == 30 assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4 @@ -817,6 +818,11 @@ def test_langchain_create_agent( == 6 ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] + == 5 + ) + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert ( chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" @@ -904,6 +910,11 @@ def test_langchain_create_agent( == 6 ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] + == 5 + ) + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert ( chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" @@ -979,6 +990,8 @@ def test_langchain_create_agent( == 6 ) + assert chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 5 + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert chat_spans[0]["data"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" @@ -1054,7 +1067,7 @@ def test_tool_execution_span( ), output_tokens=50, output_tokens_details=OutputTokensDetails( - reasoning_tokens=0, + reasoning_tokens=10, ), total_tokens=192, ), @@ -1066,7 +1079,7 @@ def test_tool_execution_span( ), output_tokens=28, output_tokens_details=OutputTokensDetails( - reasoning_tokens=0, + reasoning_tokens=11, ), total_tokens=117, ), @@ -1153,6 +1166,10 @@ def test_tool_execution_span( ] == 31 ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] + == 10 + ) assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 @@ -1168,6 +1185,10 @@ def test_tool_execution_span( ] == 10 ) + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] + == 11 + ) assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat" if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): @@ -1286,6 +1307,10 @@ def test_tool_execution_span( ] == 31 ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] + == 10 + ) assert chat_spans[0]["attributes"]["gen_ai.system"] == "openai-chat" assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 @@ -1301,6 +1326,10 @@ def test_tool_execution_span( ] == 10 ) + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] + == 11 + ) assert chat_spans[1]["attributes"]["gen_ai.system"] == "openai-chat" if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): @@ -1416,6 +1445,9 @@ def test_tool_execution_span( chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] == 31 ) + assert ( + chat_spans[0]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 10 + ) assert chat_spans[0]["data"]["gen_ai.system"] == "openai-chat" assert chat_spans[1]["data"]["gen_ai.usage.input_tokens"] == 89 @@ -1428,6 +1460,9 @@ def test_tool_execution_span( chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] == 10 ) + assert ( + chat_spans[1]["data"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 11 + ) assert chat_spans[1]["data"]["gen_ai.system"] == "openai-chat" if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13):