From 5ac927781671fc5e5e6b54e982b859e8cb53322e Mon Sep 17 00:00:00 2001 From: chelsealong Date: Thu, 17 Sep 2026 11:56:14 +0000 Subject: [PATCH] fix(evaluation): include grounding metadata in hallucinations judge context HallucinationsV1Evaluator._create_context_for_step never included InvocationEvent.grounding_metadata in the context passed to the sentence validator, unlike the sibling rubric_based_final_response_quality_v1 metric which already calls get_grounding_metadata_as_json_str. Agents that rely on model-internal grounding tools (e.g. VertexAiSearchTool, google_search) only ever surface search results via event.grounding_metadata, never as a function_response part, so genuinely grounded sentences had no supporting evidence in the judge's context and were scored unsupported. Closes: #7157 --- .../adk/evaluation/hallucinations_v1.py | 8 ++- .../evaluation/test_hallucinations_v1.py | 53 ++++++++++++++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/google/adk/evaluation/hallucinations_v1.py b/src/google/adk/evaluation/hallucinations_v1.py index 5e32389316f..e0a77b6ff18 100644 --- a/src/google/adk/evaluation/hallucinations_v1.py +++ b/src/google/adk/evaluation/hallucinations_v1.py @@ -45,6 +45,7 @@ from .evaluator import Evaluator from .evaluator import PerInvocationResult from .llm_as_judge_utils import get_eval_status +from .llm_as_judge_utils import get_grounding_metadata_as_json_str from .llm_as_judge_utils import get_text_from_content from .llm_as_judge_utils import get_tool_declarations_as_json_str @@ -126,7 +127,8 @@ 3. **For each label, provide a short rationale explaining your decision.** The rationale should be separate from the excerpt. 4. **Be very strict with your `supported`, `contradictory` and `disputed` decisions.** Unless you can find straightforward, indisputable evidence excepts *in the context* that a sentence is `supported`, `contradictory` or `disputed`, consider it `unsupported`. You should not employ world knowledge unless it is truly trivial. 5. "tool_outputs" blocks contain code execution results of the "tool_code" blocks immediately above them. If any sentence is based on "tool_outputs" results, first analyze if the corresponding "tool_code" is supported and if the results are error-free. Only if the "tool_code" block is supported, you can treat code execution results as correct. -6. If you need to cite multiple supporting excerpts, simply concatenate them. Excerpt could be summary from the context if it is too long. +6. "Grounding metadata" is trusted evidence from model-internal tools (e.g. search) whose results may not otherwise appear in "tool_outputs". A sentence entailed by "Grounding metadata" should be treated as `supported`. +7. If you need to cite multiple supporting excerpts, simply concatenate them. Excerpt could be summary from the context if it is too long. **Input Format:** @@ -447,6 +449,10 @@ def _create_context_for_step( ) context_parts.append("Tool definitions:") context_parts.append(f"{tool_declarations}\n") + context_parts.append("Grounding metadata:") + context_parts.append( + f"{get_grounding_metadata_as_json_str(InvocationEvents(invocation_events=events))}\n" + ) for event in events: if not event.content or not event.content.parts: diff --git a/tests/unittests/evaluation/test_hallucinations_v1.py b/tests/unittests/evaluation/test_hallucinations_v1.py index b7a1c42f47d..7be71a4a8d6 100644 --- a/tests/unittests/evaluation/test_hallucinations_v1.py +++ b/tests/unittests/evaluation/test_hallucinations_v1.py @@ -362,6 +362,9 @@ def test_create_context_for_intermediate_step( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ { @@ -414,6 +417,9 @@ def test_create_context_for_final_step( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ { @@ -458,6 +464,30 @@ def test_create_context_for_final_step( """ assert context.strip() == expected_context.strip() + def test_create_context_for_step_includes_grounding_metadata( + self, hallucinations_metric, create_context_data + ): + """Sentences grounded only via grounding_metadata must reach the judge.""" + app_details, events, invocation = create_context_data + grounding_event = InvocationEvent( + author="root", + content=None, + grounding_metadata=genai_types.GroundingMetadata( + web_search_queries=["recent AI news"] + ), + ) + context = hallucinations_metric._create_context_for_step( + app_details, invocation, events + [grounding_event] + ) + grounding_section = context.split("Grounding metadata:\n")[1].split( + "\n\ntool_calls:" + )[0] + parsed = json.loads(grounding_section) + assert parsed["grounding_metadata"][0]["author"] == "root" + assert parsed["grounding_metadata"][0]["grounding_metadata"][ + "web_search_queries" + ] == ["recent AI news"] + @pytest.fixture def agent_tree_data(): @@ -623,7 +653,10 @@ async def test_evaluate_invocations_multi_agents( ], "agent2": [] } -}""" +} + +Grounding metadata: +No grounding metadata was provided.""" expected_context5 = R"""Developer instructions: root: Root agent instructions. @@ -662,6 +695,9 @@ async def test_evaluate_invocations_multi_agents( } } +Grounding metadata: +No grounding metadata was provided. + Hi, I am root. tool_calls: @@ -739,6 +775,9 @@ async def test_evaluate_invocations_multi_agents( } } +Grounding metadata: +No grounding metadata was provided. + Hi, I am root. tool_calls: @@ -872,6 +911,9 @@ async def test_evaluate_invocations_agent_tree_skip_intermediate( } } +Grounding metadata: +No grounding metadata was provided. + Hi, I am root. tool_calls: @@ -1080,6 +1122,9 @@ async def test_evaluate_invocations_time_weather( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ { @@ -1125,6 +1170,9 @@ async def test_evaluate_invocations_time_weather( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ { @@ -1243,6 +1291,9 @@ async def test_evaluate_invocations_time_weather_skip_intermediate( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ {