diff --git a/src/google/adk/evaluation/hallucinations_v1.py b/src/google/adk/evaluation/hallucinations_v1.py index 5e32389316..e0a77b6ff1 100644 --- a/src/google/adk/evaluation/hallucinations_v1.py +++ b/src/google/adk/evaluation/hallucinations_v1.py @@ -45,6 +45,7 @@ from .evaluator import Evaluator from .evaluator import PerInvocationResult from .llm_as_judge_utils import get_eval_status +from .llm_as_judge_utils import get_grounding_metadata_as_json_str from .llm_as_judge_utils import get_text_from_content from .llm_as_judge_utils import get_tool_declarations_as_json_str @@ -126,7 +127,8 @@ 3. **For each label, provide a short rationale explaining your decision.** The rationale should be separate from the excerpt. 4. **Be very strict with your `supported`, `contradictory` and `disputed` decisions.** Unless you can find straightforward, indisputable evidence excepts *in the context* that a sentence is `supported`, `contradictory` or `disputed`, consider it `unsupported`. You should not employ world knowledge unless it is truly trivial. 5. "tool_outputs" blocks contain code execution results of the "tool_code" blocks immediately above them. If any sentence is based on "tool_outputs" results, first analyze if the corresponding "tool_code" is supported and if the results are error-free. Only if the "tool_code" block is supported, you can treat code execution results as correct. -6. If you need to cite multiple supporting excerpts, simply concatenate them. Excerpt could be summary from the context if it is too long. +6. "Grounding metadata" is trusted evidence from model-internal tools (e.g. search) whose results may not otherwise appear in "tool_outputs". A sentence entailed by "Grounding metadata" should be treated as `supported`. +7. If you need to cite multiple supporting excerpts, simply concatenate them. Excerpt could be summary from the context if it is too long. **Input Format:** @@ -447,6 +449,10 @@ def _create_context_for_step( ) context_parts.append("Tool definitions:") context_parts.append(f"{tool_declarations}\n") + context_parts.append("Grounding metadata:") + context_parts.append( + f"{get_grounding_metadata_as_json_str(InvocationEvents(invocation_events=events))}\n" + ) for event in events: if not event.content or not event.content.parts: diff --git a/tests/unittests/evaluation/test_hallucinations_v1.py b/tests/unittests/evaluation/test_hallucinations_v1.py index b7a1c42f47..7be71a4a8d 100644 --- a/tests/unittests/evaluation/test_hallucinations_v1.py +++ b/tests/unittests/evaluation/test_hallucinations_v1.py @@ -362,6 +362,9 @@ def test_create_context_for_intermediate_step( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ { @@ -414,6 +417,9 @@ def test_create_context_for_final_step( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ { @@ -458,6 +464,30 @@ def test_create_context_for_final_step( """ assert context.strip() == expected_context.strip() + def test_create_context_for_step_includes_grounding_metadata( + self, hallucinations_metric, create_context_data + ): + """Sentences grounded only via grounding_metadata must reach the judge.""" + app_details, events, invocation = create_context_data + grounding_event = InvocationEvent( + author="root", + content=None, + grounding_metadata=genai_types.GroundingMetadata( + web_search_queries=["recent AI news"] + ), + ) + context = hallucinations_metric._create_context_for_step( + app_details, invocation, events + [grounding_event] + ) + grounding_section = context.split("Grounding metadata:\n")[1].split( + "\n\ntool_calls:" + )[0] + parsed = json.loads(grounding_section) + assert parsed["grounding_metadata"][0]["author"] == "root" + assert parsed["grounding_metadata"][0]["grounding_metadata"][ + "web_search_queries" + ] == ["recent AI news"] + @pytest.fixture def agent_tree_data(): @@ -623,7 +653,10 @@ async def test_evaluate_invocations_multi_agents( ], "agent2": [] } -}""" +} + +Grounding metadata: +No grounding metadata was provided.""" expected_context5 = R"""Developer instructions: root: Root agent instructions. @@ -662,6 +695,9 @@ async def test_evaluate_invocations_multi_agents( } } +Grounding metadata: +No grounding metadata was provided. + Hi, I am root. tool_calls: @@ -739,6 +775,9 @@ async def test_evaluate_invocations_multi_agents( } } +Grounding metadata: +No grounding metadata was provided. + Hi, I am root. tool_calls: @@ -872,6 +911,9 @@ async def test_evaluate_invocations_agent_tree_skip_intermediate( } } +Grounding metadata: +No grounding metadata was provided. + Hi, I am root. tool_calls: @@ -1080,6 +1122,9 @@ async def test_evaluate_invocations_time_weather( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ { @@ -1125,6 +1170,9 @@ async def test_evaluate_invocations_time_weather( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ { @@ -1243,6 +1291,9 @@ async def test_evaluate_invocations_time_weather_skip_intermediate( } } +Grounding metadata: +No grounding metadata was provided. + tool_calls: [ {