diff --git a/src/google/adk/evaluation/llm_as_judge.py b/src/google/adk/evaluation/llm_as_judge.py index ce484ef58a7..9d6f59c7fb9 100644 --- a/src/google/adk/evaluation/llm_as_judge.py +++ b/src/google/adk/evaluation/llm_as_judge.py @@ -222,7 +222,11 @@ async def _evaluate_single_sample_with_sem( ) ], config=self._judge_model_options.judge_model_config - or genai_types.GenerateContentConfig(), + or genai_types.GenerateContentConfig( + automatic_function_calling=genai_types.AutomaticFunctionCallingConfig( + disable=True + ) + ), ) add_default_retry_options_if_not_present(llm_request) num_samples = self._judge_model_options.num_samples diff --git a/tests/unittests/evaluation/test_llm_as_judge.py b/tests/unittests/evaluation/test_llm_as_judge.py index d535febf4c9..f0b98f4065f 100644 --- a/tests/unittests/evaluation/test_llm_as_judge.py +++ b/tests/unittests/evaluation/test_llm_as_judge.py @@ -250,6 +250,59 @@ async def test_evaluate_invocations_with_mock( assert mock_llm_as_judge.aggregate_invocation_results.call_count == 1 +@pytest.mark.asyncio +async def test_evaluate_invocations_default_config_disables_afc( + mock_judge_model, mocker +): + # The default judge request must disable google-genai's automatic function + # calling, since the judge never calls tools and enabling it only produces + # a spurious AFC warning on every eval run. + judge = MockLlmAsJudge( + eval_metric=EvalMetric( + metric_name="test_metric", + threshold=0.5, + criterion=LlmAsAJudgeCriterion( + threshold=0.5, + judge_model_options=JudgeModelOptions( + judge_model="gemini-2.5-flash", + num_samples=1, + ), + ), + ), + criterion_type=LlmAsAJudgeCriterion, + ) + judge._judge_model = mock_judge_model + captured_requests = [] + original_generate_content_async = mock_judge_model.generate_content_async + + def capturing_generate_content_async(llm_request): + captured_requests.append(llm_request) + return original_generate_content_async(llm_request) + + judge._judge_model.generate_content_async = capturing_generate_content_async + + actual_invocations = [ + Invocation( + invocation_id="id1", + user_content=genai_types.Content( + parts=[genai_types.Part(text="user content 1")], + role="user", + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="final response 1")], + role="model", + ), + ) + ] + + await judge.evaluate_invocations(actual_invocations) + + assert len(captured_requests) == 1 + config = captured_requests[0].config + assert config.automatic_function_calling is not None + assert config.automatic_function_calling.disable is True + + @pytest.mark.asyncio async def test_evaluate_invocations_grades_criterion_only_metric( mock_judge_model,