Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/google/adk/evaluation/llm_as_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,11 @@ async def _evaluate_single_sample_with_sem(
)
],
config=self._judge_model_options.judge_model_config
or genai_types.GenerateContentConfig(),
or genai_types.GenerateContentConfig(
automatic_function_calling=genai_types.AutomaticFunctionCallingConfig(
disable=True
)
),
)
add_default_retry_options_if_not_present(llm_request)
num_samples = self._judge_model_options.num_samples
Expand Down
53 changes: 53 additions & 0 deletions tests/unittests/evaluation/test_llm_as_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,59 @@ async def test_evaluate_invocations_with_mock(
assert mock_llm_as_judge.aggregate_invocation_results.call_count == 1


@pytest.mark.asyncio
async def test_evaluate_invocations_default_config_disables_afc(
mock_judge_model, mocker
):
# The default judge request must disable google-genai's automatic function
# calling, since the judge never calls tools and enabling it only produces
# a spurious AFC warning on every eval run.
judge = MockLlmAsJudge(
eval_metric=EvalMetric(
metric_name="test_metric",
threshold=0.5,
criterion=LlmAsAJudgeCriterion(
threshold=0.5,
judge_model_options=JudgeModelOptions(
judge_model="gemini-2.5-flash",
num_samples=1,
),
),
),
criterion_type=LlmAsAJudgeCriterion,
)
judge._judge_model = mock_judge_model
captured_requests = []
original_generate_content_async = mock_judge_model.generate_content_async

def capturing_generate_content_async(llm_request):
captured_requests.append(llm_request)
return original_generate_content_async(llm_request)

judge._judge_model.generate_content_async = capturing_generate_content_async

actual_invocations = [
Invocation(
invocation_id="id1",
user_content=genai_types.Content(
parts=[genai_types.Part(text="user content 1")],
role="user",
),
final_response=genai_types.Content(
parts=[genai_types.Part(text="final response 1")],
role="model",
),
)
]

await judge.evaluate_invocations(actual_invocations)

assert len(captured_requests) == 1
config = captured_requests[0].config
assert config.automatic_function_calling is not None
assert config.automatic_function_calling.disable is True


@pytest.mark.asyncio
async def test_evaluate_invocations_grades_criterion_only_metric(
mock_judge_model,
Expand Down