fix: raise when evaluate or evaluate_eval_set evaluates zero cases

`AgentEvaluator.evaluate` and `AgentEvaluator.evaluate_eval_set` now raise `ValueError` instead of silently passing when no test cases are evaluated (e.g. empty eval set or no `*.test.json` files found in directory). Both methods also validate that `num_runs` is at least 1 up front.

Merge #6952

Fixes #6951

PiperOrigin-RevId: 982715587

Author: chelsealong

src/google/adk/evaluation/agent_evaluator.py

@@ -168,11 +168,21 @@ async def evaluate_eval_set(
       eval_set_results_manager: Optional manager used to persist the eval set
         evaluation result as `*.evalset_result.json`.
     """
+    if num_runs < 1:
+      raise ValueError(f"`num_runs` must be at least 1, got {num_runs}.")
+
     if eval_set_results_manager is not None and not app_name:
       raise ValueError(
           "app_name is required when eval_set_results_manager is provided."
       )
 
+    if not eval_set or not eval_set.eval_cases:
+      raise ValueError(
+          "No eval cases were evaluated, so there is nothing to report a"
+          " pass or a failure for. This happens when `eval_set` has no eval"
+          " cases."
+      )
+
     if criteria:
       logger.warning(
           "`criteria` field is deprecated and will be removed in future"
@@ -331,6 +341,9 @@ async def evaluate(
       eval_set_results_manager: Optional manager used to persist the eval set
         evaluation result as `*.evalset_result.json`.
     """
+    if num_runs < 1:
+      raise ValueError(f"`num_runs` must be at least 1, got {num_runs}.")
+
     if eval_set_results_manager is not None and not app_name:
       raise ValueError(
           "app_name is required when eval_set_results_manager is provided."
@@ -347,6 +360,12 @@ async def evaluate(
     else:
       test_files = [eval_dataset_file_path_or_dir]
 
+    if not test_files:
+      raise ValueError(
+          "No `*.test.json` eval files found in"
+          f" {eval_dataset_file_path_or_dir}, so there is nothing to evaluate."
+      )
+
     initial_session = AgentEvaluator._get_initial_session(initial_session_file)
 
     for test_file in test_files:

tests/unittests/evaluation/test_agent_evaluator.py

@@ -136,7 +136,7 @@ async def _empty(*args, **kwargs):
 
   await AgentEvaluator.evaluate_eval_set(
       agent_module="my.agent.module",
-      eval_set=EvalSet(eval_set_id="es1", eval_cases=[]),
+      eval_set=_make_eval_set(),
       eval_config=EvalConfig(),
       num_runs=1,
       artifact_service=my_service,
@@ -320,6 +320,42 @@ async def test_evaluate_eval_set_passes_when_metrics_pass(mocker):
   await _mock_evaluate_eval_set(mocker, passing_result)
 
 
+@pytest.mark.asyncio
+async def test_evaluate_eval_set_raises_when_no_eval_cases_were_evaluated(
+    mocker,
+):
+  """An empty `eval_set` must fail early before evaluating."""
+  mock_get_agent = mocker.patch.object(AgentEvaluator, "_get_agent_for_eval")
+  mock_get_results = mocker.patch.object(
+      AgentEvaluator, "_get_eval_results_by_eval_id"
+  )
+
+  with pytest.raises(ValueError, match="No eval cases were evaluated"):
+    await AgentEvaluator.evaluate_eval_set(
+        agent_module="my.agent.module",
+        eval_set=EvalSet(eval_set_id="es1", eval_cases=[]),
+        eval_config=EvalConfig(),
+        num_runs=1,
+    )
+
+  mock_get_agent.assert_not_called()
+  mock_get_results.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("num_runs", [0, -1])
+async def test_evaluate_eval_set_raises_when_num_runs_less_than_one(num_runs):
+  with pytest.raises(
+      ValueError, match=f"`num_runs` must be at least 1, got {num_runs}."
+  ):
+    await AgentEvaluator.evaluate_eval_set(
+        agent_module="my.agent.module",
+        eval_set=_make_eval_set(),
+        eval_config=EvalConfig(),
+        num_runs=num_runs,
+    )
+
+
 class TestGetAgentForEval:
   """Resolution of the wrapping App alongside the agent to evaluate."""
 
@@ -1006,6 +1042,33 @@ async def test_evaluate_requires_app_name_when_manager_given(mocker):
     )
 
 
+@pytest.mark.asyncio
+async def test_evaluate_raises_when_no_test_files_found(tmp_path):
+  empty_dir = tmp_path / "empty_evals"
+  empty_dir.mkdir()
+  with pytest.raises(
+      ValueError,
+      match=r"No `\*\.test\.json` eval files found in",
+  ):
+    await AgentEvaluator.evaluate(
+        agent_module="pkg.search_agent",
+        eval_dataset_file_path_or_dir=str(empty_dir),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("num_runs", [0, -1])
+async def test_evaluate_raises_when_num_runs_less_than_one(num_runs):
+  with pytest.raises(
+      ValueError, match=f"`num_runs` must be at least 1, got {num_runs}."
+  ):
+    await AgentEvaluator.evaluate(
+        agent_module="pkg.search_agent",
+        eval_dataset_file_path_or_dir="some.test.json",
+        num_runs=num_runs,
+    )
+
+
 @pytest.mark.asyncio
 async def test_evaluate_passes_results_manager_and_app_name(mocker, tmp_path):
   test_dir = tmp_path / "evals"