From 5fa14552cd06179e8ea08f0ea0d29ea6cafb1ca0 Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Thu, 1 Oct 2026 11:23:33 -0700 Subject: [PATCH] feat: Add per-step judges and cross-region routing to evals PiperOrigin-RevId: 991795392 --- agentplatform/_genai/_evals_common.py | 37 +-- .../_genai/_evals_metric_handlers.py | 26 +- agentplatform/_genai/_evals_metric_loaders.py | 13 +- agentplatform/_genai/_transformers.py | 6 +- agentplatform/_genai/evals.py | 14 + agentplatform/_genai/types/common.py | 43 ++- .../test_evaluate_predefined_metrics.py | 83 ++++++ tests/unit/agentplatform/genai/test_evals.py | 248 ++++++++++++++++++ 8 files changed, 440 insertions(+), 30 deletions(-) diff --git a/agentplatform/_genai/_evals_common.py b/agentplatform/_genai/_evals_common.py index db85f3819d..b448395ecb 100644 --- a/agentplatform/_genai/_evals_common.py +++ b/agentplatform/_genai/_evals_common.py @@ -2886,6 +2886,21 @@ def _resolve_dataset_inputs( return processed_eval_dataset, num_response_candidates +def _prebuilt_evaluation_run_metric( + resolved_metric: types.Metric, +) -> types.EvaluationRunMetric: + """Builds the evaluation run metric for a resolved RubricMetric.""" + if resolved_metric.name in _evals_constant.SUPPORTED_PREDEFINED_METRICS: + metric_config = t.t_metrics([resolved_metric])[0] + else: + metric_config = { + "predefined_metric_spec": {"metric_spec_name": resolved_metric.name} + } + return types.EvaluationRunMetric( + metric=resolved_metric.name, metric_config=metric_config + ) + + def _resolve_evaluation_run_metrics( metrics: Union[list[types.EvaluationRunMetric], list[types.Metric]], api_client: Any ) -> list[types.EvaluationRunMetric]: @@ -2903,14 +2918,7 @@ def _resolve_evaluation_run_metrics( resolved_metric = metric_instance.resolve(api_client=api_client) if resolved_metric.name: resolved_metrics_list.append( - types.EvaluationRunMetric( - metric=resolved_metric.name, - metric_config=types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name=resolved_metric.name, - ) - ), - ) + _prebuilt_evaluation_run_metric(resolved_metric) ) except Exception as e: logger.error( @@ -2944,14 +2952,7 @@ def _resolve_evaluation_run_metrics( ) if resolved_metric.name: resolved_metrics_list.append( - types.EvaluationRunMetric( - metric=resolved_metric.name, - metric_config=types.UnifiedMetric( - predefined_metric_spec=genai_types.PredefinedMetricSpec( - metric_spec_name=resolved_metric.name, - ) - ), - ) + _prebuilt_evaluation_run_metric(resolved_metric) ) else: raise TypeError( @@ -3020,6 +3021,7 @@ def _execute_evaluation( # type: ignore[no-untyped-def] dest: Optional[str] = None, location: Optional[str] = None, evaluation_service_qps: Optional[float] = None, + allow_cross_region_model: Optional[bool] = None, **kwargs, ) -> types.EvaluationResult: """Evaluates a dataset using the provided metrics. @@ -3035,6 +3037,8 @@ def _execute_evaluation( # type: ignore[no-untyped-def] evaluation_service_qps: The rate limit (queries per second) for calls to the evaluation service. Defaults to 10. Increase this value if your project has a higher EvaluateInstances API quota. + allow_cross_region_model: Opt-in flag to authorize cross-region + routing for judge models. **kwargs: Extra arguments to pass to evaluation, such as `agent_info`. Returns: @@ -3117,6 +3121,7 @@ def _execute_evaluation( # type: ignore[no-untyped-def] evaluation_result = _evals_metric_handlers.compute_metrics_and_aggregate( evaluation_run_config, evaluation_service_qps=evaluation_service_qps, + allow_cross_region_model=allow_cross_region_model, ) t2 = time.perf_counter() logger.info("Evaluation took: %f seconds", t2 - t1) diff --git a/agentplatform/_genai/_evals_metric_handlers.py b/agentplatform/_genai/_evals_metric_handlers.py index 8a7bf4c3b4..f8fafb4b53 100644 --- a/agentplatform/_genai/_evals_metric_handlers.py +++ b/agentplatform/_genai/_evals_metric_handlers.py @@ -288,6 +288,7 @@ class MetricHandler(abc.ABC, Generic[T]): def __init__(self, module: "evals.Evals", metric: T): self.module = module self.metric: T = metric + self.allow_cross_region_model: Optional[bool] = None @property @abc.abstractmethod @@ -761,6 +762,7 @@ def get_metric_result( lambda: self.module._evaluate_instances( metrics=[self.metric], instance=instance, + allow_cross_region_model=self.allow_cross_region_model, ), self.metric_name, ) @@ -981,7 +983,7 @@ def __init__(self, module: "evals.Evals", metric: types.Metric): raise ValueError( f"Metric '{self.metric.name}' is not a supported predefined metric." ) - if ( + if self.metric.name.startswith("multi_turn") and ( self.metric.judge_model or self.metric.judge_model_generation_config or self.metric.judge_model_sampling_count @@ -989,7 +991,8 @@ def __init__(self, module: "evals.Evals", metric: types.Metric): logger.warning( "Autorater config settings (judge_model, " "judge_model_generation_config, judge_model_sampling_count) " - "are ignored for predefined metric '%s'.", + "are ignored for multi-turn metric '%s'. Use " + "judge_model_step_configs to set its judges.", self.metric.name, ) @@ -1071,6 +1074,7 @@ def get_metric_result( metrics=[self.metric], instance=payload.get("instance"), autorater_config=payload.get("autorater_config"), + allow_cross_region_model=self.allow_cross_region_model, ), metric_name, ) @@ -1318,6 +1322,7 @@ def get_metric_result( metric_sources=[metric_source], instance=payload.get("instance"), autorater_config=payload.get("autorater_config"), + allow_cross_region_model=self.allow_cross_region_model, ), metric_name, ) @@ -1404,12 +1409,16 @@ def aggregate( def get_handler_for_metric( - module: "evals.Evals", metric: types.Metric + module: "evals.Evals", + metric: types.Metric, + allow_cross_region_model: Optional[bool] = None, ) -> Union[MetricHandlerType, Any]: """Returns a metric handler for the given metric.""" for condition, handler_class in _METRIC_HANDLER_MAPPING: if condition(metric): # type: ignore[no-untyped-call] - return handler_class(module=module, metric=metric) + handler = handler_class(module=module, metric=metric) + handler.allow_cross_region_model = allow_cross_region_model + return handler raise ValueError(f"Unsupported metric: {metric.name}") @@ -1548,6 +1557,7 @@ def _rate_limited_get_metric_result( def compute_metrics_and_aggregate( evaluation_run_config: EvaluationRunConfig, evaluation_service_qps: Optional[float] = None, + allow_cross_region_model: Optional[bool] = None, ) -> types.EvaluationResult: """Computes metrics and aggregates them for a given evaluation run config. @@ -1556,6 +1566,8 @@ def compute_metrics_and_aggregate( evaluation_service_qps: Optional QPS limit for the evaluation service. Defaults to _DEFAULT_EVAL_SERVICE_QPS (10). Users with higher quotas can increase this value. + allow_cross_region_model: Opt-in flag to authorize cross-region + routing for judge models. """ metric_handlers = [] all_futures = [] @@ -1574,7 +1586,11 @@ def compute_metrics_and_aggregate( for eval_metric in evaluation_run_config.metrics: metric_handlers.append( - get_handler_for_metric(evaluation_run_config.evals_module, eval_metric) + get_handler_for_metric( + evaluation_run_config.evals_module, + eval_metric, + allow_cross_region_model=allow_cross_region_model, + ) ) eval_case_count = len(evaluation_run_config.dataset.eval_cases) diff --git a/agentplatform/_genai/_evals_metric_loaders.py b/agentplatform/_genai/_evals_metric_loaders.py index 82df89278f..96b24c711c 100644 --- a/agentplatform/_genai/_evals_metric_loaders.py +++ b/agentplatform/_genai/_evals_metric_loaders.py @@ -215,8 +215,11 @@ def resolve(self, api_client: Any) -> "types.Metric": if self._resolved_metric: return self._resolved_metric + # The shared cache is keyed by name and version only, so a metric with + # overrides (such as judge_model_step_configs) must bypass it. + use_cache = not self.metric_kwargs cache_key = f"{self.name}@{self.version or 'default'}" - if cache_key in LazyLoadedPrebuiltMetric._cache: + if use_cache and cache_key in LazyLoadedPrebuiltMetric._cache: self._resolved_metric = LazyLoadedPrebuiltMetric._cache[cache_key] logger.debug("Metric '%s' found in cache.", cache_key) return self._resolved_metric @@ -225,7 +228,8 @@ def resolve(self, api_client: Any) -> "types.Metric": api_metric = self._resolve_api_predefined() if api_metric: self._resolved_metric = api_metric - LazyLoadedPrebuiltMetric._cache[cache_key] = self._resolved_metric + if use_cache: + LazyLoadedPrebuiltMetric._cache[cache_key] = self._resolved_metric return self._resolved_metric # Fallback to GCS loading for custom LLM-based Prebuilt Metrics @@ -234,8 +238,9 @@ def resolve(self, api_client: Any) -> "types.Metric": ) try: gcs_metric = self._fetch_and_parse(api_client) - final_cache_key = f"{self.name}@{self.version}" - LazyLoadedPrebuiltMetric._cache[final_cache_key] = gcs_metric + if use_cache: + final_cache_key = f"{self.name}@{self.version}" + LazyLoadedPrebuiltMetric._cache[final_cache_key] = gcs_metric self._resolved_metric = gcs_metric return self._resolved_metric except Exception as e: diff --git a/agentplatform/_genai/_transformers.py b/agentplatform/_genai/_transformers.py index f4ec56bdb4..44aafef45b 100644 --- a/agentplatform/_genai/_transformers.py +++ b/agentplatform/_genai/_transformers.py @@ -64,10 +64,14 @@ def t_metrics( elif ( metric_name and metric_name in _evals_constant.SUPPORTED_PREDEFINED_METRICS ): - metric_payload_item["predefined_metric_spec"] = { + predefined_spec: dict[str, Any] = { "metric_spec_name": metric_name, "metric_spec_parameters": metric.metric_spec_parameters, } + step_configs = getv(metric, ["judge_model_step_configs"]) + if step_configs: + predefined_spec["step_autorater_configs"] = step_configs + metric_payload_item["predefined_metric_spec"] = predefined_spec # Custom Code Execution Metric elif ( hasattr(metric, "remote_custom_function") and metric.remote_custom_function diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index ab6de8d7ca..56a6cad82e 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -392,6 +392,13 @@ def _EvaluateInstancesRequestParameters_to_vertex( ], ) + if getv(from_object, ["allow_cross_region_model"]) is not None: + setv( + to_object, + ["allowCrossRegionModel"], + getv(from_object, ["allow_cross_region_model"]), + ) + if getv(from_object, ["config"]) is not None: setv(to_object, ["config"], getv(from_object, ["config"])) @@ -1935,6 +1942,7 @@ def _evaluate_instances( metrics: Optional[list[types.MetricOrDict]] = None, instance: Optional[types.EvaluationInstanceOrDict] = None, metric_sources: Optional[list[types.MetricSourceOrDict]] = None, + allow_cross_region_model: Optional[bool] = None, config: Optional[types.EvaluateInstancesConfigOrDict] = None, ) -> types.EvaluateInstancesResponse: """ @@ -1956,6 +1964,7 @@ def _evaluate_instances( metrics=metrics, instance=instance, metric_sources=metric_sources, + allow_cross_region_model=allow_cross_region_model, config=config, ) @@ -3171,6 +3180,8 @@ def evaluate( - evaluation_service_qps: The rate limit (queries per second) for calls to the evaluation service. Defaults to 10. Increase this value if your project has a higher EvaluateInstances API quota. + - allow_cross_region_model: Opt-in flag to authorize cross-region + routing for judge models. **kwargs: Extra arguments to pass to evaluation, such as `agent_info`. Returns: @@ -3214,6 +3225,7 @@ def evaluate( dest=config.dest, location=location, evaluation_service_qps=getattr(config, "evaluation_service_qps", None), + allow_cross_region_model=getattr(config, "allow_cross_region_model", None), **kwargs, ) @@ -4944,6 +4956,7 @@ async def _evaluate_instances( metrics: Optional[list[types.MetricOrDict]] = None, instance: Optional[types.EvaluationInstanceOrDict] = None, metric_sources: Optional[list[types.MetricSourceOrDict]] = None, + allow_cross_region_model: Optional[bool] = None, config: Optional[types.EvaluateInstancesConfigOrDict] = None, ) -> types.EvaluateInstancesResponse: """ @@ -4965,6 +4978,7 @@ async def _evaluate_instances( metrics=metrics, instance=instance, metric_sources=metric_sources, + allow_cross_region_model=allow_cross_region_model, config=config, ) diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 4c3da8b398..15c2fce06d 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -1385,6 +1385,12 @@ class Metric(_common.BaseModel): default=None, description="""Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys.""", ) + judge_model_step_configs: Optional[dict[str, genai_types.AutoraterConfig]] = Field( + default=None, + description="""Per-step judge overrides for a predefined metric, keyed by step: + "intent_extraction", "rubric_generation" or "rubric_validation". Steps + without an entry keep their default judge.""", + ) # Allow extra fields to support metric-specific config fields. model_config = ConfigDict(extra="allow") @@ -1620,6 +1626,11 @@ class MetricDict(TypedDict, total=False): result_parsing_function: Optional[str] """Optional. A Python function string used to parse the raw output of the LLM judge model. The function must be named `parse_results` and accept a list of model response strings. It should return a dictionary with `score` (float) and `explanation` (str) keys.""" + judge_model_step_configs: Optional[dict[str, genai_types.AutoraterConfig]] + """Per-step judge overrides for a predefined metric, keyed by step: + "intent_extraction", "rubric_generation" or "rubric_validation". Steps + without an entry keep their default judge.""" + MetricOrDict = Union[Metric, MetricDict] @@ -2070,7 +2081,7 @@ class EvaluationRunConfig(_common.BaseModel): ) autorater_config: Optional[genai_types.AutoraterConfig] = Field( default=None, - description="""The autorater config for the evaluation run. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""", + description="""The autorater config for the evaluation run. Predefined metrics use it for every judge step without a `step_autorater_configs` override; multi-turn metrics ignore it.""", ) prompt_template: Optional[EvaluationRunPromptTemplate] = Field( default=None, description="""The prompt template used for inference.""" @@ -2100,7 +2111,7 @@ class EvaluationRunConfigDict(TypedDict, total=False): """The output config for the evaluation run.""" autorater_config: Optional[genai_types.AutoraterConfig] - """The autorater config for the evaluation run. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""" + """The autorater config for the evaluation run. Predefined metrics use it for every judge step without a `step_autorater_configs` override; multi-turn metrics ignore it.""" prompt_template: Optional[EvaluationRunPromptTemplateDict] """The prompt template used for inference.""" @@ -4702,7 +4713,7 @@ class _EvaluateInstancesRequestParameters(_common.BaseModel): ) autorater_config: Optional[genai_types.AutoraterConfig] = Field( default=None, - description="""Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""", + description="""Autorater config used for evaluation. Predefined metrics use it for every judge step without a `step_autorater_configs` override; multi-turn metrics ignore it.""", ) metrics: Optional[list[Metric]] = Field( default=None, @@ -4716,6 +4727,12 @@ class _EvaluateInstancesRequestParameters(_common.BaseModel): metric_sources: Optional[list[MetricSource]] = Field( default=None, description="""The metrics used for evaluation.""" ) + allow_cross_region_model: Optional[bool] = Field( + default=None, + description="""Allows judge models to be served from other regions. Required + when a judge model's fully-qualified resource name uses a different + region than the request.""", + ) config: Optional[EvaluateInstancesConfig] = Field(default=None, description="""""") @@ -4753,7 +4770,7 @@ class _EvaluateInstancesRequestParametersDict(TypedDict, total=False): """""" autorater_config: Optional[genai_types.AutoraterConfig] - """Autorater config used for evaluation. Not applicable for predefined metrics (PredefinedMetricSpec); the server uses its own model configuration for predefined metrics and this field is ignored.""" + """Autorater config used for evaluation. Predefined metrics use it for every judge step without a `step_autorater_configs` override; multi-turn metrics ignore it.""" metrics: Optional[list[MetricDict]] """The metrics used for evaluation. @@ -4766,6 +4783,11 @@ class _EvaluateInstancesRequestParametersDict(TypedDict, total=False): metric_sources: Optional[list[MetricSourceDict]] """The metrics used for evaluation.""" + allow_cross_region_model: Optional[bool] + """Allows judge models to be served from other regions. Required + when a judge model's fully-qualified resource name uses a different + region than the request.""" + config: Optional[EvaluateInstancesConfigDict] """""" @@ -30100,6 +30122,13 @@ class EvaluateMethodConfig(_common.BaseModel): evaluation service. Defaults to 10. Increase this value if your project has a higher EvaluateInstances API quota.""", ) + allow_cross_region_model: Optional[bool] = Field( + default=None, + description="""Allows judge models to be served from other regions. When set, + the service may route judge requests to another region if the model is + unavailable in the request's region. Required when a judge model's + fully-qualified resource name uses a different region.""", + ) class EvaluateMethodConfigDict(TypedDict, total=False): @@ -30121,6 +30150,12 @@ class EvaluateMethodConfigDict(TypedDict, total=False): evaluation service. Defaults to 10. Increase this value if your project has a higher EvaluateInstances API quota.""" + allow_cross_region_model: Optional[bool] + """Allows judge models to be served from other regions. When set, + the service may route judge requests to another region if the model is + unavailable in the request's region. Required when a judge model's + fully-qualified resource name uses a different region.""" + EvaluateMethodConfigOrDict = Union[EvaluateMethodConfig, EvaluateMethodConfigDict] diff --git a/tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py b/tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py index 0dbfd98d6d..f6266e6a02 100644 --- a/tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py +++ b/tests/unit/agentplatform/genai/replays/test_evaluate_predefined_metrics.py @@ -16,8 +16,43 @@ from tests.unit.agentplatform.genai.replays import pytest_helper from agentplatform import types +from google.genai import types as genai_types import pandas as pd +# Per-step judge tests are recorded against staging, whose API config already +# has the final step_autorater_configs shape. +_STAGING_BASE_URL = "https://us-central1-staging-aiplatform.sandbox.googleapis.com/" + + +def _haiku_dataset() -> types.EvaluationDataset: + return types.EvaluationDataset( + eval_dataset_df=pd.DataFrame( + { + "prompt": ["Write a haiku about the ocean. Use exactly three lines."], + "response": [ + "Waves fold into foam\nsalt wind carries gull voices\nthe" + " tide keeps its time" + ], + } + ), + candidate_name="gemini-2.5-flash", + ) + + +def _assert_metric_succeeded( + evaluation_result: types.EvaluationResult, metric_name: str +) -> None: + assert isinstance(evaluation_result, types.EvaluationResult) + summary = evaluation_result.summary_metrics[0] + assert summary.metric_name == metric_name + assert summary.num_cases_error == 0 + candidate_result = evaluation_result.eval_case_results[ + 0 + ].response_candidate_results[0] + metric_result = candidate_result.metric_results[metric_name] + assert metric_result.error_message is None + assert metric_result.rubric_verdicts + def test_evaluation_result(client): """Tests that evaluate() produces a correctly structured EvaluationResult.""" @@ -139,6 +174,54 @@ def test_predefined_metric_with_judge_model_ignores_autorater_config(client): assert evaluation_result.summary_metrics[0].metric_name == "safety_v1" +def test_predefined_metric_with_judge_model_step_configs(client): + """Tests that evaluate() sends per-step judges for a predefined metric.""" + client._api_client._http_options.base_url = _STAGING_BASE_URL + model_prefix = ( + f"projects/{client._api_client.project}/locations/" + f"{client._api_client.location}/publishers/google/models" + ) + metric = types.RubricMetric.INSTRUCTION_FOLLOWING( + judge_model_step_configs={ + "rubric_generation": genai_types.AutoraterConfig( + autorater_model=f"{model_prefix}/gemini-2.5-pro" + ), + "rubric_validation": genai_types.AutoraterConfig( + autorater_model=f"{model_prefix}/gemini-2.5-flash" + ), + } + ) + + evaluation_result = client.evals.evaluate( + dataset=_haiku_dataset(), metrics=[metric] + ) + + _assert_metric_succeeded(evaluation_result, "instruction_following_v1") + + +def test_partner_step_judge_with_allow_cross_region_model(client): + """Tests a partner step judge served from another region.""" + client._api_client._http_options.base_url = _STAGING_BASE_URL + metric = types.RubricMetric.INSTRUCTION_FOLLOWING( + judge_model_step_configs={ + "rubric_validation": genai_types.AutoraterConfig( + autorater_model=( + f"projects/{client._api_client.project}/locations/us-east5" + "/publishers/anthropic/models/claude-sonnet-4-5" + ) + ), + } + ) + + evaluation_result = client.evals.evaluate( + dataset=_haiku_dataset(), + metrics=[metric], + config={"allow_cross_region_model": True}, + ) + + _assert_metric_succeeded(evaluation_result, "instruction_following_v1") + + def test_multi_turn_predefined_metric(client): """Tests that evaluate works with multi-turn predefined metrics.""" prompts_data = { diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index f82da80765..232d5ca1c7 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -48,6 +48,7 @@ from google.genai import errors as genai_errors from google.genai import types as genai_types import pandas as pd +import pydantic import pytest _TEST_PROJECT = "test-project" @@ -10721,6 +10722,253 @@ async def test_create_evaluation_run_async_passes_allow_cross_region_model(self) ) +class TestJudgeModelStepConfigs: + """Tests for per-step judges and cross-region routing in evaluate().""" + + _STEP_CONFIGS = { + "rubric_validation": genai_types.AutoraterConfig( + autorater_model="claude-sonnet-4-5" + ) + } + _EVAL_CASE = agentplatform_genai_types.EvalCase( + prompt=genai_types.Content(parts=[genai_types.Part(text="Hello")]), + responses=[ + agentplatform_genai_types.ResponseCandidate( + response=genai_types.Content(parts=[genai_types.Part(text="Hi")]) + ) + ], + ) + _HANDLER_METRICS = [ + agentplatform_genai_types.Metric(name="final_response_quality_v1"), + agentplatform_genai_types.LLMMetric( + name="tone", prompt_template="Rate the tone of {response}." + ), + agentplatform_genai_types.Metric( + name="registered_metric", + metric_resource_name=( + "projects/123/locations/us-central1/evaluationMetrics/456" + ), + ), + ] + _HANDLER_IDS = ["predefined", "llm", "registered"] + + def setup_method(self): + _evals_metric_loaders.LazyLoadedPrebuiltMetric._cache.clear() + + def teardown_method(self): + _evals_metric_loaders.LazyLoadedPrebuiltMetric._cache.clear() + + @staticmethod + def _mock_api_client(): + api_client = mock.MagicMock() + api_client.vertexai = True + api_client.request.return_value.body = json.dumps( + {"metricResults": [{"score": 1.0}]} + ) + return api_client + + def test_t_metrics_sends_step_autorater_configs(self): + metric = agentplatform_genai_types.Metric( + name="final_response_quality_v1", + judge_model_step_configs=self._STEP_CONFIGS, + ) + + payload = _transformers.t_metrics([metric])[0] + + assert ( + payload["predefined_metric_spec"]["step_autorater_configs"] + == self._STEP_CONFIGS + ) + + def test_t_metrics_omits_step_autorater_configs_by_default(self): + metric = agentplatform_genai_types.Metric(name="final_response_quality_v1") + + payload = _transformers.t_metrics([metric])[0] + + assert "step_autorater_configs" not in payload["predefined_metric_spec"] + + def test_predefined_handler_sends_step_configs_and_cross_region_flag(self): + api_client = self._mock_api_client() + metric = agentplatform_genai_types.Metric( + name="final_response_quality_v1", + judge_model_step_configs=self._STEP_CONFIGS, + ) + handler = _evals_metric_handlers.get_handler_for_metric( + evals.Evals(api_client_=api_client), + metric, + allow_cross_region_model=True, + ) + + result = handler.get_metric_result(self._EVAL_CASE, response_index=0) + + assert result.score == 1.0 + request_body = api_client.request.call_args[0][2] + assert request_body["allowCrossRegionModel"] is True + assert request_body["metrics"][0]["predefined_metric_spec"][ + "step_autorater_configs" + ] == {"rubric_validation": {"autorater_model": "claude-sonnet-4-5"}} + + @pytest.mark.parametrize("metric", _HANDLER_METRICS, ids=_HANDLER_IDS) + def test_handlers_send_cross_region_flag(self, metric): + api_client = self._mock_api_client() + handler = _evals_metric_handlers.get_handler_for_metric( + evals.Evals(api_client_=api_client), + metric, + allow_cross_region_model=True, + ) + + result = handler.get_metric_result(self._EVAL_CASE, response_index=0) + + assert result.score == 1.0 + assert api_client.request.call_args[0][2]["allowCrossRegionModel"] is True + + @pytest.mark.parametrize("metric", _HANDLER_METRICS, ids=_HANDLER_IDS) + def test_handlers_omit_cross_region_flag_by_default(self, metric): + api_client = self._mock_api_client() + handler = _evals_metric_handlers.get_handler_for_metric( + evals.Evals(api_client_=api_client), metric + ) + + result = handler.get_metric_result(self._EVAL_CASE, response_index=0) + + assert result.score == 1.0 + assert "allowCrossRegionModel" not in api_client.request.call_args[0][2] + + @pytest.mark.usefixtures("mock_eval_dependencies") + def test_execute_evaluation_passes_cross_region_flag(self, mock_api_client_fixture): + with mock.patch.object( + _evals_metric_handlers, + "compute_metrics_and_aggregate", + side_effect=RuntimeError("stop"), + ) as mock_compute: + with pytest.raises(RuntimeError): + _evals_common._execute_evaluation( + api_client=mock_api_client_fixture, + dataset=agentplatform_genai_types.EvaluationDataset( + eval_dataset_df=pd.DataFrame( + [{"prompt": "p1", "response": "r1"}] + ) + ), + metrics=[agentplatform_genai_types.Metric(name="exact_match")], + allow_cross_region_model=True, + ) + + assert mock_compute.call_args.kwargs["allow_cross_region_model"] is True + + def test_compute_metrics_passes_cross_region_flag_to_handlers(self): + run_config = _evals_metric_handlers.EvaluationRunConfig( + evals_module=evals.Evals(api_client_=mock.MagicMock()), + dataset=agentplatform_genai_types.EvaluationDataset(eval_cases=[]), + metrics=[ + agentplatform_genai_types.Metric(name="final_response_quality_v1") + ], + num_response_candidates=1, + ) + + with mock.patch.object( + _evals_metric_handlers, + "get_handler_for_metric", + side_effect=RuntimeError("stop"), + ) as mock_get_handler: + with pytest.raises(RuntimeError): + _evals_metric_handlers.compute_metrics_and_aggregate( + run_config, allow_cross_region_model=True + ) + + assert mock_get_handler.call_args.kwargs["allow_cross_region_model"] is True + + @mock.patch.object(_evals_common, "_execute_evaluation") + def test_evaluate_passes_allow_cross_region_model(self, mock_execute_evaluation): + evals.Evals(api_client_=mock.MagicMock()).evaluate( + dataset=agentplatform_genai_types.EvaluationDataset( + eval_dataset_df=pd.DataFrame([{"prompt": "p1", "response": "r1"}]) + ), + metrics=[ + agentplatform_genai_types.Metric(name="final_response_quality_v1") + ], + config={"allow_cross_region_model": True}, + ) + + _, kwargs = mock_execute_evaluation.call_args + assert kwargs["allow_cross_region_model"] is True + + @mock.patch.object(_evals_metric_handlers.logger, "warning") + def test_judge_model_warning_only_for_multi_turn_metrics(self, mock_warning): + module = evals.Evals(api_client_=mock.MagicMock()) + + _evals_metric_handlers.PredefinedMetricHandler( + module=module, + metric=agentplatform_genai_types.Metric( + name="final_response_quality_v1", judge_model="gemini-2.5-pro" + ), + ) + mock_warning.assert_not_called() + + _evals_metric_handlers.PredefinedMetricHandler( + module=module, + metric=agentplatform_genai_types.Metric( + name="multi_turn_task_success_v1", judge_model="gemini-2.5-pro" + ), + ) + mock_warning.assert_called_once() + + def test_resolve_evaluation_run_metrics_sends_metric_spec_parameters(self): + metric = _evals_metric_loaders.RubricMetric.FINAL_RESPONSE_QUALITY( + metric_spec_parameters={"guidelines": "Be concise."} + ) + + resolved = _evals_common._resolve_evaluation_run_metrics( + [metric], api_client=mock.MagicMock() + ) + + predefined_spec = resolved[0].metric_config.predefined_metric_spec + assert predefined_spec.metric_spec_name == "final_response_quality_v1" + assert predefined_spec.metric_spec_parameters == {"guidelines": "Be concise."} + + @pytest.mark.parametrize( + "metric", + [ + _evals_metric_loaders.RubricMetric.FINAL_RESPONSE_QUALITY( + judge_model_step_configs=_STEP_CONFIGS + ), + agentplatform_genai_types.Metric( + name="final_response_quality_v1", + judge_model_step_configs=_STEP_CONFIGS, + ), + ], + ids=["rubric_metric", "metric"], + ) + def test_resolve_evaluation_run_metrics_step_configs(self, metric): + # google-genai gets the field once it is public in the discovery doc. + if "step_autorater_configs" in genai_types.PredefinedMetricSpec.model_fields: + resolved = _evals_common._resolve_evaluation_run_metrics( + [metric], api_client=mock.MagicMock() + ) + predefined_spec = resolved[0].metric_config.predefined_metric_spec + assert predefined_spec.step_autorater_configs == self._STEP_CONFIGS + else: + with pytest.raises( + pydantic.ValidationError, match="step_autorater_configs" + ): + _evals_common._resolve_evaluation_run_metrics( + [metric], api_client=mock.MagicMock() + ) + + def test_rubric_metric_overrides_bypass_shared_cache(self): + api_client = mock.MagicMock() + rubric_metric = _evals_metric_loaders.RubricMetric + + default_metric = rubric_metric.FINAL_RESPONSE_QUALITY.resolve(api_client) + overridden_metric = rubric_metric.FINAL_RESPONSE_QUALITY( + judge_model_step_configs=self._STEP_CONFIGS + ).resolve(api_client) + default_again = rubric_metric.FINAL_RESPONSE_QUALITY.resolve(api_client) + + assert default_metric.judge_model_step_configs is None + assert overridden_metric.judge_model_step_configs == self._STEP_CONFIGS + assert default_again.judge_model_step_configs is None + + _TEST_INTERACTION = ( "projects/test-project/locations/us-central1/interactions/test-interaction" )