From 6c26e25bd4cd4b177141e5605517a8245d426ded Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Fri, 7 Aug 2026 08:36:10 -0700 Subject: [PATCH] feat: GenAI Client(evals) - Pin SDK predefined metrics to latest Gemini 3.5 Flash versions by default PiperOrigin-RevId: 960946248 --- agentplatform/_genai/_evals_constant.py | 40 +++++++++++++++++++ agentplatform/_genai/_evals_metric_loaders.py | 7 +++- vertexai/_genai/_evals_constant.py | 40 +++++++++++++++++++ vertexai/_genai/_evals_metric_loaders.py | 7 +++- 4 files changed, 90 insertions(+), 4 deletions(-) diff --git a/agentplatform/_genai/_evals_constant.py b/agentplatform/_genai/_evals_constant.py index 321150310d..8609c05748 100644 --- a/agentplatform/_genai/_evals_constant.py +++ b/agentplatform/_genai/_evals_constant.py @@ -16,6 +16,7 @@ SUPPORTED_PREDEFINED_METRICS = frozenset( { + # v1 metrics (kept for backward compatibility). "general_quality_v1", "text_quality_v1", "instruction_following_v1", @@ -33,9 +34,48 @@ "tool_use_quality_v1", "gecko_text2image_v1", "gecko_text2video_v1", + # v2/v3 metrics backed by Gemini 3.5 Flash. + "general_quality_v2", + "instruction_following_v2", + "text_quality_v2", + "grounding_v2", + "hallucination_v2", + "safety_v3", + "multi_turn_general_quality_v2", + "multi_turn_text_quality_v2", + "tool_use_quality_v2", + "final_response_quality_v2", + "final_response_match_v3", + "final_response_reference_free_v2", + "gecko_text2image_v2", + "gecko_text2video_v2", } ) +# Maps the SDK-facing base metric name to the latest API spec name. +# When a user creates a metric without specifying a version, this resolves to +# the latest available version (backed by Gemini 3.5 Flash). +# Note: v2/v3 specs require Gemini 3.5 Flash, which is not available in all +# regions (e.g. us-central1). To use v1 explicitly for full regional +# availability: +# types.RubricMetric.GENERAL_QUALITY(version="v1") +METRIC_LATEST_SPEC_NAME: dict[str, str] = { + "general_quality": "general_quality_v2", + "instruction_following": "instruction_following_v2", + "text_quality": "text_quality_v2", + "grounding": "grounding_v2", + "hallucination": "hallucination_v2", + "safety": "safety_v3", + "multi_turn_general_quality": "multi_turn_general_quality_v2", + "multi_turn_text_quality": "multi_turn_text_quality_v2", + "tool_use_quality": "tool_use_quality_v2", + "final_response_quality": "final_response_quality_v2", + "final_response_match": "final_response_match_v3", + "final_response_reference_free": "final_response_reference_free_v2", + "gecko_text2image": "gecko_text2image_v2", + "gecko_text2video": "gecko_text2video_v2", +} + SUPPORTED_VERTEX_MAAS_MODEL_PREFIXES = frozenset( { "meta/", # Meta/Llama diff --git a/agentplatform/_genai/_evals_metric_loaders.py b/agentplatform/_genai/_evals_metric_loaders.py index e64b151f0d..82df89278f 100644 --- a/agentplatform/_genai/_evals_metric_loaders.py +++ b/agentplatform/_genai/_evals_metric_loaders.py @@ -54,7 +54,7 @@ def _get_api_metric_spec_name(self) -> Optional[str]: """Constructs the metric_spec_name for API Predefined Metrics.""" base_name = self.name.lower() if self.version: - # Explicit version provided + # Explicit version provided. version = self.version.lower() potential_name = f"{base_name}_{version}" return ( @@ -63,7 +63,10 @@ def _get_api_metric_spec_name(self) -> Optional[str]: else None ) else: - # Default versioning: Try _v1, then base name + # No version specified: resolve to the latest available version, + # falling back to _v1, then the bare base name. + if base_name in _evals_constant.METRIC_LATEST_SPEC_NAME: + return _evals_constant.METRIC_LATEST_SPEC_NAME[base_name] v1_name = f"{base_name}_v1" if v1_name in _evals_constant.SUPPORTED_PREDEFINED_METRICS: return v1_name diff --git a/vertexai/_genai/_evals_constant.py b/vertexai/_genai/_evals_constant.py index 822f8e685a..39d95a50a0 100644 --- a/vertexai/_genai/_evals_constant.py +++ b/vertexai/_genai/_evals_constant.py @@ -16,6 +16,7 @@ SUPPORTED_PREDEFINED_METRICS = frozenset( { + # v1 metrics (kept for backward compatibility). "general_quality_v1", "text_quality_v1", "instruction_following_v1", @@ -33,9 +34,48 @@ "tool_use_quality_v1", "gecko_text2image_v1", "gecko_text2video_v1", + # v2/v3 metrics backed by Gemini 3.5 Flash. + "general_quality_v2", + "instruction_following_v2", + "text_quality_v2", + "grounding_v2", + "hallucination_v2", + "safety_v3", + "multi_turn_general_quality_v2", + "multi_turn_text_quality_v2", + "tool_use_quality_v2", + "final_response_quality_v2", + "final_response_match_v3", + "final_response_reference_free_v2", + "gecko_text2image_v2", + "gecko_text2video_v2", } ) +# Maps the SDK-facing base metric name to the latest API spec name. +# When a user creates a metric without specifying a version, this resolves to +# the latest available version (backed by Gemini 3.5 Flash). +# Note: v2/v3 specs require Gemini 3.5 Flash, which is not available in all +# regions (e.g. us-central1). To use v1 explicitly for full regional +# availability: +# types.RubricMetric.GENERAL_QUALITY(version="v1") +METRIC_LATEST_SPEC_NAME: dict[str, str] = { + "general_quality": "general_quality_v2", + "instruction_following": "instruction_following_v2", + "text_quality": "text_quality_v2", + "grounding": "grounding_v2", + "hallucination": "hallucination_v2", + "safety": "safety_v3", + "multi_turn_general_quality": "multi_turn_general_quality_v2", + "multi_turn_text_quality": "multi_turn_text_quality_v2", + "tool_use_quality": "tool_use_quality_v2", + "final_response_quality": "final_response_quality_v2", + "final_response_match": "final_response_match_v3", + "final_response_reference_free": "final_response_reference_free_v2", + "gecko_text2image": "gecko_text2image_v2", + "gecko_text2video": "gecko_text2video_v2", +} + SUPPORTED_VERTEX_MAAS_MODEL_PREFIXES = frozenset( { "meta/", # Meta/Llama diff --git a/vertexai/_genai/_evals_metric_loaders.py b/vertexai/_genai/_evals_metric_loaders.py index a25e3fde43..72f2da620d 100644 --- a/vertexai/_genai/_evals_metric_loaders.py +++ b/vertexai/_genai/_evals_metric_loaders.py @@ -54,7 +54,7 @@ def _get_api_metric_spec_name(self) -> Optional[str]: """Constructs the metric_spec_name for API Predefined Metrics.""" base_name = self.name.lower() if self.version: - # Explicit version provided + # Explicit version provided. version = self.version.lower() potential_name = f"{base_name}_{version}" return ( @@ -63,7 +63,10 @@ def _get_api_metric_spec_name(self) -> Optional[str]: else None ) else: - # Default versioning: Try _v1, then base name + # No version specified: resolve to the latest available version, + # falling back to _v1, then the bare base name. + if base_name in _evals_constant.METRIC_LATEST_SPEC_NAME: + return _evals_constant.METRIC_LATEST_SPEC_NAME[base_name] v1_name = f"{base_name}_v1" if v1_name in _evals_constant.SUPPORTED_PREDEFINED_METRICS: return v1_name