Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions sagemaker-core/src/sagemaker/core/modules/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ class Compute(shapes.ResourceConfig):
def _model_validator(self) -> "Compute":
"""Convert Unassigned values to None and validate instance_preferences."""
converted = convert_unassigned_to_none(self)
# Nested preferences keep Unassigned() on unset fields, which model_dump
# flags per element; normalise them the same way as the top level.
for preference in converted.instance_preferences or ():
convert_unassigned_to_none(preference)
validate_instance_preferences(converted)
return converted

Expand Down
4 changes: 4 additions & 0 deletions sagemaker-core/src/sagemaker/core/training/configs.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,10 @@ class Compute(shapes.ResourceConfig):
def _model_validator(self) -> "Compute":
"""Convert Unassigned values to None and validate instance_preferences."""
converted = convert_unassigned_to_none(self)
# Nested preferences keep Unassigned() on unset fields, which model_dump
# flags per element; normalise them the same way as the top level.
for preference in converted.instance_preferences or ():
convert_unassigned_to_none(preference)
validate_instance_preferences(converted)
return converted

Expand Down
26 changes: 26 additions & 0 deletions sagemaker-core/tests/unit/test_compute_configs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
"""Unit tests for Compute and HyperPodCompute config classes."""

import warnings

import pytest
from sagemaker.core.modules.configs import Compute as ModulesCompute
from sagemaker.core.shapes import InstancePreference
from sagemaker.core.training.configs import Compute, HyperPodCompute


Expand Down Expand Up @@ -337,3 +341,25 @@ def test_processing_cluster_config_single_type_still_works(self):
)
assert pcc.instance_type == "ml.m5.xlarge"
assert pcc.instance_count == 1


class TestInstancePreferencesSerialization:
"""Unset fields on nested preferences must not surface as pydantic
serializer warnings on every submit, and the request payload must omit them."""

@pytest.mark.parametrize("compute_cls", [Compute, ModulesCompute])
def test_to_resource_config_emits_no_serializer_warning(self, compute_cls):
compute = compute_cls(
instance_preferences=[
InstancePreference(instance_type="ml.m5.large"),
InstancePreference(instance_type="ml.m5.xlarge"),
],
instance_count=1,
)
with warnings.catch_warnings():
warnings.simplefilter("error")
resource_config = compute._to_resource_config()
prefs = resource_config.instance_preferences
assert [p.instance_type for p in prefs] == ["ml.m5.large", "ml.m5.xlarge"]
assert all(p.training_plan_arns is None and p.instance_count is None for p in prefs)
assert resource_config.instance_count == 1
9 changes: 9 additions & 0 deletions sagemaker-train/src/sagemaker/train/base_trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -901,6 +901,15 @@ def _train_serverful_smtj(self, training_dataset=None, validation_dataset=None,
role = self.role

compute = self.compute
# Recipes are rendered for one instance type (device class, image,
# launcher), so a service-chosen type cannot apply. Fail before the
# recipe fetch, and before the type-enum check reports a confusing
# "Instance type 'None' is not supported".
if getattr(compute, "instance_preferences", None):
raise ValueError(
"Training recipes do not support ``instance_preferences``. "
"Set a single ``instance_type`` in Compute when using a recipe-based trainer."
)
customization_technique = self._customization_technique

# Resolve the recipe S3 URI from hub metadata
Expand Down
3 changes: 3 additions & 0 deletions sagemaker-train/tests/unit/train/test_base_trainer_compute.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ def test_compute_attributes_forwarded(self):
trainer = _ConcreteTrainer()
trainer.compute = MagicMock(
instance_type="ml.p4d.24xlarge",
instance_preferences=None,
instance_count=4,
volume_size_in_gb=300,
keep_alive_period_in_seconds=1200,
Expand All @@ -131,6 +132,7 @@ def test_training_plan_arn_forwarded(self):
trainer = _ConcreteTrainer()
trainer.compute = MagicMock(
instance_type="ml.p5.48xlarge",
instance_preferences=None,
instance_count=2,
volume_size_in_gb=500,
keep_alive_period_in_seconds=0,
Expand All @@ -147,6 +149,7 @@ def test_training_plan_arn_none_when_not_set(self):
trainer = _ConcreteTrainer()
trainer.compute = MagicMock(
instance_type="ml.p4d.24xlarge",
instance_preferences=None,
instance_count=1,
volume_size_in_gb=30,
keep_alive_period_in_seconds=0,
Expand Down
34 changes: 34 additions & 0 deletions sagemaker-train/tests/unit/train/test_base_trainer_serverful.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def __init__(self, **kwargs):
self.compute = MagicMock(
instance_type="ml.p4d.24xlarge",
instance_count=1,
instance_preferences=None,
volume_size_in_gb=100,
keep_alive_period_in_seconds=None,
training_plan_arn=None,
Expand Down Expand Up @@ -388,3 +389,36 @@ def test_skips_validation_when_enum_unavailable(self, mock_enum):

# Any instance type is accepted; the method returns None to signal skip.
assert trainer._validate_instance_type("ml.g5.xlarge", MagicMock()) is None


class TestInstancePreferencesRejected:
"""Recipes are rendered for one instance type, so a preference list must be
refused up front -- before the recipe fetch, and before the instance-type
enum check can report a misleading "Instance type 'None' is not supported"."""

@pytest.fixture
def trainer(self):
trainer = _ConcreteTrainer()
trainer.compute = MagicMock(
instance_type=None,
instance_count=2,
instance_preferences=[MagicMock(instance_type="ml.p5.48xlarge")],
)
return trainer

def test_rejected_before_any_recipe_or_network_access(self, trainer):
with patch("sagemaker.train.base_trainer.get_recipe_s3_uri") as fetch, pytest.raises(
ValueError, match="Training recipes do not support ``instance_preferences``"
):
trainer._train_serverful_smtj(training_dataset="s3://bucket/train.jsonl")
fetch.assert_not_called()

def test_message_names_the_fix(self, trainer):
with pytest.raises(ValueError, match="Set a single ``instance_type`` in Compute"):
trainer._train_serverful_smtj(training_dataset="s3://bucket/train.jsonl")

def test_single_instance_type_is_unaffected(self):
trainer = _ConcreteTrainer()
_, kwargs = _run_serverful(trainer)
assert kwargs["compute"].instance_type == "ml.p4d.24xlarge"
assert not kwargs["compute"].instance_preferences
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def __init__(self, **kwargs):
self.validation_dataset = None
self.compute = MagicMock(
instance_type="ml.p5.48xlarge",
instance_preferences=None,
instance_count=4,
volume_size_in_gb=100,
keep_alive_period_in_seconds=None,
Expand Down
Loading