From 7535905ee3141a63942c743ae978cf3020c06e67 Mon Sep 17 00:00:00 2001 From: alanhuangyoo Date: Fri, 28 Aug 2026 21:55:21 +0800 Subject: [PATCH 1/2] Read rope_theta from rope_parameters across Inference V2 transformers 5.0 folded the rotary settings into config.rope_parameters and dropped the rope_theta attribute. Eight V2 models still read the attribute: llama_v2, mistral, mixtral, phi, phi3, qwen_v2, qwen_v2_moe self._config.rope_theta -> AttributeError on 5.x exaone4 getattr(self._config, "rope_theta", 1000000.0) -> no error, but 1e6 instead of the 1e4 the config carries exaone4 is the worse of the two: a 100x rotary base is a silent numerical error, not a startup failure. exaone4_5 already reads both spellings, added with the model in #8121. Hoist the same lookup onto DSTransformerModelBase so every model that reaches it through the base gets it, and raise instead of guessing a base when neither spelling carries one. Same drift as #8341, which covers the v1 kernel-injection policy. Signed-off-by: alanhuangyoo --- .../v2/model_implementations/exaone4/model.py | 3 +- .../inference_transformer_base.py | 20 +++++ .../model_implementations/llama_v2/model.py | 2 +- .../v2/model_implementations/mistral/model.py | 2 +- .../v2/model_implementations/mixtral/model.py | 2 +- .../v2/model_implementations/phi/model.py | 2 +- .../v2/model_implementations/phi3/model.py | 2 +- .../v2/model_implementations/qwen_v2/model.py | 2 +- .../qwen_v2_moe/model.py | 2 +- .../model_implementations/test_rope_theta.py | 80 +++++++++++++++++++ 10 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 tests/unit/inference/v2/model_implementations/test_rope_theta.py diff --git a/deepspeed/inference/v2/model_implementations/exaone4/model.py b/deepspeed/inference/v2/model_implementations/exaone4/model.py index 5aeb87c2cdac..e12f5a513072 100644 --- a/deepspeed/inference/v2/model_implementations/exaone4/model.py +++ b/deepspeed/inference/v2/model_implementations/exaone4/model.py @@ -94,8 +94,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - rope_theta = getattr(self._config, "rope_theta", 1000000.0) - return RotateHalfConfig(theta_base=rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/deepspeed/inference/v2/model_implementations/inference_transformer_base.py b/deepspeed/inference/v2/model_implementations/inference_transformer_base.py index fae67dc8fc2a..0aabfda2e21c 100644 --- a/deepspeed/inference/v2/model_implementations/inference_transformer_base.py +++ b/deepspeed/inference/v2/model_implementations/inference_transformer_base.py @@ -165,6 +165,26 @@ def positional_embedding_config(self) -> Optional[RotateHalfConfig]: Derived helpers """ + @property + def rope_theta(self) -> float: + """The rotary base, read from wherever the installed transformers keeps it. + + transformers 5.0 folded the rotary settings into ``config.rope_parameters`` and + dropped the ``rope_theta`` attribute, so reading the attribute alone raises + against a stock config on 5.x. ``exaone4_5`` already reads both spellings; this + is the same lookup for every model that reaches it through this base. + """ + theta = getattr(self._config, "rope_theta", None) + if theta is not None: + return theta + + rope_parameters = getattr(self._config, "rope_parameters", None) or getattr(self._config, "rope_scaling", None) + if isinstance(rope_parameters, dict) and rope_parameters.get("rope_theta") is not None: + return rope_parameters["rope_theta"] + + raise ValueError(f"{type(self._config).__name__} carries no rope_theta, either as an " + "attribute or in rope_parameters/rope_scaling.") + @cached_property def n_heads_q_local(self) -> int: """ diff --git a/deepspeed/inference/v2/model_implementations/llama_v2/model.py b/deepspeed/inference/v2/model_implementations/llama_v2/model.py index a0c81f4d749e..ac23145e942e 100644 --- a/deepspeed/inference/v2/model_implementations/llama_v2/model.py +++ b/deepspeed/inference/v2/model_implementations/llama_v2/model.py @@ -107,7 +107,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Forward implementations diff --git a/deepspeed/inference/v2/model_implementations/mistral/model.py b/deepspeed/inference/v2/model_implementations/mistral/model.py index 318d362f1a64..89289a9f171a 100644 --- a/deepspeed/inference/v2/model_implementations/mistral/model.py +++ b/deepspeed/inference/v2/model_implementations/mistral/model.py @@ -106,7 +106,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Forward implementations diff --git a/deepspeed/inference/v2/model_implementations/mixtral/model.py b/deepspeed/inference/v2/model_implementations/mixtral/model.py index 878cd8e31cec..5a512fc0633a 100644 --- a/deepspeed/inference/v2/model_implementations/mixtral/model.py +++ b/deepspeed/inference/v2/model_implementations/mixtral/model.py @@ -114,7 +114,7 @@ def positional_embedding_config(self) -> Optional[RotateHalfConfig]: """ The positional embedding configuration for the model. """ - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Inherited from `DSMoETransformerModelBase` diff --git a/deepspeed/inference/v2/model_implementations/phi/model.py b/deepspeed/inference/v2/model_implementations/phi/model.py index 2d5826810cb5..41e0e8652d60 100644 --- a/deepspeed/inference/v2/model_implementations/phi/model.py +++ b/deepspeed/inference/v2/model_implementations/phi/model.py @@ -97,7 +97,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: rotary_dim = int(self._config.partial_rotary_factor * self.head_size) - return RotateHalfConfig(rotate_dim=rotary_dim, theta_base=self._config.rope_theta) + return RotateHalfConfig(rotate_dim=rotary_dim, theta_base=self.rope_theta) """ Forward implementations diff --git a/deepspeed/inference/v2/model_implementations/phi3/model.py b/deepspeed/inference/v2/model_implementations/phi3/model.py index 507bb4fc9af1..7d974d3a113e 100644 --- a/deepspeed/inference/v2/model_implementations/phi3/model.py +++ b/deepspeed/inference/v2/model_implementations/phi3/model.py @@ -106,7 +106,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Forward implementations diff --git a/deepspeed/inference/v2/model_implementations/qwen_v2/model.py b/deepspeed/inference/v2/model_implementations/qwen_v2/model.py index d535462a954d..c89ed8a65c65 100644 --- a/deepspeed/inference/v2/model_implementations/qwen_v2/model.py +++ b/deepspeed/inference/v2/model_implementations/qwen_v2/model.py @@ -100,7 +100,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) def make_norm_layer(self) -> None: """ diff --git a/deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py b/deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py index c7841b24e5fc..bc441452ce70 100644 --- a/deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py +++ b/deepspeed/inference/v2/model_implementations/qwen_v2_moe/model.py @@ -105,7 +105,7 @@ def positional_embedding_type(self) -> PositionalEmbeddingType: @property def positional_embedding_config(self) -> Optional[RotateHalfConfig]: - return RotateHalfConfig(theta_base=self._config.rope_theta) + return RotateHalfConfig(theta_base=self.rope_theta) """ Inherited from `DSMoETransformerModelBase` diff --git a/tests/unit/inference/v2/model_implementations/test_rope_theta.py b/tests/unit/inference/v2/model_implementations/test_rope_theta.py new file mode 100644 index 000000000000..2fda33c2b5f7 --- /dev/null +++ b/tests/unit/inference/v2/model_implementations/test_rope_theta.py @@ -0,0 +1,80 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team +"""Inference V2 must find rope_theta wherever the installed transformers keeps it. + +transformers 5.0 folded the rotary settings into ``config.rope_parameters`` and dropped +the ``rope_theta`` attribute. Models that read the attribute directly raise against a +stock config on 5.x; ``exaone4`` read it through a ``getattr`` default and silently used +that default instead of the value the config actually carries. +""" + +from types import SimpleNamespace + +import pytest + +from deepspeed.inference.v2.model_implementations.inference_transformer_base import DSTransformerModelBase + + +class _Model: + """Minimal stand-in that borrows the property under test.""" + + rope_theta = DSTransformerModelBase.rope_theta + + def __init__(self, config): + self._config = config + + +def test_reads_the_legacy_attribute(): + assert _Model(SimpleNamespace(rope_theta=500000.0)).rope_theta == 500000.0 + + +def test_reads_rope_parameters_when_the_attribute_is_gone(): + config = SimpleNamespace(rope_parameters={"rope_theta": 10000.0, "rope_type": "default"}) + + assert _Model(config).rope_theta == 10000.0 + + +def test_reads_rope_scaling_when_that_is_the_only_dict(): + config = SimpleNamespace(rope_scaling={"rope_theta": 1000000.0, "rope_type": "default"}) + + assert _Model(config).rope_theta == 1000000.0 + + +def test_prefers_the_attribute_when_both_are_present(): + config = SimpleNamespace(rope_theta=500000.0, rope_parameters={"rope_theta": 10000.0}) + + assert _Model(config).rope_theta == 500000.0 + + +def test_raises_instead_of_guessing_when_nothing_carries_it(): + # exaone4 used to fall back to 1e6 here, which is a silent 100x error against a + # config whose real base is 1e4. + with pytest.raises(ValueError, match="rope_theta"): + _ = _Model(SimpleNamespace()).rope_theta + + +@pytest.mark.parametrize( + "module_name, config_name", + [ + ("llama", "LlamaConfig"), + ("mistral", "MistralConfig"), + ("mixtral", "MixtralConfig"), + ("phi", "PhiConfig"), + ("phi3", "Phi3Config"), + ("qwen2", "Qwen2Config"), + ("qwen2_moe", "Qwen2MoeConfig"), + ("exaone4", "Exaone4Config"), + ], +) +def test_resolves_against_the_installed_transformers_configs(module_name, config_name): + """Every config backing a V2 model must yield a base through one spelling or the other.""" + importlib = pytest.importorskip("importlib") + try: + module = importlib.import_module(f"transformers.models.{module_name}.configuration_{module_name}") + config = getattr(module, config_name)() + except (ImportError, AttributeError): + pytest.skip(f"{config_name} is not available in the installed transformers") + + assert _Model(config).rope_theta > 0 From 92ab1aff1952fd03c530062b3ee48c2dfbf3d0f4 Mon Sep 17 00:00:00 2001 From: alanhuangyoo Date: Fri, 4 Sep 2026 12:19:41 +0800 Subject: [PATCH 2/2] Resolve rope_theta from per-layer-type rope_parameters A config that sets RoPE per layer type nests the settings one level deeper, keyed by the layer type, and standardize_rope_params leaves the class default at the top level of the same dict: rope_parameters: {'sliding_attention': {'rope_theta': 1000000.0, ...}, 'full_attention': {'rope_theta': 16000000.0, ...}, 'rope_theta': 10000.0, 'rope_type': 'default'} Reading the top level returns 10000.0, which is the class default rather than anything the checkpoint asked for, so the property resolved to a wrong base quietly instead of raising. The nested entries now win. When the layer types disagree the config is refused rather than resolved to one of them. Every caller of this property feeds a single RotateHalfConfig.theta_base for the whole model, so there is no shape in which picking either base is right for the layers using the other one. Released EXAONE-4 configs carry a flat rope_parameters and are unaffected. Signed-off-by: alanhuangyoo --- .../inference_transformer_base.py | 29 +++++++++-- .../model_implementations/test_rope_theta.py | 48 +++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/deepspeed/inference/v2/model_implementations/inference_transformer_base.py b/deepspeed/inference/v2/model_implementations/inference_transformer_base.py index 0aabfda2e21c..270fa57e887d 100644 --- a/deepspeed/inference/v2/model_implementations/inference_transformer_base.py +++ b/deepspeed/inference/v2/model_implementations/inference_transformer_base.py @@ -171,16 +171,37 @@ def rope_theta(self) -> float: transformers 5.0 folded the rotary settings into ``config.rope_parameters`` and dropped the ``rope_theta`` attribute, so reading the attribute alone raises - against a stock config on 5.x. ``exaone4_5`` already reads both spellings; this - is the same lookup for every model that reaches it through this base. + against a stock config on 5.x. + + A config that sets RoPE per layer type gets nested one level deeper, keyed by + the layer type, and ``standardize_rope_params`` leaves the class default at the + top level of the same dict. Reading the top level there returns that default + rather than anything the checkpoint asked for, so the nested entries win. Every + caller of this property feeds a single ``RotateHalfConfig.theta_base`` for the + whole model, so distinct per-layer bases cannot be represented and are refused + rather than silently resolved to one of them. """ theta = getattr(self._config, "rope_theta", None) if theta is not None: return theta rope_parameters = getattr(self._config, "rope_parameters", None) or getattr(self._config, "rope_scaling", None) - if isinstance(rope_parameters, dict) and rope_parameters.get("rope_theta") is not None: - return rope_parameters["rope_theta"] + if isinstance(rope_parameters, dict): + per_layer = { + layer_type: parameters["rope_theta"] + for layer_type, parameters in rope_parameters.items() + if isinstance(parameters, dict) and parameters.get("rope_theta") is not None + } + if per_layer: + distinct = set(per_layer.values()) + if len(distinct) > 1: + raise ValueError(f"{type(self._config).__name__} sets a different rope_theta per " + f"layer type ({per_layer}); Inference V2 applies one rotary base " + "to every layer and cannot represent this config.") + return distinct.pop() + + if rope_parameters.get("rope_theta") is not None: + return rope_parameters["rope_theta"] raise ValueError(f"{type(self._config).__name__} carries no rope_theta, either as an " "attribute or in rope_parameters/rope_scaling.") diff --git a/tests/unit/inference/v2/model_implementations/test_rope_theta.py b/tests/unit/inference/v2/model_implementations/test_rope_theta.py index 2fda33c2b5f7..e02355e97a97 100644 --- a/tests/unit/inference/v2/model_implementations/test_rope_theta.py +++ b/tests/unit/inference/v2/model_implementations/test_rope_theta.py @@ -55,6 +55,54 @@ def test_raises_instead_of_guessing_when_nothing_carries_it(): _ = _Model(SimpleNamespace()).rope_theta +def test_unwraps_a_per_layer_type_dict(): + """A config that sets RoPE per layer type nests one level deeper. + + ``standardize_rope_params`` leaves the class default at the top level of the same + dict, so reading the top level returns 10000.0 rather than the 1000000.0 the + checkpoint asked for. + """ + config = SimpleNamespace( + rope_parameters={ + "sliding_attention": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "full_attention": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "rope_theta": 10000.0, + "rope_type": "default", + }) + + assert _Model(config).rope_theta == 1000000.0 + + +def test_refuses_a_config_whose_layer_types_disagree(): + """Callers feed one ``RotateHalfConfig.theta_base`` for the whole model. + + Picking either base would be wrong for the layers using the other one, so this is + refused rather than resolved. + """ + config = SimpleNamespace( + rope_parameters={ + "sliding_attention": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "full_attention": { + "rope_theta": 16000000.0, + "rope_type": "default" + }, + "rope_theta": 10000.0, + "rope_type": "default", + }) + + with pytest.raises(ValueError, match="different rope_theta per layer type"): + _ = _Model(config).rope_theta + + @pytest.mark.parametrize( "module_name, config_name", [