From 8c08ca6fb884d12ccc4ebd21883ce77f085eb78c Mon Sep 17 00:00:00 2001 From: mayang02 Date: Tue, 25 Aug 2026 09:22:23 +0800 Subject: [PATCH 1/2] [XPU] Fix PaddleFormers fallback model issues - Fix TypeError when calling the PaddleFormers fallback model - Do not rotate twice when the model already applied RoPE - Fix position_ids for the adjust_batch token layout --- .../layers/backends/xpu/attention.py | 15 ++++++ .../models/paddleformers/__init__.py | 12 +++++ .../models/paddleformers/base.py | 41 +++++++++++++- .../model_executor/test_paddleformers_base.py | 54 +++++++++++++++++++ 4 files changed, 120 insertions(+), 2 deletions(-) diff --git a/fastdeploy/model_executor/layers/backends/xpu/attention.py b/fastdeploy/model_executor/layers/backends/xpu/attention.py index dbc41d6dd19..8d349148e2e 100644 --- a/fastdeploy/model_executor/layers/backends/xpu/attention.py +++ b/fastdeploy/model_executor/layers/backends/xpu/attention.py @@ -169,6 +169,21 @@ def forward_mixed( forward_mixed """ metadata = self.attention_metadata + # In the PaddleFormers fallback path the model already applied RoPE + # (forward_meta.rope_already_applied=True), so feed identity RoPE (cos=1, sin=0) + # to block_attn to avoid rotating twice. Aligned with append_attn_backend. + if getattr(forward_meta, "rope_already_applied", False) and metadata.rotary_embs is not None: + need_recreate = ( + not hasattr(self, "_identity_rotary_embs") + or self._identity_rotary_embs is None + or self._identity_rotary_embs.shape != metadata.rotary_embs.shape + ) + if need_recreate: + identity = paddle.zeros_like(metadata.rotary_embs) + identity[0] = 1.0 + identity[1] = 0.0 + self._identity_rotary_embs = identity + metadata.rotary_embs = self._identity_rotary_embs if self.pd_disaggregation_mode == "per_query": metadata.kv_signal_data_list[layer.layer_id] = init_signal_layerwise( metadata.kv_signal_metadata, diff --git a/fastdeploy/model_executor/models/paddleformers/__init__.py b/fastdeploy/model_executor/models/paddleformers/__init__.py index 315b72b8775..c311f1b8000 100644 --- a/fastdeploy/model_executor/models/paddleformers/__init__.py +++ b/fastdeploy/model_executor/models/paddleformers/__init__.py @@ -40,6 +40,18 @@ class PaddleFormersForCausalLM(CausalLMMixin, PaddleFormersModelBase, ModelForCa def name(cls): return "PaddleFormersForCausalLM" + def __call__(self, inputs=None, forward_meta=None, **kwargs): + # Some model runners (e.g. xpu_model_runner.execute_model) call the model + # positionally, while support_graph_optimization installs a kwargs-only + # __call__. Bridge the two calling conventions. + if isinstance(inputs, dict): + kwargs.update(inputs) + elif inputs is not None: + kwargs["ids_remove_padding"] = inputs + if forward_meta is not None: + kwargs["forward_meta"] = forward_meta + return super().__call__(**kwargs) + if is_paddlefleet_available(): from .base_fleet import PaddleFleetModelBase diff --git a/fastdeploy/model_executor/models/paddleformers/base.py b/fastdeploy/model_executor/models/paddleformers/base.py index becd6e1f092..6ce0a0e7c97 100644 --- a/fastdeploy/model_executor/models/paddleformers/base.py +++ b/fastdeploy/model_executor/models/paddleformers/base.py @@ -20,6 +20,7 @@ from collections.abc import Iterable from typing import TYPE_CHECKING +import numpy as np import paddle from paddle import nn from paddleformers.nn.attention.interface import ALL_ATTENTION_FUNCTIONS @@ -807,6 +808,41 @@ def embed_input_ids(self, input_ids: paddle.Tensor) -> paddle.Tensor: inputs_embeds *= self.embed_scale return inputs_embeds + @staticmethod + def _build_position_ids_from_lods(forward_meta, num_tokens): + """RoPE positions for the XPU packed-token layout, or None on other devices. + + `xpu_pre_process` runs `adjust_batch`, which regroups the packed tokens into + [encoder requests ..., decoder requests ...] instead of batch order. Positions + therefore cannot be derived from batch_id_per_token / cu_seqlens_q, which still + describe the pre-`adjust_batch` order: in a mixed prefill+decode step that + mismatch gives every token a wrong position. + """ + len_info = getattr(forward_meta, "len_info_cpu", None) + if len_info is None: + return None + len_info = len_info.numpy() + enc_batch, dec_batch = int(len_info[0]), int(len_info[1]) + enc_lod = forward_meta.encoder_seq_lod_cpu.numpy() + dec_lod = forward_meta.decoder_seq_lod_cpu.numpy() + # prefix_len / decoder_context_len_cache hold seq_lens_decoder (already computed + # tokens) for the encoder and decoder requests respectively, in packed order. + enc_base = forward_meta.prefix_len_cpu.numpy() + dec_base = forward_meta.decoder_context_len_cache_cpu.numpy() + + positions = np.zeros([num_tokens], dtype="int64") + for i in range(enc_batch): + start, end = int(enc_lod[i]), int(enc_lod[i + 1]) + base = int(enc_base[i]) + positions[start:end] = np.arange(base, base + end - start, dtype="int64") + enc_tokens = int(enc_lod[enc_batch]) + for i in range(dec_batch): + start = enc_tokens + int(dec_lod[i]) + end = enc_tokens + int(dec_lod[i + 1]) + base = int(dec_base[i]) + positions[start:end] = np.arange(base, base + end - start, dtype="int64") + return paddle.to_tensor(positions) + @paddle.no_grad() def forward( self, @@ -829,7 +865,8 @@ def forward( batch_id_per_token = forward_meta.batch_id_per_token # [num_tokens] seq_lens_decoder = forward_meta.seq_lens_decoder # [batch_size, 1] - if batch_id_per_token is not None and seq_lens_decoder is not None: + position_ids = self._build_position_ids_from_lods(forward_meta, num_tokens) + if position_ids is None and batch_id_per_token is not None and seq_lens_decoder is not None: decoder_offsets = seq_lens_decoder.squeeze(-1) # [batch_size] token_decoder_offsets = paddle.index_select(decoder_offsets, batch_id_per_token, axis=0) # [num_tokens] @@ -841,7 +878,7 @@ def forward( else: relative_positions = paddle.zeros([num_tokens], dtype="int64") position_ids = token_decoder_offsets.astype("int64") + relative_positions - else: + elif position_ids is None: position_ids = paddle.arange(num_tokens, dtype="int64") if seq_lens_decoder is not None: position_ids = position_ids + seq_lens_decoder[0, 0].astype("int64") diff --git a/tests/model_executor/test_paddleformers_base.py b/tests/model_executor/test_paddleformers_base.py index d762e883566..1e77981d4fb 100644 --- a/tests/model_executor/test_paddleformers_base.py +++ b/tests/model_executor/test_paddleformers_base.py @@ -44,6 +44,7 @@ ) from fastdeploy.model_executor.layers.normalization import RMSNorm from fastdeploy.model_executor.models.paddleformers.base import ( + PaddleFormersModelBase, PaddleFormersRMSNormWrapper, getattr_iter, maybe_prefix, @@ -2633,5 +2634,58 @@ def set_value(self, value): layer_torch.weight_loader(param, fused_weight, None) +class TestBuildPositionIdsFromLods: + """position_ids must follow the XPU packed-token layout produced by adjust_batch. + + On XPU, `xpu_pre_process` calls `adjust_batch`, which regroups the packed tokens + into [encoder requests ..., decoder requests ...] instead of batch order, so + positions may not be derived from batch_id_per_token / cu_seqlens_q. + """ + + @staticmethod + def _forward_meta(enc_lens, enc_prefix, dec_cached): + enc_lod = [0] + for n in enc_lens: + enc_lod.append(enc_lod[-1] + n) + dec_lod = [0] + for _ in dec_cached: + dec_lod.append(dec_lod[-1] + 1) + + def cpu(values): + return paddle.to_tensor(values, dtype="int32", place=paddle.CPUPlace()) + + return SimpleNamespace( + len_info_cpu=cpu([len(enc_lens), len(dec_cached), enc_lod[-1]]), + encoder_seq_lod_cpu=cpu(enc_lod), + decoder_seq_lod_cpu=cpu(dec_lod), + prefix_len_cpu=cpu(enc_prefix or [0]), + decoder_context_len_cache_cpu=cpu(dec_cached or [0]), + ) + + def _positions(self, enc_lens, enc_prefix, dec_cached): + num_tokens = sum(enc_lens) + len(dec_cached) + forward_meta = self._forward_meta(enc_lens, enc_prefix, dec_cached) + out = PaddleFormersModelBase._build_position_ids_from_lods(forward_meta, num_tokens) + return out.numpy().tolist() + + def test_returns_none_without_xpu_metadata(self): + assert PaddleFormersModelBase._build_position_ids_from_lods(SimpleNamespace(), 4) is None + + def test_pure_prefill(self): + assert self._positions([3, 2], [0, 0], []) == [0, 1, 2, 0, 1] + + def test_pure_decode(self): + assert self._positions([], [], [7, 12]) == [7, 12] + + def test_mixed_prefill_and_decode(self): + # One decoding request (14 tokens cached) plus one 3-token prefill: adjust_batch + # puts the encoder tokens first, so the decode position comes last. + assert self._positions([3], [0], [14]) == [0, 1, 2, 14] + + def test_prefill_with_prefix_cache(self): + # 5 cached tokens (prefix cache / chunked prefill) then 2 new tokens. + assert self._positions([2], [5], []) == [5, 6] + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 8048f8aeb256da53e6f57a98c127e192cdc94efa Mon Sep 17 00:00:00 2001 From: mayang02 Date: Tue, 25 Aug 2026 09:56:21 +0800 Subject: [PATCH 2/2] Address code review: unify identity RoPE and harden LOD handling - Move _get_identity_rotary_embs to the AttentionBackend base class with layout-adaptive indexing (fixes the multimodal [batch, 2, ...] layout) - Replace the inline identity RoPE block in XPUAttentionBackend with the base class method; drop the duplicated copies in append_attn_backend and decode_unified_attention_backend - Add None guards for optional LOD fields in _build_position_ids_from_lods - Add docstrings to the new test methods --- .../layers/attention/append_attn_backend.py | 25 ------------- .../attention/base_attention_backend.py | 35 +++++++++++++++++++ .../decode_unified_attention_backend.py | 25 ------------- .../layers/backends/xpu/attention.py | 12 +------ .../models/paddleformers/base.py | 20 ++++++++--- .../model_executor/test_paddleformers_base.py | 5 +++ 6 files changed, 56 insertions(+), 66 deletions(-) diff --git a/fastdeploy/model_executor/layers/attention/append_attn_backend.py b/fastdeploy/model_executor/layers/attention/append_attn_backend.py index 57467858dc0..3f27af57320 100644 --- a/fastdeploy/model_executor/layers/attention/append_attn_backend.py +++ b/fastdeploy/model_executor/layers/attention/append_attn_backend.py @@ -258,31 +258,6 @@ def get_attention_meta(self) -> AttentionMetadata: """get_attention_meta""" return self.attention_metadata - def _get_identity_rotary_embs(self, original_rotary_embs: paddle.Tensor) -> paddle.Tensor: - """ - Create identity rotary embeddings (cos=1, sin=0) that make RoPE a no-op. - - This is used when RoPE has already been applied externally (e.g., by PaddleFormers). - The identity transformation ensures: x * cos(0) + y * sin(0) = x, preserving the input. - - NOTE: Shape can change between prefill/decode, so we check if cached shape matches. - """ - # Check if we need to recreate (shape mismatch or not cached) - need_recreate = ( - not hasattr(self, "_identity_rotary_embs") - or self._identity_rotary_embs is None - or self._identity_rotary_embs.shape != original_rotary_embs.shape - ) - - if need_recreate: - # Create identity RoPE: cos=1, sin=0 - identity = paddle.zeros_like(original_rotary_embs) - identity[0] = 1.0 # cos = 1 - identity[1] = 0.0 # sin = 0 - self._identity_rotary_embs = identity - - return self._identity_rotary_embs - def get_kv_cache_shape( self, max_num_blocks: int, diff --git a/fastdeploy/model_executor/layers/attention/base_attention_backend.py b/fastdeploy/model_executor/layers/attention/base_attention_backend.py index 6123408cac2..878548a802d 100644 --- a/fastdeploy/model_executor/layers/attention/base_attention_backend.py +++ b/fastdeploy/model_executor/layers/attention/base_attention_backend.py @@ -49,6 +49,41 @@ def init_attention_metadata(self, forward_meta: ForwardMeta): """Initialize the forward metadata.""" raise NotImplementedError + def _get_identity_rotary_embs(self, original_rotary_embs: paddle.Tensor) -> paddle.Tensor: + """ + Create identity rotary embeddings (cos=1, sin=0) that make RoPE a no-op. + + This is used when RoPE has already been applied externally (e.g., by PaddleFormers). + The identity transformation ensures: x * cos(0) + y * sin(0) = x, preserving the input. + + Text models pack rotary embs as [2, batch, seq, 1, head_dim] (axis 0 = [cos, sin]), + while multimodal models use [batch, 2, 1, max_len, 1, head_dim] (axis 1 = [cos, sin]), + so the cos/sin axis is located by shape. + + NOTE: Shape can change between prefill/decode, so we check if cached shape matches. + """ + # Check if we need to recreate (shape mismatch or not cached) + need_recreate = ( + not hasattr(self, "_identity_rotary_embs") + or self._identity_rotary_embs is None + or self._identity_rotary_embs.shape != original_rotary_embs.shape + ) + + if need_recreate: + # Create identity RoPE: cos=1, sin=0 + identity = paddle.zeros_like(original_rotary_embs) + if identity.shape[0] != 2 and len(identity.shape) > 1 and identity.shape[1] == 2: + # Multimodal layout: [batch, 2, 1, max_len, 1, head_dim] + identity[:, 0] = 1.0 # cos = 1 + identity[:, 1] = 0.0 # sin = 0 + else: + # Text layout: [2, batch, seq, 1, head_dim] + identity[0] = 1.0 # cos = 1 + identity[1] = 0.0 # sin = 0 + self._identity_rotary_embs = identity + + return self._identity_rotary_embs + def create_kv_cache( self, num_layers: int, diff --git a/fastdeploy/model_executor/layers/attention/decode_unified_attention_backend.py b/fastdeploy/model_executor/layers/attention/decode_unified_attention_backend.py index 71ae5ec76e2..51afd6d7631 100644 --- a/fastdeploy/model_executor/layers/attention/decode_unified_attention_backend.py +++ b/fastdeploy/model_executor/layers/attention/decode_unified_attention_backend.py @@ -186,31 +186,6 @@ def get_attention_meta(self) -> AttentionMetadata: """get_attention_meta""" return self.attention_metadata - def _get_identity_rotary_embs(self, original_rotary_embs: paddle.Tensor) -> paddle.Tensor: - """ - Create identity rotary embeddings (cos=1, sin=0) that make RoPE a no-op. - - This is used when RoPE has already been applied externally (e.g., by PaddleFormers). - The identity transformation ensures: x * cos(0) + y * sin(0) = x, preserving the input. - - NOTE: Shape can change between prefill/decode, so we check if cached shape matches. - """ - # Check if we need to recreate (shape mismatch or not cached) - need_recreate = ( - not hasattr(self, "_identity_rotary_embs") - or self._identity_rotary_embs is None - or self._identity_rotary_embs.shape != original_rotary_embs.shape - ) - - if need_recreate: - # Create identity RoPE: cos=1, sin=0 - identity = paddle.zeros_like(original_rotary_embs) - identity[0] = 1.0 # cos = 1 - identity[1] = 0.0 # sin = 0 - self._identity_rotary_embs = identity - - return self._identity_rotary_embs - def get_kv_cache_shape( self, max_num_blocks: int, diff --git a/fastdeploy/model_executor/layers/backends/xpu/attention.py b/fastdeploy/model_executor/layers/backends/xpu/attention.py index 8d349148e2e..689437d1657 100644 --- a/fastdeploy/model_executor/layers/backends/xpu/attention.py +++ b/fastdeploy/model_executor/layers/backends/xpu/attention.py @@ -173,17 +173,7 @@ def forward_mixed( # (forward_meta.rope_already_applied=True), so feed identity RoPE (cos=1, sin=0) # to block_attn to avoid rotating twice. Aligned with append_attn_backend. if getattr(forward_meta, "rope_already_applied", False) and metadata.rotary_embs is not None: - need_recreate = ( - not hasattr(self, "_identity_rotary_embs") - or self._identity_rotary_embs is None - or self._identity_rotary_embs.shape != metadata.rotary_embs.shape - ) - if need_recreate: - identity = paddle.zeros_like(metadata.rotary_embs) - identity[0] = 1.0 - identity[1] = 0.0 - self._identity_rotary_embs = identity - metadata.rotary_embs = self._identity_rotary_embs + metadata.rotary_embs = self._get_identity_rotary_embs(metadata.rotary_embs) if self.pd_disaggregation_mode == "per_query": metadata.kv_signal_data_list[layer.layer_id] = init_signal_layerwise( metadata.kv_signal_metadata, diff --git a/fastdeploy/model_executor/models/paddleformers/base.py b/fastdeploy/model_executor/models/paddleformers/base.py index 6ce0a0e7c97..cee774fe9a6 100644 --- a/fastdeploy/model_executor/models/paddleformers/base.py +++ b/fastdeploy/model_executor/models/paddleformers/base.py @@ -819,16 +819,26 @@ def _build_position_ids_from_lods(forward_meta, num_tokens): mismatch gives every token a wrong position. """ len_info = getattr(forward_meta, "len_info_cpu", None) - if len_info is None: + enc_lod_cpu = getattr(forward_meta, "encoder_seq_lod_cpu", None) + dec_lod_cpu = getattr(forward_meta, "decoder_seq_lod_cpu", None) + enc_base_cpu = getattr(forward_meta, "prefix_len_cpu", None) + dec_base_cpu = getattr(forward_meta, "decoder_context_len_cache_cpu", None) + if ( + len_info is None + or enc_lod_cpu is None + or dec_lod_cpu is None + or enc_base_cpu is None + or dec_base_cpu is None + ): return None len_info = len_info.numpy() enc_batch, dec_batch = int(len_info[0]), int(len_info[1]) - enc_lod = forward_meta.encoder_seq_lod_cpu.numpy() - dec_lod = forward_meta.decoder_seq_lod_cpu.numpy() + enc_lod = enc_lod_cpu.numpy() + dec_lod = dec_lod_cpu.numpy() # prefix_len / decoder_context_len_cache hold seq_lens_decoder (already computed # tokens) for the encoder and decoder requests respectively, in packed order. - enc_base = forward_meta.prefix_len_cpu.numpy() - dec_base = forward_meta.decoder_context_len_cache_cpu.numpy() + enc_base = enc_base_cpu.numpy() + dec_base = dec_base_cpu.numpy() positions = np.zeros([num_tokens], dtype="int64") for i in range(enc_batch): diff --git a/tests/model_executor/test_paddleformers_base.py b/tests/model_executor/test_paddleformers_base.py index 1e77981d4fb..091f280abae 100644 --- a/tests/model_executor/test_paddleformers_base.py +++ b/tests/model_executor/test_paddleformers_base.py @@ -2669,20 +2669,25 @@ def _positions(self, enc_lens, enc_prefix, dec_cached): return out.numpy().tolist() def test_returns_none_without_xpu_metadata(self): + """Return None when XPU LOD metadata is absent (non-XPU devices).""" assert PaddleFormersModelBase._build_position_ids_from_lods(SimpleNamespace(), 4) is None def test_pure_prefill(self): + """Pure prefill positions start from 0 within each request.""" assert self._positions([3, 2], [0, 0], []) == [0, 1, 2, 0, 1] def test_pure_decode(self): + """Pure decode positions equal the cached token count per request.""" assert self._positions([], [], [7, 12]) == [7, 12] def test_mixed_prefill_and_decode(self): + """Mixed step packs encoder tokens first, decode position comes last.""" # One decoding request (14 tokens cached) plus one 3-token prefill: adjust_batch # puts the encoder tokens first, so the decode position comes last. assert self._positions([3], [0], [14]) == [0, 1, 2, 14] def test_prefill_with_prefix_cache(self): + """Prefill with prefix cache starts numbering at the cached length.""" # 5 cached tokens (prefix cache / chunked prefill) then 2 new tokens. assert self._positions([2], [5], []) == [5, 6]