[core] move latent upscalers and condition embedders out of pipelines

docs/source/en/api/pipelines/ltx2.md

@@ -39,7 +39,7 @@ Sample usage of text-to-video two stages pipeline
 import torch
 from diffusers import FlowMatchEulerDiscreteScheduler
 from diffusers.pipelines.ltx2 import LTX2Pipeline, LTX2LatentUpsamplePipeline
-from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
+from diffusers.models.latent_upscaler import LTX2LatentUpsamplerModel
 from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES
 from diffusers.utils import encode_video
 
@@ -125,7 +125,7 @@ Fastest two-stages generation pipeline using a distilled checkpoint.
 ```py
 import torch
 from diffusers.pipelines.ltx2 import LTX2Pipeline, LTX2LatentUpsamplePipeline
-from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
+from diffusers.models.latent_upscaler import LTX2LatentUpsamplerModel
 from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES
 from diffusers.utils import encode_video
 
@@ -203,7 +203,7 @@ You can use `LTX2ConditionPipeline` to specify image and/or video conditions at
 ```py
 import torch
 from diffusers import LTX2ConditionPipeline, LTX2LatentUpsamplePipeline
-from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
+from diffusers.models.latent_upscaler import LTX2LatentUpsamplerModel
 from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition
 from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES
 from diffusers.utils import encode_video
@@ -569,7 +569,7 @@ Stage 1 runs at half the target resolution, the upsampler doubles it, and stage
 ```py
 import torch
 from diffusers.pipelines.ltx2 import LTX2Pipeline, LTX2LatentUpsamplePipeline
-from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
+from diffusers.models.latent_upscaler import LTX2LatentUpsamplerModel
 from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES
 from diffusers.utils import encode_video
 
@@ -1119,7 +1119,7 @@ from diffusers import (
     LTX2LatentUpsamplePipeline,
     LTXEulerAncestralRFScheduler,
 )
-from diffusers.pipelines.ltx2 import LTX2LatentUpsamplerModel
+from diffusers.models.latent_upscaler import LTX2LatentUpsamplerModel
 from diffusers.pipelines.ltx2.utils import trim_canvas
 from diffusers.pipelines.ltx2.pipeline_ltx2_condition import LTX2VideoCondition
 from diffusers.pipelines.ltx2.utils import STAGE_2_DISTILLED_SIGMA_VALUES

docs/source/en/api/pipelines/ltx_video.md

@@ -334,7 +334,7 @@ export_to_video(video, "output.mp4", fps=24)
   import torch
   from diffusers import LTXConditionPipeline, LTXLatentUpsamplePipeline
   from diffusers.pipelines.ltx.pipeline_ltx_condition import LTXVideoCondition
-  from diffusers.pipelines.ltx.modeling_latent_upsampler import LTXLatentUpsamplerModel
+  from diffusers.models.latent_upscaler import LTXLatentUpsamplerModel
   from diffusers.utils import export_to_video, load_video
 
   pipeline = LTXConditionPipeline.from_pretrained("Lightricks/LTX-Video-0.9.8-13B-distilled", dtype=torch.bfloat16)

scripts/convert_ace_step_to_diffusers.py

@@ -330,6 +330,7 @@ def _rename_attn_keys(key: str) -> str:
     from transformers import AutoModel, AutoTokenizer
 
     from diffusers import (
+        AceStepConditionEncoder,
         AceStepPipeline,
         AceStepTransformer1DModel,
         AutoencoderOobleck,
@@ -338,7 +339,6 @@ def _rename_attn_keys(key: str) -> str:
     from diffusers.pipelines.ace_step import (
         AceStepAudioTokenDetokenizer,
         AceStepAudioTokenizer,
-        AceStepConditionEncoder,
     )
 
     # Drop metadata keys — they're re-populated by `save_pretrained` at save time.

scripts/convert_ltx2_to_diffusers.py

@@ -14,14 +14,14 @@
     AutoencoderKLLTX2Video,
     FlowMatchEulerDiscreteScheduler,
     LTX2LatentUpsamplePipeline,
+    LTX2LatentUpsamplerModel,
     LTX2Pipeline,
+    LTX2TextConnectors,
     LTX2VideoDiffusionDecoderModel,
     LTX2VideoTransformer3DModel,
 )
 from diffusers.pipelines.ltx2 import (
     LTX2DurationHead,
-    LTX2LatentUpsamplerModel,
-    LTX2TextConnectors,
     LTX2Vocoder,
     LTX2VocoderWithBWE,
 )

scripts/convert_ltx_to_diffusers.py

@@ -15,7 +15,7 @@
     LTXPipeline,
     LTXVideoTransformer3DModel,
 )
-from diffusers.pipelines.ltx.modeling_latent_upsampler import LTXLatentUpsamplerModel
+from diffusers.models.latent_upscaler import LTXLatentUpsamplerModel
 
 
 def remove_keys_(key: str, state_dict: Dict[str, Any]):

scripts/smoke_test_relocated_pretrained.py

@@ -0,0 +1,109 @@
+import os
+import sys
+import traceback
+import warnings
+
+import torch
+
+import diffusers
+
+
+# Stable Audio 3 is not published in diffusers format; convert it locally with
+# `scripts/convert_stable_audio_3_to_diffusers.py` and point `SA3_CHECKPOINT` at the output directory.
+SA3_CHECKPOINT = os.environ.get("SA3_CHECKPOINT", "stabilityai/stable-audio-3-medium")
+
+
+# (class_name, repo, subfolder, kwargs) — subfolder=None loads from the repo root.
+MODEL_CHECKPOINTS = [
+    ("CLIPImageProjection", "anhnct/Gligen_Text_Image", "image_project", {}),
+    ("AudioLDM2ProjectionModel", "cvssp/audioldm2", "projection_model", {}),
+    ("StableAudioProjectionModel", "stabilityai/stable-audio-open-1.0", "projection_model", {}),
+    ("StableAudio3DurationEmbedder", SA3_CHECKPOINT, "duration_embedder", {}),
+    # black-forest-labs/FLUX.1-Redux-dev is gated: requires accepting the license + `hf auth login`.
+    ("ReduxImageEncoder", "black-forest-labs/FLUX.1-Redux-dev", "image_embedder", {"torch_dtype": torch.bfloat16}),
+    ("LTXLatentUpsamplerModel", "Lightricks/ltxv-spatial-upscaler-0.9.7", "latent_upsampler", {}),
+    ("LTX2LatentUpsamplerModel", "Lightricks/LTX-2", "latent_upsampler", {"torch_dtype": torch.bfloat16}),
+    ("LTX2TextConnectors", "Lightricks/LTX-2", "connectors", {"torch_dtype": torch.bfloat16}),
+    ("AceStepConditionEncoder", "ACE-Step/acestep-v15-xl-turbo-diffusers", "condition_encoder", {}),
+]
+
+
+# (pipeline_class_name, repo, kwargs). These `model_index.json` files reference the relocated classes by their old
+# pipeline folder (e.g. `["ltx2", "LTX2TextConnectors"]`).
+PIPELINE_CHECKPOINTS = [
+    ("StableDiffusionGLIGENTextImagePipeline", "anhnct/Gligen_Text_Image", {"torch_dtype": torch.float16}),
+    ("AudioLDM2Pipeline", "cvssp/audioldm2", {"torch_dtype": torch.float16}),
+    ("StableAudioPipeline", "stabilityai/stable-audio-open-1.0", {"torch_dtype": torch.float16}),
+    ("StableAudio3Pipeline", SA3_CHECKPOINT, {"torch_dtype": torch.float16}),
+    ("FluxPriorReduxPipeline", "black-forest-labs/FLUX.1-Redux-dev", {"torch_dtype": torch.bfloat16}),
+    ("LTXLatentUpsamplePipeline", "Lightricks/ltxv-spatial-upscaler-0.9.7", {"torch_dtype": torch.bfloat16}),
+    ("LTX2Pipeline", "Lightricks/LTX-2", {"torch_dtype": torch.bfloat16}),
+    ("AceStepPipeline", "ACE-Step/acestep-v15-xl-turbo-diffusers", {"torch_dtype": torch.bfloat16}),
+    (
+        "SanaWMLTX2Refiner",
+        "Efficient-Large-Model/SANA-WM_bidirectional-diffusers-refiner",
+        {"torch_dtype": torch.bfloat16},
+    ),
+]
+
+
+def _try_load(class_name: str, repo: str, kwargs: dict, subfolder: str | None = None) -> str | None:
+    """Load the checkpoint; return None on success, or the failure reason."""
+    try:
+        cls = getattr(diffusers, class_name)
+        load_kwargs = dict(kwargs)
+        if subfolder is not None:
+            load_kwargs["subfolder"] = subfolder
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always")
+            cls.from_pretrained(repo, **load_kwargs)
+    except Exception:
+        return traceback.format_exc()
+
+    relocation_warnings = [
+        str(w.message)
+        for w in caught
+        if issubclass(w.category, FutureWarning) and "diffusers.pipelines." in str(w.message)
+    ]
+    if relocation_warnings:
+        return "Loading emitted relocation deprecation warning(s):\n" + "\n".join(relocation_warnings)
+    return None
+
+
+def main() -> int:
+    failures: list[tuple[str, str, str]] = []  # (class_name, target, reason)
+
+    print("=== Model components ===")
+    for class_name, repo, subfolder, kwargs in MODEL_CHECKPOINTS:
+        target = repo + (f":{subfolder}" if subfolder else "")
+        print(f"\n[{class_name}]  {target}")
+        err = _try_load(class_name, repo, kwargs, subfolder)
+        if err is None:
+            print("  PASS")
+        else:
+            print("  FAIL")
+            failures.append((class_name, target, err))
+
+    print("\n=== Pipelines ===")
+    for class_name, repo, kwargs in PIPELINE_CHECKPOINTS:
+        print(f"\n[{class_name}]  {repo}")
+        err = _try_load(class_name, repo, kwargs)
+        if err is None:
+            print("  PASS")
+        else:
+            print("  FAIL")
+            failures.append((class_name, repo, err))
+
+    print()
+    if failures:
+        print(f"FAILED: {len(failures)} case(s).\n")
+        for class_name, target, err in failures:
+            print(f"--- {class_name}  ({target}) ---")
+            print(err)
+        return 1
+    print("OK: all cases passed.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())