diff --git a/src/diffusers/modular_pipelines/wan/before_denoise.py b/src/diffusers/modular_pipelines/wan/before_denoise.py index 6b0874037b0d..f4f67965a0d8 100644 --- a/src/diffusers/modular_pipelines/wan/before_denoise.py +++ b/src/diffusers/modular_pipelines/wan/before_denoise.py @@ -252,7 +252,7 @@ def __call__(self, components: WanModularPipeline, state: PipelineState) -> Pipe self.check_inputs(components, block_state) block_state.batch_size = block_state.prompt_embeds.shape[0] - block_state.dtype = block_state.prompt_embeds.dtype + block_state.dtype = components.transformer.dtype _, seq_len, _ = block_state.prompt_embeds.shape block_state.prompt_embeds = block_state.prompt_embeds.repeat(1, block_state.num_videos_per_prompt, 1) diff --git a/src/diffusers/modular_pipelines/wan/modular_pipeline.py b/src/diffusers/modular_pipelines/wan/modular_pipeline.py index a360440c9251..5948faa68fda 100644 --- a/src/diffusers/modular_pipelines/wan/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/wan/modular_pipeline.py @@ -70,14 +70,14 @@ def patch_size_spatial(self): def vae_scale_factor_spatial(self): vae_scale_factor = 8 if hasattr(self, "vae") and self.vae is not None: - vae_scale_factor = 2 ** len(self.vae.temperal_downsample) + vae_scale_factor = self.vae.config.scale_factor_spatial return vae_scale_factor @property def vae_scale_factor_temporal(self): vae_scale_factor = 4 if hasattr(self, "vae") and self.vae is not None: - vae_scale_factor = 2 ** sum(self.vae.temperal_downsample) + vae_scale_factor = self.vae.config.scale_factor_temporal return vae_scale_factor @property diff --git a/src/diffusers/pipelines/wan/image_processor.py b/src/diffusers/pipelines/wan/image_processor.py index fa18150fcc6e..b2f026c885f6 100644 --- a/src/diffusers/pipelines/wan/image_processor.py +++ b/src/diffusers/pipelines/wan/image_processor.py @@ -68,7 +68,17 @@ def __init__( do_convert_grayscale: bool = False, fill_color: str | float | tuple[float, ...] | None = 0, ): - super().__init__() + super().__init__( + do_resize=do_resize, + vae_scale_factor=vae_scale_factor, + vae_latent_channels=vae_latent_channels, + resample=resample, + reducing_gap=reducing_gap, + do_normalize=do_normalize, + do_binarize=do_binarize, + do_convert_rgb=do_convert_rgb, + do_convert_grayscale=do_convert_grayscale, + ) if do_convert_rgb and do_convert_grayscale: raise ValueError( "`do_convert_rgb` and `do_convert_grayscale` can not both be set to `True`," diff --git a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py index 8061f67ab6b9..4bf7ed807ecd 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_i2v.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_i2v.py @@ -351,8 +351,10 @@ def check_inputs( raise ValueError( "Provide either `image` or `prompt_embeds`. Cannot leave both `image` and `image_embeds` undefined." ) - if image is not None and not isinstance(image, torch.Tensor) and not isinstance(image, PIL.Image.Image): - raise ValueError(f"`image` has to be of type `torch.Tensor` or `PIL.Image.Image` but is {type(image)}") + if image is not None and not isinstance(image, (torch.Tensor, PIL.Image.Image, list, tuple)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image`, or a list of them but is {type(image)}" + ) if height % 16 != 0 or width % 16 != 0: raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.") @@ -453,7 +455,7 @@ def prepare_latents( latent_condition = torch.cat(latent_condition) else: latent_condition = retrieve_latents(self.vae.encode(video_condition), sample_mode="argmax") - latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1) + latent_condition = latent_condition.repeat_interleave(batch_size // latent_condition.shape[0], dim=0) latent_condition = latent_condition.to(dtype) latent_condition = (latent_condition - latents_mean) * latents_std @@ -698,7 +700,12 @@ def __call__( image_embeds = self.encode_image(image, device) else: image_embeds = self.encode_image([image, last_image], device) - image_embeds = image_embeds.repeat(batch_size, 1, 1) + if last_image is None: + image_embeds = image_embeds.repeat_interleave( + batch_size * num_videos_per_prompt // image_embeds.shape[0], dim=0 + ) + else: + image_embeds = image_embeds.repeat(batch_size, 1, 1) image_embeds = image_embeds.to(transformer_dtype) # 4. Prepare timesteps diff --git a/src/diffusers/pipelines/wan/pipeline_wan_vace.py b/src/diffusers/pipelines/wan/pipeline_wan_vace.py index b0896d382d67..6c7285750f41 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_vace.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_vace.py @@ -196,8 +196,8 @@ def __init__( scheduler=scheduler, ) self.register_to_config(boundary_ratio=boundary_ratio) - self.vae_scale_factor_temporal = 2 ** sum(self.vae.temperal_downsample) if getattr(self, "vae", None) else 4 - self.vae_scale_factor_spatial = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 + self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4 + self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8 self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds @@ -1020,8 +1020,8 @@ def __call__( self._current_timestep = None + latents = latents[:, :, num_reference_images:] if not output_type == "latent": - latents = latents[:, :, num_reference_images:] latents = latents.to(vae_dtype) latents_mean = ( torch.tensor(self.vae.config.latents_mean) diff --git a/src/diffusers/pipelines/wan/pipeline_wan_video2video.py b/src/diffusers/pipelines/wan/pipeline_wan_video2video.py index 8993475a2851..a782c79b6c5c 100644 --- a/src/diffusers/pipelines/wan/pipeline_wan_video2video.py +++ b/src/diffusers/pipelines/wan/pipeline_wan_video2video.py @@ -214,8 +214,8 @@ def __init__( scheduler=scheduler, ) - self.vae_scale_factor_temporal = 2 ** sum(self.vae.temperal_downsample) if getattr(self, "vae", None) else 4 - self.vae_scale_factor_spatial = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 + self.vae_scale_factor_temporal = self.vae.config.scale_factor_temporal if getattr(self, "vae", None) else 4 + self.vae_scale_factor_spatial = self.vae.config.scale_factor_spatial if getattr(self, "vae", None) else 8 self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) # Copied from diffusers.pipelines.wan.pipeline_wan.WanPipeline._get_t5_prompt_embeds